# -*- coding: utf-8 -*-
# @Author: MENG
# @Time  : 2022-3-18
import redis
import requests
from langid import langid
from pyquery import PyQuery as pq
import time
import json
import pymongo
from kafka import KafkaProducer
from requests.packages import urllib3
from datetime import datetime, timedelta
urllib3.disable_warnings()
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='zzsn@9988').ZZSN['人民网-习讲话数据库_copy']

def newsdata(art_content_dict,art_type_dict,dic_lables):
    for key, value in art_content_dict.items():
        labels = []
        tags = art_type_dict.get(key)
        if tags is None:
            tags = []
        value_new = value
        value_new['tags'] = tags
        # todo:lables映射
        for tag in tags:
            labelRemarks = tag['type']
            relationName = tag['name']
            item = labelRemarks + "|" + relationName
            item_value = dic_lables[item]
            labelMark = item_value.split("|")[0]
            relationId = item_value.split("|")[1]
            label = {
                "labelMark": labelMark,
                "labelRemarks": labelRemarks,
                "relationId": relationId,
                "relationName": relationName
            }
            labels.append(label)
        value_new['labels'] = labels
        value_new['subjectId'] = "1534423014825668610"
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        value_new['createDate'] = time_now
        value_new['checkStatus'] = "1"
        value_new['deleteFlag'] = "0"
        value_new['topNum'] = "0"
        value_new['summary'] = ""
        post_dict = value_new
        for i in range(5):
            try:
                db_storage.update_one({'id': post_dict['id']}, {'$set': {'tags': tags}})
                break
            except:
                time.sleep(2)
                continue
        if post_dict['is_repeat'] == '1':
            continue

        try:
            del post_dict['is_repeat']
            del post_dict['tags']
            # 发送kafka
            producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], max_request_size=1024 * 1024 * 20)
            kafka_result = producer.send("research_center_fourth",
                                         json.dumps(post_dict, ensure_ascii=False).encode('utf8'))

            print(kafka_result.get(timeout=10))

            dic_result = {
                'success': 'ture',
                'message': '操作成功',
                'code': '200',
            }
            print(dic_result)
            old_dic = post_dict
            try:
                del old_dic['contentWithTag']
                del old_dic['lang']
                del old_dic['labels']
                del old_dic['createDate']
                del old_dic['checkStatus']
                del old_dic['deleteFlag']
                del old_dic['topNum']
                del old_dic['summary']
                # post_url = 'http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak'
                # headers = {'Content-Type': 'application/json'}
                # resp_json = requests.post(url=post_url, headers=headers, verify=False, data=json.dumps(old_dic)).json()
                # print('推送：', resp_json['msg'])
            except:
                print('数据传接口失败，正在重试！')

        except Exception as e:
            dic_result = {
                'success': 'false',
                'message': '操作失败',
                'code': '204',
                'e': e
            }
            print(dic_result)
            time.sleep(5)
            db_storage.delete_one({'id': post_dict['id']})
            continue

# 习讲话数据库 新增数据
def get_content():
    art_content_dict = {}
    article_id_list = []
    headers = {
        'Proxy-Connection': 'keep-alive',
        'Accept': '*/*',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cookie': 'sfr=1; sso_c=0; __jsluid_h=5b9f09f6fdae46fadb89e1e02dca3238; wdcid=04fccdf5121158c0; wdses=72d07de4316a36a5; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg; wdlast=1646734820; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg'
    }
    for page in range(1, 0, -1):
        url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year=0&form=0&type=0&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
        try:
            resp_json = requests.request("GET", url, headers=headers, verify=False).json()
            time.sleep(0.2)
            data_list = resp_json['list']
        except:
            print('请求错误1')
            continue
        for data_dict in data_list[::-1]:
            article_id = data_dict['article_id']
            is_article_id = db_storage.find_one({'id': article_id})
            if is_article_id:
                continue
            title = data_dict['title']
            pub_time = data_dict['input_date']
            current_date = datetime.now()
            yesterday = current_date - timedelta(days=1)
            # 格式化日期
            yesterday_date = yesterday.strftime("%Y-%m-%d")
            if pub_time <= yesterday_date:
                continue
            title_dict_list = db_storage.find({'title': title, 'is_repeat': ''})
            is_repeat = ''
            for title_dict in title_dict_list:
                pub_time1 = title_dict['publishDate']
                time_ = int(int(time.mktime(time.strptime(pub_time, "%Y-%m-%d %H:%M:%S"))) - int(time.mktime(time.strptime(pub_time1, "%Y-%m-%d %H:%M:%S"))))
                if -259200 < time_ < 259200:
                    is_repeat = '1'
                    break
            article_id_list.append(article_id)
            href = 'http://jhsjk.people.cn/article/' + article_id
            try:
                href_text = requests.request("GET", href, headers=headers, verify=False).content
                time.sleep(0.2)
                doc_href = pq(href_text)
                content_html1 = str(doc_href('.d2txt_con.clearfix'))
                content_html2 = str(doc_href('.editor.clearfix'))
            except:
                print('请求错误2')
                continue
            content_html = content_html1 + '\n' + content_html2
            content = pq(content_html).text()
            lang = langid.classify(content)
            if lang == '':
                lang = 'cn'
            if lang[0] == '':
                lang = 'cn'
            else:
                lang = lang[0]
            if content.strip() == '':
                print(href, '内容为空')
                continue
            origin = data_dict['origin_name']
            a_dict = {
                'id': "1534423014825668610" + article_id,
                'title': title,
                'author': '',
                'origin': origin,
                'contentWithTag': content_html,
                'content': content,
                'publishDate': pub_time,
                'sourceAddress': href,
                'tags': [],
                'lang': lang,
                'is_repeat': is_repeat
            }
            art_content_dict[article_id] = a_dict
            db_a_dict = a_dict.copy()
            db_storage.insert_one(db_a_dict)
            if is_repeat == '':
                print(href)
    if article_id_list is []:
        return
    art_type_dict = {}
    result_lists = [
        ['类型', '讲话', '706', '69'], ['类型', '会议', '701', '178'], ['类型', '活动', '702', '63'], ['类型', '考察', '703', '72'],
        ['类型', '会见', '704', '174'], ['类型', '出访', '705', '188'], ['类型', '函电', '707', '194'], ['类型', '其他', '708', '203'],
        ['时间', '2023', '2023', '11'], ['时间', '2022', '2022', '10'], ['时间', '2021', '2021', '9'],
        ['时间', '2019', '2019', '8'],
        ['时间', '2018', '2018', '7'], ['时间', '2017', '2017', '6'], ['时间', '2016', '2016', '5'],
        ['时间', '2015', '2015', '4'],
        ['时间', '2014', '2014', '3'], ['时间', '2013', '2013', '2'], ['时间', '2012', '2012', '1'],
        ['领域', '经济', '101', '18'], ['领域', '政治', '102', '21'], ['领域', '文化', '103', '14'], ['领域', '社会', '104', '15'],
        ['领域', '生态', '105', '7'], ['领域', '党建', '106', '9'], ['领域', '国防', '107', '6'], ['领域', '外交', '108', '50'],

    ]
    dic_lables = {
        "类型|讲话": "important_speech_type|1700334917807710209",
        "类型|会议": "important_speech_type|1700334936166178818",
        "类型|活动": "important_speech_type|1700334960560250881",
        "类型|考察": "important_speech_type|1700334978285379585",
        "类型|会见": "important_speech_type|1700335044605714433",
        "类型|出访": "important_speech_type|1700335078852206593",
        "类型|函电": "important_speech_type|1700335099689508866",
        "类型|其他": "important_speech_type|1700335118056366082",
        "时间|2012": "important_speech_time|1700334545970077697",
        "时间|2013": "important_speech_time|1700334647757447170",
        "时间|2014": "important_speech_time|1700334667915272194",
        "时间|2015": "important_speech_time|1700334686550564865",
        "时间|2016": "important_speech_time|1700334704925810689",
        "时间|2017": "important_speech_time|1700334722529304578",
        "时间|2018": "important_speech_time|1700334738320859137",
        "时间|2019": "important_speech_time|1700334758302523393",
        "时间|2020": "important_speech_time|1700334777827008514",
        "时间|2021": "important_speech_time|1700334797477322753",
        "时间|2022": "important_speech_time|1700334814468448258",
        "时间|2023": "important_speech_time|1700334832495566850",
        "领域|经济": "important_speech_area|1700335225803841537",
        "领域|政治": "important_speech_area|1700335248096567297",
        "领域|文化": "important_speech_area|1700335379638329345",
        "领域|社会": "important_speech_area|1700335412873994242",
        "领域|生态": "important_speech_area|1700335541211308033",
        "领域|党建": "important_speech_area|1700335587780665346",
        "领域|国防": "important_speech_area|1700335615895085058",
        "领域|外交": "important_speech_area|1700335820430319618",

    }

    for result_list in result_lists:
        sort = result_list[0]
        sort_text = result_list[1]
        print(sort_text)
        if sort == '类型':
            form = result_list[2]
            type_ = '0'
            year = '0'
        elif sort == '时间':
            form = '0'
            type_ = '0'
            year = result_list[2]
        else:
            form = '0'
            type_ = result_list[2]
            year = '0'
        # total_page = result_list[3]
        total_page = 10
        for page in range(1, int(total_page)):
            url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year={year}&form={form}&type={type_}&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
            payload = {}
            try:
                resp_json = requests.request("GET", url, headers=headers, verify=False, data=payload).json()
                data_list = resp_json['list']
            except:
                print('请求错误3')
                continue
            for data_dict in data_list:
                article_id = data_dict['article_id']
                if article_id in article_id_list:
                    type_dict = {
                        "type": sort,
                        "name": sort_text,
                    }
                    type_list = art_type_dict.get(article_id)
                    if type_list is None:
                        type_lists = [type_dict]
                    else:
                        type_lists = type_list + [type_dict]
                    new_lst = []

                    # 遍历原列表的字典元素
                    for item in type_lists:
                        # 如果字典元素不在新列表中，则添加到新列表中
                        if item not in new_lst:
                            new_lst.append(item)

                    art_type_dict[article_id] = new_lst

    newsdata(art_content_dict, art_type_dict, dic_lables)


if __name__ == '__main__':
    try:
        get_content()
    except Exception as e:
        print(e)
        pass





"""
#本地访问地址 http://192.168.1.190:8008/api/reptile/autoSaveXJPSpeak
#服务器地址   http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak

#请求方式： post


#请求体数据,JSON格式
{
	"id": "32396175",
	"title": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"author": "",
	"origin": "人民日报客户端",
	"content": "带标签",
	"publishDate": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"sourceAddress": "http://jhsjk.people.cn/article/32396175",
	"tags": [{
		"type": "类型",
		"name": "考察"
	}, {
		"type": "领域",
		"name": "文化"
	}]
}}
"""