# -*- coding: utf-8 -*-
# @Author: MENG
# @Time  : 2022-3-18
import requests
from pyquery import PyQuery as pq
import time
import json
import pymongo
from requests.packages import urllib3
urllib3.disable_warnings()
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='zzsn@9988').caiji['人民网-习讲话数据库_copy']


# 习讲话数据库 新增数据
def get_content():
    art_content_dict = {}
    article_id_list = []
    headers = {
        'Proxy-Connection': 'keep-alive',
        'Accept': '*/*',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cookie': 'sfr=1; sso_c=0; __jsluid_h=5b9f09f6fdae46fadb89e1e02dca3238; wdcid=04fccdf5121158c0; wdses=72d07de4316a36a5; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg; wdlast=1646734820; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg'
    }
    for page in range(3, 0, -1):
        url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year=0&form=0&type=0&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
        try:
            resp_json = requests.request("GET", url, headers=headers, verify=False).json()
            time.sleep(0.2)
            data_list = resp_json['list']
        except:
            print('请求错误1')
            continue
        for data_dict in data_list[::-1]:
            article_id = data_dict['article_id']
            is_article_id = db_storage.find_one({'id': article_id})
            if is_article_id:
                continue
            title = data_dict['title']
            pub_time = data_dict['input_date']
            title_dict_list = db_storage.find({'title': title, 'is_repeat': ''})
            is_repeat = ''
            for title_dict in title_dict_list:
                pub_time1 = title_dict['publishDate']
                time_ = int(int(time.mktime(time.strptime(pub_time, "%Y-%m-%d %H:%M:%S"))) - int(time.mktime(time.strptime(pub_time1, "%Y-%m-%d %H:%M:%S"))))
                if -259200 < time_ < 259200:
                    is_repeat = '1'
                    break
            article_id_list.append(article_id)
            href = 'http://jhsjk.people.cn/article/' + article_id
            try:
                href_text = requests.request("GET", href, headers=headers, verify=False).content
                time.sleep(0.2)
                doc_href = pq(href_text)
                content_html1 = str(doc_href('.d2txt_con.clearfix'))
                content_html2 = str(doc_href('.editor.clearfix'))
            except:
                print('请求错误2')
                continue
            content_html = content_html1 + '\n' + content_html2
            content = pq(content_html).text()
            if content.strip() == '':
                print(href, '内容为空')
                continue
            origin = data_dict['origin_name']
            a_dict = {
                'id': article_id,
                'title': title,
                'author': '',
                'origin': origin,
                'content': content_html,
                'publishDate': pub_time,
                'sourceAddress': href,
                'tags': [],
                'is_repeat': is_repeat
            }
            art_content_dict[article_id] = a_dict
            db_a_dict = a_dict.copy()
            db_storage.insert_one(db_a_dict)
            if is_repeat == '':
                print(href)
    if article_id_list is []:
        return
    art_type_dict = {}
    result_lists = [
        ['类型', '讲话', '706', '69'], ['类型', '会议', '701', '178'], ['类型', '活动', '702', '63'], ['类型', '考察', '703', '72'],
        ['类型', '会见', '704', '174'], ['类型', '出访', '705', '188'], ['类型', '函电', '707', '194'], ['类型', '其他', '708', '203'],
        ['领域', '经济', '101', '18'], ['领域', '政治', '102', '21'], ['领域', '文化', '103', '14'], ['领域', '社会', '104', '15'],
        ['领域', '生态', '105', '7'], ['领域', '党建', '106', '9'], ['领域', '国防', '107', '6'], ['领域', '外交', '108', '50'],
    ]
    for result_list in result_lists:
        sort = result_list[0]
        sort_text = result_list[1]
        print(sort_text)
        if sort == '类型':
            form = result_list[2]
            type_ = '0'
        else:
            form = '0'
            type_ = result_list[2]
        # total_page = result_list[3]
        total_page = 2
        for page in range(1, int(total_page)):
            url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year=0&form={form}&type={type_}&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
            payload = {}
            try:
                resp_json = requests.request("GET", url, headers=headers, verify=False, data=payload).json()
                data_list = resp_json['list']
            except:
                print('请求错误3')
                continue
            for data_dict in data_list:
                article_id = data_dict['article_id']
                if article_id in article_id_list:
                    type_dict = {
                        "type": sort,
                        "name": sort_text,
                    }
                    type_list = art_type_dict.get(article_id)
                    if type_list is None:
                        type_lists = [type_dict]
                    else:
                        type_lists = type_list + [type_dict]
                    art_type_dict[article_id] = type_lists
    for key, value in art_content_dict.items():
        tags = art_type_dict.get(key)
        if tags is None:
            tags = []
        value['tags'] = tags
        post_dict = value
        db_storage.update_one({'id': post_dict['id']}, {'$set': {'tags': tags}})
        if post_dict['is_repeat'] == '1':
            continue
        try:
            del post_dict['is_repeat']
            # labels = []
            # for tags_dict in post_dict['tags']:
            #     labels_dict = {
            #         'abelRemarks': tags_dict.get('type'),
            #         'relationName': tags_dict.get('name'),
            #     }
            #     labels.append(labels_dict)
            # aaa_dict = {
            #     'sid': '1533647545473859586',
            #     'title': post_dict['title'],
            #     'content': '',
            #     'contentWithTag': post_dict['content'],
            #     'summary': '',
            #     'author': '',
            #     'origin': post_dict['origin'],
            #     'publishDate': post_dict['publishDate'],
            #     'sourceAddress': post_dict['sourceAddress'],
            #     'labels': labels
            # }
            post_url = 'http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak'
            headers = {'Content-Type': 'application/json'}
            resp_json = requests.post(url=post_url, headers=headers, verify=False, data=json.dumps(post_dict)).json()
            print('推送：', resp_json['msg'])
        except:
            print('数据传接口失败，正在重试！')
            time.sleep(5)
            db_storage.delete_one({'id': post_dict['id']})
            continue


if __name__ == '__main__':
    try:
        get_content()
    except Exception as e:
        pass





"""
#本地访问地址 http://192.168.1.190:8008/api/reptile/autoSaveXJPSpeak
#服务器地址   http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak

#请求方式： post


#请求体数据,JSON格式
{
	"id": "32396175",
	"title": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"author": "",
	"origin": "人民日报客户端",
	"content": "带标签",
	"publishDate": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"sourceAddress": "http://jhsjk.people.cn/article/32396175",
	"tags": [{
		"type": "类型",
		"name": "考察"
	}, {
		"type": "领域",
		"name": "文化"
	}]
}}
"""