import time

import requests
from bs4 import BeautifulSoup

import os
import pandas as pd
import numpy as np
from retry import retry

import BaseCore

baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()

from reits import Policy
policy = Policy()


topic = 'research_center_fourth'
webname = '海南省人民政府'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
}

@retry(tries=3, delay=1)
def getSoup(url):
    ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers, proxies=ip)
    if req.url == 'https://www.hainan.gov.cn/hainan/xhtml/404.html':
        return ''
    req.encoding = req.apparent_encoding
    soup = BeautifulSoup(req.text, 'html.parser')
    if soup:
        return soup
    else:
        raise Exception("重试")



def getPageSize(type):
    url = f'https://www.hainan.gov.cn/s?siteCode=4600000001&searchWord=REITs&column={type}&wordPlace=0&orderBy=1&startTime=&endTime=&isSecondSearch=undefined&pageSize=10&pageNum=0&timeStamp=0&labelHN=&uc=0&checkHandle=1&strFileType=0&countKey=%200&sonSiteCode=&areaSearchFlag=&secondSearchWords=&left_right_flag=1'
    try:
        soup = getSoup(url)
    except:
        return 0
    total = int(soup.find('div', class_='results-list').find('span').text.lstrip().strip())
    if total % 10 == 0:
        pageSize = int(total / 10)
    else:
        pageSize = int(total / 10) + 1
    return pageSize


def getContent(url, publishDate, num):
    try:
        soup = getSoup(url)
    except:
        return '', '', ''
    policy.paserUrl(soup, url)
    if soup == '':
        return '','',''
    try:
        writtenDate = soup.find('div', class_='zwgk_comr1').text.replace(' ', '').split('成文日期：')[1].split('标题')[
            0].lstrip().strip()
    except:
        writtenDate = None
    contentWithTag = soup.find('div', attrs={'id':'font'})

    try:
        scripts = contentWithTag.find_all('script')
        for script in scripts:
            script.decompose()
    except:
        pass
    try:
        styles = contentWithTag.find_all('style')
        for style in styles:
            style.decompose()
    except:
        pass
    try:
        content = contentWithTag.text.lstrip().strip()
    except:
        print(url)
    return writtenDate, content, contentWithTag


def getData(div, num):
    title = div.find('a', class_='titlec').get('title').replace('\n', '').replace('\r', '').lstrip().strip()
    href = div.find('a', class_='titlec').get('href')
    publishDate = div.find('span', class_='quily-con').text.lstrip().strip()
    origin = div.find('a', class_='address-con').text.lstrip().strip()
    try:
        table = div.find('div', class_='search-results').find('table').text
        organ = table.split('发文机关：')[1].split('文号：')[0].lstrip().strip()
        pub_hao = table.split('文号：')[1].lstrip().strip()
    except:
        organ = ''
        pub_hao = ''
    try:
        summary = div.find('p', class_='p-text-color').text.lstrip().strip()
    except:
        summary = ''
    writtenDate, content, contentWithTag = getContent(href, publishDate, num)
    if content == '':
        return
    contentWithTag_str = str(contentWithTag)
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    lang = baseCore.detect_language(content)
    dic_info = {
        'attachmentIds': [],
        'subjectId': '1729315113088765953',
        'lang': lang,
        'author': '',
        'content': content,
        'contentWithTag': contentWithTag_str,
        'deleteFlag': 0,
        'checkStatus': 1,
        'id': '1729315113088765953'+str(int(time.time())),
        'title': title,
        'publishDate': publishDate,
        'origin': origin,
        'sourceAddress': href,
        'writtenDate': writtenDate,
        'organ': organ,
        'topicClassification': '',
        'issuedNumber': pub_hao,
        'summary': summary,
        'createDate': time_now,
        'sid': '1729045345312100353'
    }
    try:
        baseCore.sendkafka(dic_info, topic)
        baseCore.r.sadd('REITs::' + webname, href)
        log.info(f'采集成功--{title}--{href}')
    except Exception as e:
        return
    return


def doJob():
    num = 1
    types = [2682,2677]
    for type in types:
        pageSize = getPageSize(type)
        for page in range(pageSize):
            url = f'https://www.hainan.gov.cn/s?siteCode=4600000001&searchWord=REITs&column={type}&wordPlace=0&orderBy=1&startTime=&endTime=&isSecondSearch=undefined&pageSize=10&pageNum={page}&timeStamp=0&labelHN=&uc=0&checkHandle=1&strFileType=0&countKey=%200&sonSiteCode=&areaSearchFlag=&secondSearchWords=&left_right_flag=1'
            try:
                soup = getSoup(url)
            except:
                continue
            div_list = soup.select('#showPage > div')
            del (div_list[-1])
            for div in div_list:
                href = div.find('a', class_='titlec').get('href')
                # 根据链接判重
                is_member = baseCore.r.sismember('REITs::' + webname, href)
                if is_member:
                    continue

                getData(div, num)
                num += 1



if __name__ == '__main__':
    doJob()
    baseCore.close()
