import json
import threading
import time
import uuid

import redis
import requests
from retry import retry
from elasticsearch import Elasticsearch
from base import BaseCore
from obs import ObsClient
import fitz
from urllib.parse import unquote
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
baseCore = BaseCore.BaseCore()

# 使用连接池
# cnx_ = baseCore.pool_11.connection()
# cursor_ = cnx_.cursor()

cnx_ = baseCore.cnx_
cursor_ = cnx_.cursor()

lock = threading.Lock()
pathType = 'QYNotice/'
taskType = '企业研报/东方财富网'

pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6)

class EsMethod(object):

    def __init__(self):
        # 创建Elasticsearch对象，并提供账号信息
        self.es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'), timeout=300)
        self.index_name = 'researchreportdata'

    def queryatt(self,index_name,pnum):
       body = {

           "query": {
                  "bool": {
                      "must_not": [
                        {
                          "exists": {
                            "field": "content"
                          }
                        }
                      ],
                      "must": [
                        {
                          "match": {
                            "type": "1"
                          }
                        }
                      ]
                    }
                  },
              "sort": [
                    {
                      "createDate": {
                        "order": "desc"
                      }
                    }
             ],
           "track_total_hits": True,
           "size": 200,
           "from": pnum
       }

       filter_path = ['hits.hits._id',
                      'hits.total.value',
                      'hits.hits._source.title',
                      'hits.hits._source.sourceAddress',
                      'hits.hits._source.createDate',
                      'hits.hits._source.origin'
                      ]  # 字段2
       result = self.es.search(index=index_name
                               , doc_type='_doc'
                               , filter_path=filter_path
                               , body=body)
       # log.info(result)
       return result

def main(page, p, esMethod):
    redis_conn = redis.Redis(connection_pool=pool)
    result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
    total = result['hits']['total']['value']
    # if total == 0:
    #     log.info('++++已没有数据+++++')
    #     return
    try:
        msglist = result['hits']['hits']
    except:
        log.info(f'error-----{result}')
        return
    log.info(f'---第{page}页{len(msglist)}条数据----共{total}条数据----')

    for mms in msglist:
        id = mms['_id']
        title = mms['_source']['title']
        sourceAddress = mms['_source']['sourceAddress']
        origin = mms['_source']['origin']
        log.info(f'{id}--{title}--{origin}--{sourceAddress}---')
        if origin == 'SEC美国证券交易委员会':
            redis_conn.lrem('NianbaoUS:id', 0, id)
            redis_conn.lpush('NianbaoUS:id', id)
        else:
            redis_conn.lrem(f'NianbaoOT_{origin}:id', 0, id)
            redis_conn.lpush(f'NianbaoOT_{origin}:id', id)


def run_threads(num_threads,esMethod,j):
    threads = []

    for i in range(num_threads):
        page = j + i + 1
        p = j + i * 200
        thread = threading.Thread(target=main, args=(page, p, esMethod))

        threads.append(thread)
        thread.start()

    for thread in threads:
        thread.join()



if __name__ == "__main__":
    j = 0
    for i in range(10):
        esMethod = EsMethod()
        # result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
        # total = result['hits']['total']['value']
        # if total == 0:
        #     log.info('++++已没有数据+++++')
        #     break
        start = time.time()
        num_threads = 5
        run_threads(num_threads, esMethod, j)
        j += 1000

        log.info(f'5线程 每个处理200条数据 总耗时{time.time() - start}秒')