
import configparser
import csv
import glob
import os
import shutil
import time

import pandas as pd
import redis
import requests
from datetime import datetime

'''
海关下载数据类型和参数分类组合
CODE_TS #商品编码  ORIGIN_COUNTRY  #贸易伙伴 TRADE_MODE #贸易方式 TRADE_CO_PORT #收发货地址
1.设置进出口类型 （默认进口，出口，进出口都进行下载）采用遍历的方式
2.设置查询起止时间 默认最新一个月的单月数据，和累计的数据下载 
3.设置币种 默认是usd  
4.查询字段分组 1.商品详情 四个都设置
5.单个统计数据下载 下载单个分组的数据
6.排序方式，使用默认的编码排序

7.下载文件路径设置和命名规则
d:/hg/2023/7/
数据默认存储位置 D://hg 
其它路径从参数中读取 
一级 年份 
二级月份 
三级月份类型单月，累计 
四级 币种 
五级 字段分组
六级 文件名

3、采集单个字段的统计数据

4.临时文件
1）将请求下载的文件放到临时目录中，
2）对临时的目录文件进行数据的过滤修改重命名保存到对应目录下
3）将临时文件删除
4）根据文件名和列表记录做对比，来下载缺失的文件

5.数据下载分类 
1）按照类型分组获取对应的每月的最新编码信息
2）根据字段编码和商品进行对应统计信息的下载
3）根据商品编码下载数据
'''

class HgDownFile(object):

    def __init__(self):
        self.downUrl="http://stats.customs.gov.cn/queryData/downloadQueryData"
        # 创建ConfigParser对象
        self.config = configparser.ConfigParser()
        # 读取配置文件
        self.config.read('config.ini')
        self.r = redis.Redis(host=self.config.get('redis', 'host'),
                             port=self.config.get('redis', 'port'),
                             password=self.config.get('redis', 'pass'), db=0)

    def getcookie(self):
        cookie=self.r.spop('hgcookie')
        # cookie=self.r.srandmember('hgcookie')
        while cookie is None:
            time.sleep(10)
            cookie=self.r.srandmember('hgcookie')
            if cookie is not None:
                break
        cookie=cookie.decode('utf-8')
        cookie=cookie.strip('"')
        return cookie
    #请求下载文件
    def reqDownFile(self,data):
        header={
            'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Encoding':'gzip, deflate',
            'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control':'max-age=0',
            'Content-Type':'application/x-www-form-urlencoded',
            'Host':'stats.customs.gov.cn',
            'Origin':'http://stats.customs.gov.cn',
            'Proxy-Connection':'keep-alive',
            'Upgrade-Insecure-Requests':'1',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.64',
            'Cookie': self.getcookie()
        }
        data=data
        proxy={}
        # response=requests.post(url=self.downUrl,data=data,headers=header,verify=False,timeout=20)
        statuscode=410
        filename='数据文件.csv'
        while statuscode != 200:
            # time.sleep(5)
            try:
                header={
                    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                    'Accept-Encoding':'gzip, deflate',
                    'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
                    'Cache-Control':'max-age=0',
                    'Content-Type':'application/x-www-form-urlencoded',
                    'Host':'stats.customs.gov.cn',
                    'Origin':'http://stats.customs.gov.cn',
                    'Proxy-Connection':'keep-alive',
                    'Upgrade-Insecure-Requests':'1',
                    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.64',
                    'Cookie': self.getcookie()
                }
                response=requests.post(url=self.downUrl,data=data,headers=header,verify=False,timeout=20)
                # response.encoding = response.apparent_encoding
                response.encoding = 'GB2312'
                statuscode=response.status_code
                if statuscode == 200:
                    try:
                        csv_content = response.text
                        count = csv_content.count("\n")
                        csv_content=csv_content.replace('\0', '')
                        print(count)
                        # filename='数据文件.csv'
                        tmppath='D:\\hg\\tmp'
                        # save_dir = os.path.dirname(tmppath)
                        os.makedirs(tmppath, exist_ok=True)
                        filename = os.path.join(tmppath, filename)
                        with open(filename, 'w') as file:
                            file.write(csv_content)
                        print('CSV文件下载保存成功！')
                        break
                    except Exception as e:
                        print(e)
                        statuscode=411

                else:
                    print('CSV文件下载保存失败！')

            except Exception as e:
                print(data)
                print(e)
                statuscode=412
                continue
            print(f'statuscode:{statuscode}')
        return filename
    #统计数据的文件路径设置单个字段
    def filepath(self,iEType,currencyType,year,startMonth,endMonth,outerField1):
        path='D:\\hg\\'
        field_name=self.getfieldName(outerField1)
        iEType_name=self.getiETypeName(iEType)
        if startMonth<endMonth:
            start_str = '01'
            end_str = "{:02d}".format(endMonth)
            try:
                filename=str(year)+start_str+'-'+end_str+'--'+field_name+'--'+iEType_name+'.csv'
                filepath=path+str(year)+'\\'+end_str+'\\累计\\'
            except Exception as e:
                print(e)
        else:
            end_str = "{:02d}".format(endMonth)
            filename=str(year)+end_str+'--'+field_name+'--'+iEType_name+'.csv'
            filepath=path+str(year)+'\\'+end_str+'\\单月\\'

        save_dir = os.path.dirname(filepath)
        os.makedirs(save_dir, exist_ok=True)
        filename = os.path.join(save_dir, filename)

        return filename

    #统计数据的文件路径设置单个字段
    def codeFilepath(self,iEType,currencyType,year,startMonth,endMonth,outerField1,code):
        path='D:\\hg\\'
        field_name=self.getfieldName(outerField1)
        iEType_name=self.getiETypeName(iEType)
        if startMonth<endMonth:
            start_str = '01'
            end_str = "{:02d}".format(endMonth)
            filename=str(year)+start_str+'-'+end_str+'--'+field_name+'--'+iEType_name+'-'+str(code)+'.csv'
            filepath=path+str(year)+'\\'+end_str+'\\累计\\'+field_name+'\\'
        else:
            end_str = "{:02d}".format(endMonth)
            filename=str(year)+end_str+'--'+field_name+'--'+iEType_name+'-'+str(code)+'.csv'
            filepath=path+str(year)+'\\'+end_str+'\\单月\\'+field_name+'\\'

        save_dir = os.path.dirname(filepath)
        os.makedirs(save_dir, exist_ok=True)
        filename = os.path.join(save_dir, filename)

        return filename

    def getfieldName(self,outerField1):
        field_name=''
        if 'CODE_TS' in outerField1:
            #商品信息
            field_name='商品'
        elif 'ORIGIN_COUNTRY' in outerField1:
            #国家
            field_name='贸易伙伴'
        elif 'TRADE_MODE' in outerField1:
            #
            field_name='贸易方式'
        elif 'TRADE_CO_PORT' in outerField1:
            #国内省份
            field_name='收发货地址'
        return field_name

    def getiETypeName(self,iEType):
        iETypeName=''
        if 0==iEType:
            iETypeName='出口'
        elif 1==iEType:
            iETypeName='进口'
        elif 10==iEType:
            iETypeName='进出口'

        return iETypeName

    #单个字段的参数设置
    def setparam(self,iEType,currencyType,year,startMonth,endMonth,outerField1):
        if year>2021:
            selectTableState= 1  #202201前的数据为2 后的数据是1
        else:
            selectTableState= 2  #202201前的数据为2 后的数据是1
        param={
            'pageSize': 10,
            'iEType': iEType,
            'currencyType': currencyType,
            'year': year,
            'startMonth': startMonth,
            'endMonth': endMonth,
            'monthFlag':'',
            'unitFlag': False,
            'unitFlag1': False,
            'codeLength': '8',
            'outerField1': outerField1,
            'outerField2':'',
            'outerField3':'',
            'outerField4':'',
            'outerValue1':'',
            'outerValue2':'',
            'outerValue3':'',
            'outerValue4':'',
            'orderType': 'CODE ASC DEFAULT',
            'selectTableState': selectTableState,  #202201前的数据为2 后的数据是1
            'currentStartTime': '202201',
        }
        return param

    #联合查询字段的参数设置
    def setcodesAndProductparam(self,iEType,currencyType,year,startMonth,endMonth,outerField1,filedCode):
        if year>2021:
            selectTableState= 1  #202201前的数据为2 后的数据是1
        else:
            selectTableState= 2  #202201前的数据为2 后的数据是1
        param={
            'pageSize': 10,
            'iEType': iEType,
            'currencyType': currencyType,
            'year': year,
            'startMonth': startMonth,
            'endMonth': endMonth,
            'monthFlag':'',
            'unitFlag': False,
            'unitFlag1': False,
            'codeLength': '8',
            'outerField1': outerField1,
            'outerField2':'CODE_TS',
            'outerField3':'',
            'outerField4':'',
            'outerValue1': filedCode,
            'outerValue2':'',
            'outerValue3':'',
            'outerValue4':'',
            'orderType': 'CODE ASC DEFAULT',
            'selectTableState': selectTableState,
            'currentStartTime': '202201',
        }
        return param

    #将临时文件放复制到目录中
    def tmpToFile(self,tmpfilename,filePathName):
        # 打开csv文件
        with open(tmpfilename, 'r') as file:
            # 创建csv阅读器
            csv_reader = csv.reader(file)
            # 使用len()函数获取行数
            line_count = len(list(csv_reader))
        if line_count > 9995:
            print('csv文件行数过大需要对编码进行拆分')
            os.remove(tmpfilename)
            return ''
        else:
            shutil.copy(tmpfilename, filePathName)
            # 打开输入文件和输出文件
            # with open(tmpfilename, 'r') as input_csv, open(filePathName, 'w', newline='') as output_csv:
            #     # 创建CSV读取器和写入器
            #     csv_reader = csv.reader(input_csv)
            #     csv_writer = csv.writer(output_csv)
            #     # 逐行读取输入文件，并将每一行写入输出文件
            #     for row in csv_reader:
            #         csv_writer.writerow(row)
            os.remove(tmpfilename)
        return   filePathName

    def readcsv(self,filePath):
        codes=[]
        with open(filePath, newline='') as csvfile:
            reader = csv.reader(csvfile)
            #跳过第一条数据
            next(reader)
            for row in reader:
                # print(row[0])
                codes.append(row[0])
        return codes
    #下载获取字段的编码信息
    def field1Down(self,year,endMonth):
        fieldFileList=[]
        current_date = datetime.now()
        # year = current_date.year
        # year = int(self.config.get('param', 'year'))
        year = int(year)
        month = current_date.month
        iETypes=[0,1,10]
        outerFields=['CODE_TS','ORIGIN_COUNTRY','TRADE_MODE','TRADE_CO_PORT']
        # outerFields=['CODE_TS']
        currencyType='usd'
        # endMonth=self.r.get('newMonth')
        # endMonth=int(endMonth.decode('utf-8'))
        # endMonth=int(self.config.get('param', 'endMonth'))
        # if endMonth != (month-1):
        #     return
        if endMonth==1:
            startMonths=[1]
        else:
            startMonths=[1,endMonth]
        for startMonth in startMonths:
            for iEType in iETypes:
                for outerField1 in outerFields:
                    param=self.setparam(iEType,currencyType,year,startMonth,endMonth,outerField1)
                    filePathName=self.filepath(iEType,currencyType,year,startMonth,endMonth,outerField1)
                    fieldFileList.append(filePathName)
                    if os.path.exists(filePathName):
                        continue
                    tmpfilename=self.reqDownFile(param)
                    saveFileName=self.tmpToFile(tmpfilename,filePathName)
                    print(saveFileName)

        return fieldFileList
    #下载贸易方式商品，贸易伙伴商品，注册地商品 的统计信息
    #1.从单个统计文件中获取对应的贸易编码，
    #2.对每个贸易编码进行文件下载
    #3.对下载的文件进行合并清洗重命名
    def fieldCodeDown(self,iEType,currencyType,year,startMonth,endMonth,outerField1,codes):
        codeFileList=[]
        for code in codes:
            param=self.setcodesAndProductparam(iEType,currencyType,year,startMonth,endMonth,outerField1,code)
            filePathName=self.codeFilepath(iEType,currencyType,year,startMonth,endMonth,outerField1,code)
            if os.path.exists(filePathName):
                print(f'文件已存在{filePathName}')
                codeFileList.append(filePathName)
                continue
            tmpfilename=self.reqDownFile(param)
            saveFileName=self.tmpToFile(tmpfilename,filePathName)
            print(saveFileName)
            codeFileList.append(saveFileName)
        return codeFileList

    def codeFieldDown(self,fieldFileList,year,endMonth):
        current_date = datetime.now()
        # year = current_date.year
        year = int(year)
        # endMonth=self.r.get('newMonth')
        # endMonth=int(endMonth.decode('utf-8'))
        # endMonth=int(self.config.get('param', 'endMonth'))
        for fieldFile in fieldFileList:
            #['CODE_TS','ORIGIN_COUNTRY','TRADE_MODE','TRADE_CO_PORT']
            try:
                if '商品' in fieldFile:
                    continue
                if '贸易伙伴' in fieldFile:
                    outerField1=['ORIGIN_COUNTRY']
                if '贸易方式' in fieldFile:
                    outerField1=['TRADE_MODE']
                if '收发货地址' in fieldFile:
                    outerField1=['TRADE_CO_PORT']
                if '单月' in fieldFile:
                    startMonth=endMonth
                if '累计' in fieldFile:
                    startMonth=1
                if '--进口' in fieldFile:
                    iEType=1
                if '--出口' in fieldFile:
                    iEType=0
                if '--进出口' in fieldFile:
                    iEType=10
                currencyType='usd'
                codes=hgDownFile.readcsv(fieldFile)
                codeFileList=hgDownFile.fieldCodeDown(iEType,currencyType,year,startMonth,endMonth,outerField1,codes)
                while len(codes)!= len(codeFileList):
                    codeFileList=hgDownFile.fieldCodeDown(iEType,currencyType,year,startMonth,endMonth,outerField1,codes)

            except Exception as e:
                print(e)

    def combinFile(self,refile,outFile):
        # 查找当前目录及其子目录下所有以.txt结尾的文件
        files = glob.glob(r"D:\hg\2023\07\单月\贸易伙伴\202307--贸易伙伴--出口-*.csv", recursive=True)
        files = glob.glob(refile, recursive=True)
        # print(files)
        filess=[]
        for ii in files:
            file = pd.read_csv(ii, encoding='gbk')
            filess.append(file)
        combined = pd.concat(filess)
        # 保存合并后的文件
        # combined.to_csv(r'D:\hg\2023\07\单月\202307--贸易伙伴商品--出口.csv', index=False, encoding='gbk')
        combined.to_csv(outFile, index=False, encoding='gbk')

    def combinFiles(self,refile,outFile):
        # 查找当前目录及其子目录下所有以.txt结尾的文件
        files = glob.glob(r"D:\hg\2023\07\单月\贸易伙伴\202307--贸易伙伴--出口-*.csv", recursive=True)
        files = glob.glob(refile, recursive=True)
        # 创建一个空的DataFrame用于存储合并后的数据
        merged_data = pd.DataFrame()
        # 逐个读取CSV文件并合并到DataFrame中
        for file in files:
            data = pd.read_csv(file, encoding='gbk',dtype=str)
            # 去掉最后一列
            data = data.iloc[:, :-1]
            dad=pd.DataFrame(data,dtype=str)
            merged_data = merged_data.append(dad, ignore_index=True)
        # 将合并后的数据保存到新的CSV文件中
        merged_data.to_csv(outFile, encoding='gbk', index=False, quoting=1, quotechar='"', escapechar='\\')
        print('合并完成！！')

    def fileNameleiji(self):
        # refile=r'D:\hg\2023\07\累计\贸易方式\202301-07--贸易方式--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--贸易方式商品--进口.csv'
        # refile=r'D:\hg\2023\07\累计\贸易方式\202301-07--贸易方式--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--贸易方式商品--出口.csv'
        # refile=r'D:\hg\2023\07\累计\贸易方式\202301-07--贸易方式--进出口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--贸易方式商品--进出口.csv'
        # refile=r'D:\hg\2023\07\累计\收发货地址\202301-07--收发货地址--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--收发货地址商品--进口.csv'
        # refile=r'D:\hg\2023\07\累计\收发货地址\202301-07--收发货地址--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--收发货地址商品--出口.csv'
        # refile=r'D:\hg\2023\07\累计\收发货地址\202301-07--收发货地址--进出口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--收发货地址商品--进出口.csv'
        # refile=r'D:\hg\2023\07\累计\贸易伙伴\202301-07--贸易伙伴--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--贸易伙伴商品--进口.csv'
        # refile=r'D:\hg\2023\07\累计\贸易伙伴\202301-07--贸易伙伴--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202301-07--贸易伙伴商品--出口.csv'
        refile=r'D:\hg\2023\07\累计\贸易伙伴\202301-07--贸易伙伴--进出口-*.csv'
        outFile=r'D:\hg\2023\07\202301-07--贸易伙伴商品--进出口.csv'
        self.combinFile(refile,outFile)

    def fileNamedanyue(self):
        # refile=r'D:\hg\2023\07\单月\贸易方式\202307--贸易方式--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--贸易方式商品--进口.csv'
        # refile=r'D:\hg\2023\07\单月\贸易方式\202307--贸易方式--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--贸易方式商品--出口.csv'
        # refile=r'D:\hg\2023\07\单月\贸易方式\202307--贸易方式--进出口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--贸易方式商品--进出口.csv'
        # refile=r'D:\hg\2023\07\单月\收发货地址\202307--收发货地址--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--收发货地址商品--出口.csv'
        # refile=r'D:\hg\2023\07\单月\收发货地址\202307--收发货地址--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--收发货地址商品--进口.csv'
        # refile=r'D:\hg\2023\07\单月\收发货地址\202307--收发货地址- -进出口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--收发货地址商品--进出口.csv'
        # refile=r'D:\hg\2023\07\单月\贸易伙伴\202307--贸易伙伴--进口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--贸易伙伴商品--进口.csv'
        # refile=r'D:\hg\2023\07\单月\贸易伙伴\202307--贸易伙伴--出口-*.csv'
        # outFile=r'D:\hg\2023\07\202307--贸易伙伴商品--出口.csv'
        refile=r'D:\hg\2023\07\单月\贸易伙伴\202307--贸易伙伴--进出口-*.csv'
        outFile=r'D:\hg\2023\07\202307--贸易伙伴商品--进出口.csv'
        self.combinFile(refile,outFile)


if __name__ == '__main__':
    hgDownFile=HgDownFile()
    # hgDownFile.fileNameleiji()
    # hgDownFile.fileNamedanyue()
    # hgDownFile.tmpToFile(tmpfilename,filePathName)

    ss=hgDownFile.config.get('param', 'endMonth')
    yss=hgDownFile.config.get('param', 'year')
    for ye in yss.split(','):
        year=int(ye)
        for s in ss.split(','):
            endMonth=int(s)
            fieldFileList=hgDownFile.field1Down(year,endMonth)
            while len(fieldFileList)< 12:
                fieldFileList=hgDownFile.field1Down(year,endMonth)
                if len(fieldFileList)>= 12:
                    break
            for i in range(1,3):
                hgDownFile.codeFieldDown(fieldFileList,year,endMonth)
