Skip to content

S
SCBG-PYTHON
提交 34cbd322 作者: bruxellse_li
操作

python-service

上级 6411f842
正在显示 包含 9171 行增加 和 0 行删除
TextRewriting.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/3/27 11:50
# @Author : bruxellse_li
# @File : TextRewriting.py
# @Project : 从word中提取指定表格
from datetime import datetime
from wsgiref.handlers import format_date_time
from time import mktime
import hashlib
import base64
import hmac
from urllib.parse import urlencode
import json
import requests
import ast
'''
1、文本改写 Web API 调用示例
2、运行前：请先填写Appid、APIKey、APISecret 相关信息
'''
class AssembleHeaderException(Exception):
def __init__(self, msg):
self.message = msg
class Url:
def __init__(this, host, path, schema):
this.host = host
this.path = path
this.schema = schema
pass
class work_wsParam(object):
def __init__(self, APPID, APIKey, APISecret, level, url):
self.APPID = APPID
self.APIKey = APIKey
self.APISecret = APISecret
# self.url = 'https://api.xf-yun.com/v1/private/se3acbe7f'
self.url = url
self.level = level
def parse_url(self, requset_url):
stidx = requset_url.index("://")
host = requset_url[stidx + 3:]
schema = requset_url[:stidx + 3]
edidx = host.index("/")
if edidx <= 0:
raise AssembleHeaderException("invalid request url:" + requset_url)
path = host[edidx:]
host = host[:edidx]
u = Url(host, path, schema)
return u
def init_header(self):
headers = {
'content-type': "application/json",
'host': 'api.xf-yun.com'
}
return headers
def get_body(self, text):
data = {
"header": {
"app_id": self.APPID,
"status": 3,
},
"parameter": {
"se3acbe7f": {
"level": self.level,
"result": {
"encoding": "utf8",
"compress": "raw",
"format": "json"
}
}
},
"payload": {
"input1": {
"encoding": "utf8",
"compress": "raw",
"format": "plain",
"status": 3,
"text": str(base64.b64encode(text.encode('utf-8')), 'utf-8')
}
}
}
body = json.dumps(data)
return body
def assemble_ws_auth_url(wsParam, requset_url, method="POST", api_key="", api_secret=""):
u = wsParam.parse_url(requset_url)
# u = parse_url(requset_url)
host = u.host
path = u.path
now = datetime.now()
date = format_date_time(mktime(now.timetuple()))
# print(date)
# date = "Thu, 12 Dec 2019 01:57:27 GMT"
signature_origin = "host: {}\ndate: {}\n{} {} HTTP/1.1".format(host, date, method, path)
# print("----2", signature_origin)
signature_sha = hmac.new(api_secret.encode('utf-8'), signature_origin.encode('utf-8'),
digestmod=hashlib.sha256).digest()
signature_sha = base64.b64encode(signature_sha).decode(encoding='utf-8')
authorization_origin = "api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"" % (
api_key, "hmac-sha256", "host date request-line", signature_sha)
# print("----1:", authorization_origin)
authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
# print(authorization_origin)
values = {
"host": host,
"date": date,
"authorization": authorization
}
return requset_url + "?" + urlencode(values)
def get_result(wsParam, text):
request_url = assemble_ws_auth_url(wsParam, wsParam.url, "POST", wsParam.APIKey, wsParam.APISecret)
# request_url = assemble_ws_auth_url(url, "POST", APIKey, APISecret)
# print("request_url:", request_url)
response = requests.post(request_url, data=wsParam.get_body(text), headers=wsParam.init_header())
# print("response:", response)
str_result = response.content.decode('utf8')
json_result = json.loads(str_result)
# print("response-content:", json_result)
if json_result. __contains__('header') and json_result['header']['code'] == 0:
renew_text = json_result['payload']['result']['text']
# print("\n改写结果：", str(base64.b64decode(renew_text), 'utf-8'))
result_text = str(base64.b64decode(renew_text), 'utf-8')
trans_result = ast.literal_eval(result_text)
str_result = trans_result[0][0]
else:
str_result = "改写失败！请检查服务是否断开"
# 改写结果保存到文件
# result_file = open(".\改写结果.txt",'w',encoding='utf-8')
# result_file.write(str(base64.b64decode(renew_text), 'utf-8'))
return str_result
def get_list_result(text):
APPID = "51718e1a"
APISecret = "ZTAwYjcyZTRlZTQ3M2FmY2RlMDZiYjEx"
APIKey = "ec11462dcbb1d8d5d8ec15612f7243e7"
url = 'https://api.xf-yun.com/v1/private/se3acbe7f'
# level = "<L3>" # 改写等级 <L1> ~ <L6> 等级越高，改写程度越深
level_list = ["<L2>", "<L3>", "<L4>"]
result_one = []
for level in level_list:
wsParam = work_wsParam(APPID, APIKey, APISecret, level, url)
text_one = get_result(wsParam, text)
result_one.append(text_one)
return result_one
if __name__ == "__main__":
text = "2021年，本单位资产实行分类管理，建立健全了资产内部管理制度；单位加强对实物资产和无形资产的管理，" \
"明确相关部门和岗位的职责权限，强化对配置、使用和处置等关键环节的管控；明确资产使用和保管责任人，落实资产使用人在资产管理中的责任。"
print(get_list_result(text))
# # APPID = "51718e1a"
# # APISecret = "ZTAwYjcyZTRlZTQ3M2FmY2RlMDZiYjEx"
# # APIKey = "ec11462dcbb1d8d5d8ec15612f7243e7"
# # url = 'https://api.xf-yun.com/v1/private/se3acbe7f'
# # level = "<L3>" #改写等级 <L1> ~ <L6> 等级越高，改写程度越深
# text = "随着我国城市化脚步的不断加快，园林工程建设的数量也在不断上升，城市对于园林工程的质量要求也随之上升，然而就当前我国园" \
# "林工程管理的实践而言，就园林工程质量管理这一环节还存在许多不足之处，本文在探讨园林工程质量内涵的基础上，深入进行质量" \
# "管理策略探讨，目的是保障我国园林工程施工质量和提升整体发展效率。"
#
# get_result(text)
# wsParam = work_wsParam(APPID, APIKey, APISecret, level, url)
# request_url = assemble_ws_auth_url(wsParam.url, "POST", wsParam.APIKey, wsParam.APISecret)
# print(get_result(text, request_url))
app_run.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/3/20 11:43
# @Author : bruxellse_li
# @File : app_run.py
# @Project : 从word中提取指定表格
from docx import Document
from flask import Flask, send_file, jsonify
from flask import request, Response
import requests
from extract_table import get_choose_table, get_other_table, get_other1_table
from extract_factor import get_text_from_docx, Extract, get_cover_content_from_docx, Other_Extract
import json, re
from utils.log import logger
import subprocess
from pathlib import Path
import traceback
from fdfs_client.client import *
from generate.gen_user_report_auto_generated import main_process
from generate.platform_generated import pl_process
from TextRewriting import get_list_result
from sentence_split import qx_correct, ner_correct
from utils.database_mysql import DatabaseMySQL
# 定义数据库链接基础信息
database_config = {
'host': '114.115.205.50',
# 'host': '114.116.44.11',
'port': 3306,
'user': 'root',
# 'password': 'f7s0&7qqtK',
'password': 'yotop@123456',
'database': 'clb_project'
}
dbm = DatabaseMySQL(database_config=database_config)
temp_url = "http://114.115.215.96/"
app = Flask(__name__)
UPLOAD_FOLDER = r'data/' # 上传路径
Path(UPLOAD_FOLDER).mkdir(parents=True, exist_ok=True)
abs_path = os.path.split(os.path.realpath(__file__))[0]
ALLOWED_EXTENSIONS = set(['doc', 'docx']) # 允许上传的文件类型
# 文件上传服务器定义
tracker_conf = get_tracker_conf("data/fdfs_client.conf")
client = Fdfs_client(tracker_conf)
# 跨域支持1
from flask_cors import CORS
CORS(app, supports_credentials=True)
def allowed_file(filename):
# 验证上传的文件名是否符合要求，文件名必须带点并且符合允许上传的文件类型要求，两者都满足则返回 true
if '.' in filename and filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS:
return filename
def doc2docx(doc_path, docx_path):
# 使用LibreOffice将doc文件转换为docx文件
subprocess.call(['libreoffice', '--headless', '--convert-to', 'docx', doc_path, '--outdir', os.path.dirname(docx_path)])
# 将转换后的docx文件重命名为目标文件名
os.rename(os.path.splitext(doc_path)[0] + '.docx', docx_path)
@app.route('/extract_special_table', methods=['POST'])
def extract_special_table():
# table_params ——['资产负债表', '收入费用表（1）', '收入费用表（2）']
data = request.get_json()
file_request = data["path"]
filename = file_request.split("/")[-1]
if ".doc" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
doc_path = os.path.join(UPLOAD_FOLDER, filename)
# 获取文件路径前缀
new_path = os.path.splitext(doc_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(doc_path, new_path)
document = Document(new_path)
elif ".docx" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
document = Document(os.path.join(UPLOAD_FOLDER, filename))
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
table_names = ["以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明"]
data_result = get_other_table(document, table_names)
if data_result["以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明"]:
data_result = data_result
else:
temp_table_result = get_other_table(document, ["本单位无以名义金额计量的资产"])
del data_result["以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明"]
data_result.update(temp_table_result)
os.remove(os.path.join(UPLOAD_FOLDER, filename))
return Response(json.dumps(data_result, ensure_ascii=False), content_type='application/json')
@app.route('/extract_other_table1', methods=['POST'])
def extract_special_table1():
# table_params ——['资产负债表', '收入费用表（1）', '收入费用表（2）']
data = request.get_json()
file_request = data["path"]
filename = file_request.split("/")[-1]
if ".doc" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
doc_path = os.path.join(UPLOAD_FOLDER, filename)
# 获取文件路径前缀
new_path = os.path.splitext(doc_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(doc_path, new_path)
document = Document(new_path)
elif ".docx" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
document = Document(os.path.join(UPLOAD_FOLDER, filename))
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
table_names = ['货币资金明细信息如下']
data_result = get_other1_table(document, table_names)
os.remove(os.path.join(UPLOAD_FOLDER, filename))
return Response(json.dumps(data_result, ensure_ascii=False), content_type='application/json')
@app.route('/extract_table', methods=['POST'])
def extract_table():
# table_params ——['资产负债表', '收入费用表（1）', '收入费用表（2）']
data = request.get_json()
file_request = data["path"]
filename = file_request.split("/")[-1]
if ".doc" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
doc_path = os.path.join(UPLOAD_FOLDER, filename)
print(doc_path)
# 获取文件路径前缀
new_path = os.path.splitext(doc_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(doc_path, new_path)
document = Document(new_path)
elif ".docx" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
document = Document(os.path.join(UPLOAD_FOLDER, filename))
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
table_params = ['资产负债表', '收入费用表（1）', '收入费用表（2）']
data = get_choose_table(document, table_params)
# 解析上级关系
# 处理资产负债表
temp_list = data["资产负债表"]
temp_dict = {}
for temp in temp_list:
temp_text = re.sub(":", "：", temp["项目"])
pro_text = temp_text[-1]
if pro_text == "：":
temp_dict.update({"temp_key": temp_text})
continue
elif pro_text.isdigit():
temp["项目"] = temp_text[:-1]
temp["上级项目"] = temp_dict["temp_key"].strip("：")
else:
temp["上级项目"] = temp_dict["temp_key"].strip("：")
# 处理收入费用表（1）
temp_list_0 = data["收入费用表（1）"]
temp_dict_0 = {"temp_key": "收入合计"}
for temp_0 in temp_list_0:
# 先判断字段名是否为数字
temp_text = temp_0["项目"]
pro_text = temp_text[-1]
if pro_text.isdigit():
temp_0["项目"] = temp_text[:-1]
else:
temp_0["项目"] = temp_text
if temp_0["项目"].strip() == "收入合计":
temp_dict_0.update({"temp_key": "费用合计"})
else:
if temp_0["项目"].strip() == "费用合计" or temp_0["项目"].strip() == "本年盈余":
continue
else:
temp_0["上级项目"] = temp_dict_0["temp_key"]
# 处理收入费用表（2）
temp_list_1 = data["收入费用表（2）"]
temp_dict_1 = {"temp_key": "收入合计"}
for temp_1 in temp_list_1:
# 先判断字段名是否为数字
temp_text = temp_1["项目"]
pro_text = temp_text[-1]
if pro_text.isdigit():
temp_1["项目"] = temp_text[:-1]
else:
temp_1["项目"] = temp_text
if temp_1["项目"].strip() == "收入合计":
temp_dict_1.update({"temp_key": "费用合计"})
else:
if temp_1["项目"].strip() == "费用合计" or temp_1["项目"].strip() == "本年盈余":
continue
else:
temp_1["上级项目"] = temp_dict_1["temp_key"]
os.remove(os.path.join(UPLOAD_FOLDER, filename))
return Response(json.dumps(data, ensure_ascii=False), content_type='application/json')
@app.route('/extract_factor', methods=['POST'])
def extract_factor():
data = request.get_json()
file_request = data["path"]
filename = file_request.split("/")[-1]
if ".doc" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
doc_path = os.path.join(UPLOAD_FOLDER, filename)
new_path = os.path.splitext(doc_path)[0] + '.docx'
doc2docx(doc_path, new_path)
document = get_text_from_docx(new_path)
elif ".docx" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
document = get_text_from_docx(os.path.join(UPLOAD_FOLDER, filename))
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
data = Extract().extract_result(document)
os.remove(os.path.join(UPLOAD_FOLDER, filename))
return Response(json.dumps(data, ensure_ascii=False), content_type='application/json')
@app.route('/generate_report', methods=['POST'])
def generate_report():
"""
template_path : 模板文件下载地址
document_path: 半成品文件下载地址
report_name: 报告名称
data_object: 待填充数据
:return:
"""
data = request.get_json()
template_request = data["template_path"]
doc_request = data["document_path"]
report_name = data["report_name"] + ".docx"
data_object = data["object"]["data_object"]
tables_dict = data["object"]["tables_dict"]
current_filename = time.strftime('%Y_%m_%d-%H_%M_%S') + ".docx"
save_path = UPLOAD_FOLDER + "/" + current_filename
# 先判断是否是docx 格式
template_filename = template_request.split("/")[-1]
# 文件先下载再判断是否转换
if ".doc" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
temp_template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# 获取文件路径前缀
template_path = os.path.splitext(temp_template_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(temp_template_path, template_path)
elif ".docx" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
template_path = os.path.join(UPLOAD_FOLDER, template_filename)
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
doc_filename = doc_request.split("/")[-1]
if ".doc" in doc_request:
r1 = requests.get(doc_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, doc_filename), "wb") as f1:
for chunk in r1.iter_content(chunk_size=512):
f1.write(chunk)
temp_doc_path = os.path.join(UPLOAD_FOLDER, doc_filename)
# 获取文件路径前缀
doc_path = os.path.splitext(temp_doc_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(temp_doc_path, doc_path)
half_work_path = doc_path
elif ".docx" in doc_request:
r1 = requests.get(doc_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, doc_filename), "wb") as f1:
for chunk in r1.iter_content(chunk_size=512):
f1.write(chunk)
half_work_path = os.path.join(UPLOAD_FOLDER, doc_filename)
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
main_process(half_work_path, tables_dict, template_path, report_name, data_object, save_path)
os.remove(os.path.join(UPLOAD_FOLDER, template_filename))
os.remove(os.path.join(UPLOAD_FOLDER, doc_filename))
send_path = os.path.join(UPLOAD_FOLDER, report_name)
# "data/财务报告.docx"
ret_upload = client.upload_by_filename(send_path)
return ret_upload["Remote file_id"]
@app.route('/gx_app', methods=['POST'])
def gx_app():
try:
data = request.get_json()
text = data["content"] if "content" in data else ""
list_result = get_list_result(text)
dict_result = {
'code': 300,
'msg': 'success',
'isHandleSuccess': True,
'result_data': list_result
}
except Exception as e:
traceback.print_exc()
dict_result = {
'code': 500,
'msg': 'fail' + str(e),
'isHandleSuccess': False,
'result_data': None
}
return jsonify(dict_result)
@app.route('/ner_error', methods=(['POST']))
def ner_error():
data = request.get_json()
text = data['content']
param = data["param"].strip()
try:
content, right_result = ner_correct(text, param)
except Exception as e:
return json.dumps({'code': '500',
'message': 'failure' + str(e),
'resultData': None})
return json.dumps({'code': '200',
'message': 'success',
'resultData': {'contentDetails': right_result,
'correctContent': content}})
@app.route('/qx_error', methods=(['POST']))
def qx_error():
data = request.get_json()
text = data['content']
try:
content, right_result = qx_correct(text)
except Exception as e:
return json.dumps({'code': '500',
'message': 'failure' + str(e),
'resultData': None})
return json.dumps({'code': '200',
'message': 'success',
'resultData': {'contentDetails': right_result,
'correctContent': content}})
@app.route('/platform_report', methods=['POST'])
def platform_report():
"""
task_id: 任务id
:return:
"""
data = request.get_json()
task_id = data["task_id"].strip()
# todo: 基于任务id来获取数据信息
# dataset_sql = '''SELECT ds.id,ds.param_value,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
# inner join clb_report_data_set ds on te.data_set_id = ds.id
# where t.id = {};'''.format(task_id)
dataset_sql = """SELECT ds.id,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
inner join clb_report_data_set ds on te.data_set_id = ds.id
where t.id = {};""".format(task_id)
dbm = DatabaseMySQL(database_config=database_config)
dataset_result = dbm.query(sql=dataset_sql)[0]
dataset_id, temp_path = dataset_result["id"], dataset_result["file_path"]
# dataset_id, param_value, temp_path = dataset_result["id"], dataset_result["param_value"], dataset_result[
# "file_path"]
# print(type(param_value))
# param_value = json.loads(param_value)
# todo: 再基于数据集id 获取数据集地址，参数，返回数据类型，数据对象
# data_source_sql = """select ds.url,ds.params,ds.type,ds.data_name from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
# where m.data_set_id = {};""".format(dataset_id)
data_source_sql = """select ds.url,m.param_value,ds.type,m.return_object from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
where m.data_set_id = {};""".format(dataset_id)
datasource_result_list = dbm.query(sql=data_source_sql)
# 关闭数据库连接
# dbm.close()
# todo: 第一步：基于模板路径和模板url 获取下载模板链接
template_request = temp_url + "/" + temp_path
# 先判断是否是docx 格式
template_filename = template_request.split("/")[-1]
# print(template_filename)
# print(template_request)
# 文件先下载再判断是否转换
if ".doc" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
temp_template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# 获取文件路径前缀
template_path = os.path.splitext(temp_template_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(temp_template_path, template_path)
elif ".docx" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
template_path = os.path.join(UPLOAD_FOLDER, template_filename)
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
# todo: 第二步：基于数据源信息获取数据对象
dict_data = dict()
for datasource_result in datasource_result_list:
dataset_url = datasource_result["url"]
params = datasource_result["param_value"]
# 4月23号调整
dict_param = json.loads(params)
# {"id": 2, "name": "nihao"}
connect_list = []
for key, value in dict_param.items():
connect_param = str(key) + "=" + str(value)
connect_list.append(connect_param)
# params_list = params.split(",")
# connect_list = []
# for temp_param in params_list:
# connect_param = temp_param + "=" + str(param_value[temp_param])
# connect_list.append(connect_param)
request_str = "&".join(connect_list)
dataset_request = dataset_url + "?" + request_str
str_dataset_info = requests.get(dataset_request, stream=True)
logger.info(str_dataset_info.content)
dataset_info = json.loads(str_dataset_info.content)
data_name = datasource_result["return_object"]
dict_data[data_name] = dataset_info["result"]
logger.info(dict_data)
# 定义平台临时文件名
report_name = "平台模板样例报告.docx"
pl_process(template_path, dict_data, report_name)
os.remove(os.path.join(UPLOAD_FOLDER, template_filename))
send_path = os.path.join(UPLOAD_FOLDER, report_name)
# "data/财务报告.docx"
ret_upload = client.upload_by_filename(send_path)
return ret_upload["Remote file_id"]
@app.route('/extract_cover_factor', methods=['POST'])
def extract_cover_factor():
data = request.get_json()
file_request = data["path"]
filename = file_request.split("/")[-1]
if ".doc" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
doc_path = os.path.join(UPLOAD_FOLDER, filename)
new_path = os.path.splitext(doc_path)[0] + '.docx'
doc2docx(doc_path, new_path)
cover_contents, other_contents = get_cover_content_from_docx(new_path)
elif ".docx" in file_request:
r = requests.get(file_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
cover_contents, other_contents = get_cover_content_from_docx(os.path.join(UPLOAD_FOLDER, filename))
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
cover_pattern = re.compile(r"([0-9]{0,4}).*(?=(财务报告))")
# print(content)
cover_group = cover_pattern.search(cover_contents)
if cover_group:
cover_text = cover_group.group().strip()
else:
cover_text = ""
# todo: 处理其它的字段信息
other_extract = Other_Extract()
other_data = other_extract.extract_other_result(other_contents)
other_data["reportTitle"] = cover_text
os.remove(os.path.join(UPLOAD_FOLDER, filename))
return Response(json.dumps(other_data, ensure_ascii=False), content_type='application/json')
if __name__ == '__main__':
app.config['JSON_AS_ASCII'] = False
app.config['JSONIFY_MIMETYPE'] = "application/json;charset=utf-8"
app.run(host='0.0.0.0', port=4000, debug=False)
base/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/7/31 10:21
base/app/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/4/21 9:30
base/app/base_app.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : base_app
# @Author : LiuYan
# @Time : 2021/4/21 9:30
import json
from flask import Flask, Blueprint, request, jsonify, render_template
from flask_cors import *
from base.config.base_config import DB_URI
from utils.log import logger
app = Flask(__name__, static_url_path='', static_folder='../../static', template_folder='../../static')
CORS(app, supports_credentials=True)
app.config['SQLALCHEMY_DATABASE_URI'] = DB_URI
base/config/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/4/16 18:03
base/config/base_config.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : base_config
# @Author : LiuYan
# @Time : 2021/4/16 18:06
from abc import abstractmethod, ABC
# root_dir = '/home/zzsn/liuyan/zzsn_gurag'
root_dir = '..' # deploy
DIALECT = 'mysql'
DRIVER = 'pymysql'
USERNAME = 'root'
PASSWORD = 'zzsn9988'
HOST = '39.105.62.235'
PORT = '3306'
DATABASE = 'gurag'
# 连接数据的URI
DB_URI = '{}+{}://{}:{}@{}:{}/{}?charset=utf8'.format(DIALECT, DRIVER, USERNAME, PASSWORD, HOST, PORT, DATABASE)
SQLALCHEMY_DATABASE_URI = DB_URI
SQLALCHEMY_TRACK_MODIFICATIONS = True
SWAGGER_TITLE = 'API'
SWAGGER_DESC = 'API接口'
# 地址，必须带上端口号
SWAGGER_HOST = '0.0.0.0:7010'
base/dao/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/12/1 10:23
base/dao/base_dao.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : base_dao
# @Author : LiuYan
# @Time : 2021/12/1 10:23
from abc import abstractmethod, ABC
from flask_sqlalchemy import SQLAlchemy
from base.app.base_app import app
db = SQLAlchemy(app)
class BaseDao(ABC):
@abstractmethod
def __init__(self):
super(BaseDao, self).__init__()
@abstractmethod
def load_config(self):
"""
Add the config you need.
:return: config(YamlDict)
"""
pass
copy_content.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/3/7 17:29
# @Author : ctt
# @File : copy_table
# @Project : 表格复制
from copy import deepcopy
from docx import Document
"""
prep_p = p.insert_paragraph_before("段落前插入内容)
document.add_page_break() # 插入分页符
"""
import re
import json
import pandas as pd
from docx import Document
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table, _Row
from docx.text.paragraph import Paragraph
from docx.shared import Pt
# 定义待复制内容的匹配模式
start_pattern = re.compile(r'(?<=[0-9][\.．]会计报表重要项目的明细信息及说明)$')
end_pattern = re.compile(r'(?<=[0-9][\.．]需要说明的其他事项)$|(?<=[0-9][\.．]需要说明的其他事项。)（略）$')
def iter_block_items(parent):
"""
Generate a reference to each paragraph and table child within *parent*,
in document order. Each returned value is an instance of either Table or
Paragraph. *parent* would most commonly be a reference to a main
Document object, but also works for a _Cell object, which itself can
contain paragraphs and tables.
"""
if isinstance(parent, _Document):
parent_elm = parent.element.body
elif isinstance(parent, _Cell):
parent_elm = parent._tc
elif isinstance(parent, _Row):
parent_elm = parent._tr
else:
raise ValueError("something's not right")
for child in parent_elm.iterchildren():
if isinstance(child, CT_P):
yield Paragraph(child, parent)
elif isinstance(child, CT_Tbl):
yield Table(child, parent)
# todo: 先复制内容到模板中，保存更新后的模板
def copy_content_main(doc_path: str, temp_path: str):
doc = Document(doc_path)
# 新建临时文档
new_doc = Document()
start_found = False
end_found = False
for element in doc.element.body.xpath("w:p | w:tbl"):
if isinstance(element, CT_P):
para = Paragraph(element, doc)
start_results = re.findall(start_pattern, para.text)
if start_results:
# print(para.text)
start_found = True
continue
if isinstance(element, CT_P):
para = Paragraph(element, doc)
end_results = re.findall(end_pattern, para.text)
if end_results:
# print(para.text)
end_found = True
break
# 复制文本段落
if start_found and not end_found and isinstance(element, CT_P):
para = Paragraph(element, doc)
new_doc.add_paragraph(para.text)
# 复制表格
if start_found and not end_found and isinstance(element, CT_Tbl):
table = Table(element, doc)
new_table = deepcopy(table._element)
# 在目标文档添加一个空段落
new_doc.add_paragraph('')
# 获取新段落
new_paragraph = new_doc.paragraphs[-1]
# 在新段落中添加表格
new_paragraph._element.addprevious(new_table)
# 遍历文档中的段落，去除多余的空白段落
for para in new_doc.paragraphs:
# 使用正则表达式匹配空白段落（只包含空格和换行符）
if re.match(r'^\s*$', para.text):
# 删除空白段落
new_doc._element.body.remove(para._element)
# 获取待插入内容在目标文档中的位置
source_doc = Document(temp_path)
start_index = None
for index, para in enumerate(source_doc.paragraphs):
start_result = re.findall(start_pattern, para.text)
if start_result:
start_index = index
break
target_paragraph = source_doc.paragraphs[start_index]
# 遍历源文档中的所有元素
for element in reversed(new_doc.element.body):
# 如果是段落，就在目标段落之后添加
if isinstance(element, CT_P):
# 考虑样式发生变化，对此进行调整
para = Paragraph(element, doc)
# 设置字体和字号
new_para = source_doc.add_paragraph(para.text, style='Normal')
font = new_para.runs[0].font
font.name = "宋体"
font.size = Pt(12)
new_para.paragraph_format.space_before = Pt(12)
new_para.paragraph_format.first_line_indent = Pt(25)
source_doc.element.body.insert(source_doc.element.body.index(target_paragraph._element) + 1,
new_para._element)
# 如果是表格，也在目标段落之后添加
elif isinstance(element, CT_Tbl):
source_doc.element.body.insert(source_doc.element.body.index(target_paragraph._element) + 1, element)
source_doc.save(temp_path)
return None
if __name__ == '__main__':
doc_path = "data/3月23测试半成品.docx"
# doc_path = 'data/特殊教育学校(1).docx'
temp_path = "data/new_财务报告模板.docx"
copy_content_main(doc_path, temp_path)
# docx_file = r'wKjIbGQeSb6AUq1aAAgAABcLaMw312.docx'
# doc = Document(docx_file)
# new_doc = Document()
# start_found = False
# end_found = False
# for element in doc.element.body.xpath("w:p | w:tbl"):
# if isinstance(element, CT_P):
# para = Paragraph(element, doc)
# start_results = re.findall(start_pattern, para.text)
# if start_results:
# start_found = True
# continue
#
# if isinstance(element, CT_P):
# para = Paragraph(element, doc)
# end_results = re.findall(end_pattern, para.text)
# if end_results:
# end_found = True
# break
#
# # 复制文本段落
# if start_found and not end_found and isinstance(element, CT_P):
# para = Paragraph(element, doc)
# new_paragraph = new_doc.add_paragraph(para.text)
#
# # 复制表格
# if start_found and not end_found and isinstance(element, CT_Tbl):
# table = Table(element, doc)
# new_table = deepcopy(table._element)
# # 在目标文档添加一个空段落
# new_doc.add_paragraph('')
# # 获取新段落
# new_paragraph = new_doc.paragraphs[-1]
# # 在新段落中添加表格
# new_paragraph._element.addprevious(new_table)
#
# # 遍历文档中的段落，去除多余的空白段落
# for para in new_doc.paragraphs:
# # 使用正则表达式匹配空白段落（只包含空格和换行符）
# if re.match(r'^\s*$', para.text):
# # 删除空白段落
# new_doc._element.body.remove(para._element)
#
# # # 保存目标文档
# # new_doc.save('new.docx')
#
# # 获取文档的所有元素
# elements = new_doc._element
#
# # 获取待插入内容在目标文档中的位置
#
# source_doc = Document("data/01系统导出模板.docx")
# start_index = None
# for index, para in enumerate(source_doc.paragraphs):
# start_result = re.findall(start_pattern, para.text)
# if start_result:
# start_index = index
# break
#
# target_paragraph = source_doc.paragraphs[start_index]
# # 遍历源文档中的所有元素
# for element in reversed(new_doc.element.body):
# # 如果是段落，就在目标段落之后添加
# if isinstance(element, CT_P):
# # 考虑样式发生变化，对此进行调整
# para = Paragraph(element, doc)
# # 设置字体和字号
# # font = source_doc.styles['Normal'].font
# # font.name = '宋体'
# # font.size = Pt(12)
# new_para = source_doc.add_paragraph(para.text, style='Normal')
# font = new_para.runs[0].font
# font.name = "宋体"
# font.size = Pt(12)
# new_para.paragraph_format.space_before = Pt(12)
# new_para.paragraph_format.first_line_indent = Pt(25)
# source_doc.element.body.insert(source_doc.element.body.index(target_paragraph._element) + 1, new_para._element)
#
# # 如果是表格，也在目标段落之后添加
# elif isinstance(element, CT_Tbl):
# source_doc.element.body.insert(source_doc.element.body.index(target_paragraph._element) + 1, element)
#
# source_doc.save("data/new_01系统导出模板.docx")
# import datetime
# start_time = datetime.datetime.now()
# # 参数：tables_dict、docx_file、save_path、template_path
# tables_dict = {
# "table13": "以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明",
# "table5": "收入费用表（2）",
# "table4": "收入费用表（1）",
# "table3": "资产负债表续表2",
# "table2": "资产负债表续表1",
# "table1": "资产负债表",
# "table9": "（17）其他应付款明细信息如下：",
# "table8": "（9）无形资产明细信息如下：",
# "table10": "（24）其他收入明细信息如下：",
# "table7": "（7）固定资产明细信息如下：",
# "table11": "（25）业务活动费用明细信息如下：",
# "table6": "（1）货币资金明细信息如下：",
# "table12": "（28）商品和服务费用明细信息如下："
# }
# # tables_dict = {'table1': '资产负债表', 'table2': '资产负债表续表1', 'table3': '资产负债表续表2', 'table4': '收入费用表（1）', 'table5': '收入费用表（2）', 'table6': '（1）货币资金明细信息如下：',
# # "table7": "（7）固定资产明细信息如下：", "table8": "（9）无形资产明细信息如下：", "table9": "（17）其他应付款明细信息如下：", "table10": "（24）其他收入明细信息如下：",
# # "table11": "（25）业务活动费用明细信息如下：", "table12": "（28）商品和服务费用明细信息如下：", }
# docx_file = r'data/3月23测试半成品.docx'
# document = Document(docx_file)
# data_result = get_choose_table(document, list(tables_dict.values()))
# print(data_result)
# generate_report(data_result, save_path=r'报告文件.docx', template_path=r'data/3月23测试模板.docx', tables_dict=tables_dict)
#
copy_table.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/3/7 17:29
# @Author : ctt
# @File : copy_table
# @Project : 表格复制
from copy import deepcopy
from docx import Document
"""
prep_p = p.insert_paragraph_before("段落前插入内容)
document.add_page_break() # 插入分页符
"""
import re
import json
import pandas as pd
from docx import Document
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table, _Row
from docx.text.paragraph import Paragraph
from docx.enum.section import WD_SECTION_START
from docx.enum.text import WD_BREAK
import docx
def iter_block_items(parent):
"""
Generate a reference to each paragraph and table child within *parent*,
in document order. Each returned value is an instance of either Table or
Paragraph. *parent* would most commonly be a reference to a main
Document object, but also works for a _Cell object, which itself can
contain paragraphs and tables.
"""
if isinstance(parent, _Document):
parent_elm = parent.element.body
elif isinstance(parent, _Cell):
parent_elm = parent._tc
elif isinstance(parent, _Row):
parent_elm = parent._tr
else:
raise ValueError("something's not right")
for child in parent_elm.iterchildren():
if isinstance(child, CT_P):
yield Paragraph(child, parent)
elif isinstance(child, CT_Tbl):
yield Table(child, parent)
def parase_table(table):
out_df = pd.DataFrame()
for i, row in enumerate(table.rows[:]):
row_content = []
for cell in row.cells:
c = cell.text.strip()
row_content.append(c)
out_df = pd.concat([out_df, pd.DataFrame(row_content)], axis=1, ignore_index=True)
return out_df.T
def get_table_position(para):
pattern = re.compile(r'(?<={{).*?(?=}})')
match = pattern.findall(para.text)
if match:
return match
return False
def get_choose_table(document, table_names: list):
'''
:param document:
:param table_names: 要提取的表格名称
:return: {'表名': [表1， 表1续表]}
'''
table_names_rule = '|'.join(table_names)
table_names_data = {}
[table_names_data.update({key: []}) for key in table_names]
dw_pattern = re.compile(r''+table_names_rule)
i = 1
for block in iter_block_items(document):
# 处理段落
if isinstance(block, Paragraph):
dw = dw_pattern.findall(block.text)
# 通过字符串匹配找到目标表格位置，并复制表格
elif isinstance(block, Table) and dw:
new_table = deepcopy(block.table)
table_names_data[dw[0]].append(new_table._element)
# 处理包含目标信息的表格（表头包含目标信息）
elif isinstance(block, Table):
# 按行解析表格并存储成df格式
table_df = parase_table(block.table)
if table_df[0][0] in table_names:
# print(table_names_data[table_df[0][0]])
new_table = deepcopy(block.table)
# print(new_table._element)
table_names_data[table_df[0][0] + "续表" + str(i)] = [new_table._element]
i += 1
# table_names_data[table_df[0][0]].append(new_table._element)
# print(table_names_data[table_df[0][0]])
return table_names_data
def new_document():
document = Document()
# 文档添加段落
para = document.add_paragraph()
# 在段落后面追加文本
# run = para.add_run()
# run.add_break()
return para._p
def generate_report(table_names_data, save_path, template_path, tables_dict):
document = Document(template_path)
pattern = re.compile(r'(?<={{).*?(?=}})')
# block 块对象主要包括标题、段落、图片、表、列表
# run 内联对象为块对象的组成部分，块对象的所有内容都包含在内联对象中，一个块对象由一个或多个内联对象组成。修改字体、字号、文字颜色需要用到run
# for block in iter_block_items(document):
for block in document.paragraphs:
if isinstance(block, Paragraph):
match = pattern.findall(block.text)
if match and "table" in match[0]:
table_name = match[0]
for _ in table_names_data[tables_dict[table_name]]:
# white_row = new_document()
# 在XML 级别上进行操作，即在元素之后直接添加内容，将任何尾部文本移动到新插入的元素后面，目的是使得新元素成为紧随其后的兄弟元素
# block._p.addnext(white_row)
block._p.addnext(_)
p = block._element
p.getparent().remove(p)
block._p = block._element = None
# 清除模板定义中的续表
pattern_clear = re.compile(r'(?<=续表)[0-9]')
for block in iter_block_items(document):
if isinstance(block, Paragraph):
match = pattern_clear.findall(block.text)
if match:
p = block._element
p.getparent().remove(p)
block._p = block._element = None
document.save(save_path)
if __name__ == '__main__':
import datetime
start_time = datetime.datetime.now()
# 参数：tables_dict、docx_file、save_path、template_path
tables_dict = {
"table13": "以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明",
"table5": "收入费用表（2）",
"table4": "收入费用表（1）",
"table3": "资产负债表续表2",
"table2": "资产负债表续表1",
"table1": "资产负债表",
"table9": "（17）其他应付款明细信息如下：",
"table8": "（9）无形资产明细信息如下：",
"table10": "（24）其他收入明细信息如下：",
"table7": "（7）固定资产明细信息如下：",
"table11": "（25）业务活动费用明细信息如下：",
"table6": "（1）货币资金明细信息如下：",
"table12": "（28）商品和服务费用明细信息如下："
}
# tables_dict = {'table1': '资产负债表', 'table2': '资产负债表续表1', 'table3': '资产负债表续表2', 'table4': '收入费用表（1）', 'table5': '收入费用表（2）', 'table6': '（1）货币资金明细信息如下：',
# "table7": "（7）固定资产明细信息如下：", "table8": "（9）无形资产明细信息如下：", "table9": "（17）其他应付款明细信息如下：", "table10": "（24）其他收入明细信息如下：",
# "table11": "（25）业务活动费用明细信息如下：", "table12": "（28）商品和服务费用明细信息如下：", }
docx_file = r'data/3月23测试半成品.docx'
document = Document(docx_file)
data_result = get_choose_table(document, list(tables_dict.values()))
print(data_result)
generate_report(data_result, save_path=r'data/报告文件.docx', template_path=r'data/new_财务报告模板.docx', tables_dict=tables_dict)
dao/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/12/1 10:11
from base.dao.base_dao import db
from dao.user import User
from dao.dataset import Dataset
from dao.dataset_field import DatasetField
from dao.template import Template
db.create_all()
dao/dataset.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : dataset
# @Author : LiuYan
# @Time : 2021/12/1 17:10
import datetime
from base.dao.base_dao import db
user_dataset_table = db.Table('user_dataset',
db.Column('user_id', db.Integer, db.ForeignKey('user.id'), primary_key=True),
db.Column('dataset_id', db.Integer, db.ForeignKey('dataset.id'), primary_key=True))
class Dataset(db.Model):
"""
user -- dataset --> ManyToMany
"""
__tablename__ = 'dataset'
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # *_主键id
project_name = db.Column(db.String(50), nullable=False) # *_项目标识
topic_name = db.Column(db.String(50), nullable=True) # __专题标识
datasource_name = db.Column(db.String(50), nullable=True) # __数据源标识
dataset_name = db.Column(db.String(50), nullable=False) # *_数据集名称
dataset_type = db.Column(db.Integer, nullable=False) # *_数据集类型 0: Dict | 1: List[Dict]
dataset_describe = db.Column(db.String(5000), nullable=False) # *_数据集描述
dataset_source_type = db.Column(db.Integer, nullable=False) # *_数据集来源类型 0: 业务API注册 | 1: 数据库SQL注册
dataset_url = db.Column(db.String(500), nullable=False) # *_数据集请求API接口
parameter = db.Column(db.String(500), nullable=True) # __API接口请求参数
database_type = db.Column(db.Integer, nullable=False) # *_数据库类型 0: mysql | 1: oracle | -1: (数据集来源类型为0)
database_config = db.Column(db.JSON, nullable=False) # *_数据库连接配置
database_sql = db.Column(db.String(5000), nullable=False) # *_数据库查询语句
status = db.Column(db.Integer, nullable=True) # __注册状态 0: 失败 1: 成功
create_by = db.Column(db.String(50), nullable=True) # __创建人
create_time = db.Column(db.DateTime, default=datetime.datetime.now()) # __创建日期
update_by = db.Column(db.String(50), nullable=True) # __更新人
update_time = db.Column(db.DateTime, onupdate=datetime.datetime.now()) # __更新日期
users = db.relationship('User', secondary=user_dataset_table, backref='users')
# 联合唯一索引
__table_args__ = (
db.UniqueConstraint('project_name', 'dataset_name', name='juc_project_name_dataset_name'),
)
dao/dataset_field.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : dataset_field
# @Author : LiuYan
# @Time : 2021/12/1 17:22
import datetime
from base.dao.base_dao import db
class DatasetField(db.Model):
"""
Dataset -- DatasetField -> OneToMany
"""
__tablename__ = 'dataset_field'
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # *_主键id
field_name = db.Column(db.String(50), nullable=False) # *_字段名
field_type = db.Column(db.Integer, nullable=False) # *_字段类型 0: Number 1: String
field_describe = db.Column(db.String(5000), nullable=False) # *_字段描述
status = db.Column(db.Integer, nullable=True) # __注册状态 0: 失败 1: 成功
create_by = db.Column(db.String(50), nullable=True) # __创建人
create_time = db.Column(db.DateTime, default=datetime.datetime.now()) # __创建日期
update_by = db.Column(db.String(50), nullable=True) # __更新人
update_time = db.Column(db.DateTime, onupdate=datetime.datetime.now()) # __更新日期
dataset_id = db.Column(db.Integer, db.ForeignKey('dataset.id'), nullable=False)
dataset = db.relationship('Dataset', backref='dataset_fields') # 关联关系，不是字段
# 联合唯一索引
__table_args__ = (
db.UniqueConstraint('field_name', 'dataset_id', name='juc_field_name_dataset_id'),
)
dao/template.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : template
# @Author : LiuYan
# @Time : 2021/12/1 17:13
import datetime
from base.dao.base_dao import db
class Template(db.Model):
"""
user -- template --> OneToMany
"""
__tablename__ = 'template'
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # *_主键id
project_name = db.Column(db.String(50), nullable=False) # *_项目名称
topic_name = db.Column(db.String(50), nullable=True) # __专题名称
datasource_name = db.Column(db.String(50), nullable=True) # __数据源名称
template_name = db.Column(db.String(50), nullable=False) # *_模板/报告名
template_path = db.Column(db.String(500), nullable=False) # *_模板路径
template_describe = db.Column(db.String(5000), nullable=False) # *_模板描述
create_by = db.Column(db.String(50), nullable=True) # __创建人
create_time = db.Column(db.DateTime, default=datetime.datetime.now()) # __创建日期
update_by = db.Column(db.String(50), nullable=True) # __更新人
update_time = db.Column(db.DateTime, onupdate=datetime.datetime.now()) # __更新日期
user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False)
dao/test/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/12/1 11:51
dao/test/role.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : role
# @Author : LiuYan
# @Time : 2021/12/1 11:25
from base.dao.base_dao import db
class Role(db.Model):
__tablename__ = '_role'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(64), unique=True)
dao/test/student.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : student
# @Author : LiuYan
# @Time : 2021/12/1 15:47
from base.dao.base_dao import db
class Student(db.Model):
__tablename__ = '_student'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(50), nullable=False)
age = db.Column(db.Integer, nullable=True)
if __name__ == '__main__':
"""
单表创建 增删改查
"""
# create
db.create_all()
# add
# student1 = Student(name='yan')
# student2 = Student(name='ying', age=18)
# db.session.add_all([student1, student2])
# db.session.commit()
# query
# list_student = Student.query.all()
# print(list_student)
# for student in list_student:
# print(student.id, student.name, student.age)
# update
# student = Student.query.filter(Student.name == 'yan').first()
# student.age = 20
# db.session.commit()
# delete
# student = Student.query.filter(Student.name == 'yan').first()
# db.session.delete(student)
# db.session.commit()
student = Student.query.filter(Student.name == 'yan')
for s in student:
db.session.delete(s)
db.session.commit()
pass
dao/test/test_dao.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : test_dao
# @Author : LiuYan
# @Time : 2021/12/1 11:51
from dao.test.role import Role
from dao.test.user import User
from base.dao.base_dao import db
if __name__ == '__main__':
"""
一对多表 创建 增删改查
"""
# create
db.create_all()
# add
new_user = User(name='liuyan')
db.session.add(new_user)
db.session.commit()
list_user = User.query.all()
print(list_user)
dao/test/user.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : user
# @Author : LiuYan
# @Time : 2021/12/1 10:15
from base.dao.base_dao import db, BaseDao
class User(db.Model):
__tablename__ = '_user'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(50), nullable=False)
age = db.Column(db.Integer, nullable=True)
role_id = db.Column(db.Integer, db.ForeignKey('_role.id'), nullable=False)
role = db.relationship('Role', backref='Users')
dao/user.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : user
# @Author : LiuYan
# @Time : 2021/12/1 17:13
import datetime
from base.dao.base_dao import db
class User(db.Model):
__tablename__ = 'user'
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # *_主键id
user_name = db.Column(db.String(50), nullable=False) # *_客户标识
project_name = db.Column(db.String(50), nullable=False) # *_项目标识
topic_name = db.Column(db.String(50), nullable=True) # __专题标识
create_by = db.Column(db.String(50), nullable=True) # __创建人
create_time = db.Column(db.DateTime, default=datetime.datetime.now()) # __创建日期
update_by = db.Column(db.String(50), nullable=True) # __更新人
update_time = db.Column(db.DateTime, onupdate=datetime.datetime.now()) # __更新日期
template = db.relationship('Template', backref='user') # 关联关系，不是字段
data/fdfs_client.conf 0 → 100644
# connect timeout in seconds
# default value is 30s
connect_timeout=30
# network timeout in seconds
# default value is 30s
network_timeout=60
# the base path to store log files
#base_path=/home/tarena/django-project/cc_shop1/cc_shop1/logs
# tracker_server can ocur more than once, and tracker_server format is
# "host:port", host can be hostname or ip address
tracker_server=114.115.215.96:22122
#standard log level as syslog, case insensitive, value list:
### emerg for emergency
### alert
### crit for critical
### error
### warn for warning
### notice
### info
### debug
log_level=info
# if use connection pool
# default value is false
# since V4.05
use_connection_pool = false
# connections whose the idle time exceeds this time will be closed
# unit: second
# default value is 3600
# since V4.05
connection_pool_max_idle_time = 3600
# if load FastDFS parameters from tracker server
# since V4.05
# default value is false
load_fdfs_parameters_from_tracker=false
# if use storage ID instead of IP address
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# default value is false
# since V4.05
use_storage_id = false
# specify storage ids filename, can use relative or absolute path
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# since V4.05
storage_ids_filename = storage_ids.conf
#HTTP settings
http.tracker_server_port=4000
#use "#include" directive to include HTTP other settiongs
##include http.conf
\ No newline at end of file
extract_factor.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/2/25 10:51
# @Author : ctt
# @File : 文本内容提取
# @Project : untitled1
import re
from docx import Document
import pandas as pd
class Other_Extract:
def __init__(self):
self.unitName_pattern = re.compile(r'(?<=部门（单位）名称：).*?(\n)')
self.unitChargePeople_pattern = re.compile(r'(?<=单位负责人：).*?(\n)')
self.financeChargePeople_pattern = re.compile(r'(?<=财务负责人：).*?(\n)')
self.filledPeople_pattern = re.compile(r'(?<=编制人：).*?(\n)')
self.year_pattern = re.compile(r'(?<=报送日期：).*?(\n)')
@staticmethod
def match(pattern, text):
pattern_group = pattern.search(text)
if pattern_group:
return pattern_group.group().strip()
return None
def extract_other_result(self, text):
unitName = self.match(self.unitName_pattern, text)
unitChargePeople = self.match(self.unitChargePeople_pattern, text)
financeChargePeople = self.match(self.financeChargePeople_pattern, text)
filledPeople = self.match(self.filledPeople_pattern, text)
year = self.match(self.year_pattern, text)
return {'unitName': unitName,
'unitChargePeople': unitChargePeople,
'financeChargePeople': financeChargePeople,
'filledPeople': filledPeople,
'year': year}
class Extract:
# {“主要职能”：””, “机构情况”：””, “人员情况”：””, “当年取得的主要事业成效”}
def __init__(self):
# self.main_functions = re.compile(r'(?<=[0-9][\.．]主要职能[。\n])(.|\n)*?(?=[0-9][\.．]机构情况[。\n])')
self.main_functions = re.compile(r'(?<=[0-9][\.．]主要职能[。\n])(.|\n)*?(?=（[一二三四五六七八九十]）当年取得的主要事业成效[。\n])')
# self.institutional_situation = re.compile(r'(?<=[0-9][\.．]机构情况[。\n])(.|\n)*?(?=[0-9][\.．]人员情况[。\n])')
# self.personnel_situation = re.compile(r'(?<=[0-9][\.．]人员情况[。\n])(.|\n)*?(?=（[一二三四五六七八九十]）当年取得的主要事业成效[。\n])')
self.business_results = re.compile(r'(?<=（[一二三四五六七八九十]）当年取得的主要事业成效[。\n])(.|\n)*?(?=[一二三四五六七八九十]、收入支出预算执行情况分析)')
# self.patterns = [self.main_functions, self.institutional_situation, self.personnel_situation, self.business_results]
@staticmethod
def match(pattern, text):
pattern_group = pattern.search(text)
if pattern_group:
return pattern_group.group().strip()
return None
def extract_result(self, text):
main_functions = self.match(self.main_functions, text)
# institutional_situation = self.match(self.institutional_situation, text)
# personnel_situation = self.match(self.personnel_situation, text)
business_results = self.match(self.business_results, text)
return {'主要职能': main_functions,
# '机构情况': institutional_situation,
# '人员情况': personnel_situation,
'当年取得的主要事业成效': business_results}
def get_text_from_docx(filepath):
'''
获取word文档的所有文本内容
:param filepath:
:return:
'''
document = Document(filepath)
contents = []
for paragraph in document.paragraphs:
if '<w:numPr>' in paragraph._element.xml:
contents.append('1.'+paragraph.text)
contents.append('\n')
else:
contents.append(paragraph.text)
contents.append('\n')
return ''.join(contents)
def get_cover_content_from_docx(filepath):
'''
获取word文档的所有文本内容
:param filepath:
:return:
'''
document = Document(filepath)
contents = []
# 第一步遍历段落存储信息
for paragraph in document.paragraphs:
if '<w:numPr>' in paragraph._element.xml:
contents.append('1.' + paragraph.text)
contents.append('\n')
else:
contents.append(paragraph.text)
contents.append('\n')
# 第二步取前15段获取封面标题信息
target_content = []
for content in contents[:14]:
if content.replace("\xa0", "").strip():
target_content.append(content.strip())
# print(contents[14:35])
# 第三步取15段获取其它信息
other_content = []
for temp_content in contents[14:35]:
if temp_content.replace("\xa0", "").strip():
other_content.append(temp_content.strip())
other_content.append('\n')
return "".join(target_content), ''.join(other_content)
if __name__ == '__main__':
new_path = "data/2022年度安岳县元坝镇人民政府部门决算分析报告(1).docx"
document = get_text_from_docx(new_path)
data = Extract().extract_result(document)
print(data)
# fifth_area_pattern = re.compile(r'(?<=[0-9][\.．]会计报表重要项目的明细信息及说明[。\n])(.|\n)*?(?=[0-9][\.．]需要说明的其他事项[。\n])')
# filepath = "wKjIbGQeSb6AUq1aAAgAABcLaMw312.docx"
# document = Document(filepath)
# documents = get_text_from_docx(filepath)
#
# area_group = fifth_area_pattern.search(documents)
# if area_group:
# area_text = area_group.group().strip("1.").strip()
# else:
# area_text = ""
#
# print(area_text)
# cover_contents, other_contents = get_cover_content_from_docx(filepath)
# cover_pattern = re.compile(r"([0-9]{0,4}).*(?=(财务报告))")
#
# # print(content)
# cover_group = cover_pattern.search(cover_contents)
# if cover_group:
# cover_text = cover_group.group().strip()
# else:
# cover_text = ""
#
# other_extract = Other_Extract()
# other_data = other_extract.extract_other_result(other_contents)
# other_data["reportTitle"] = cover_text
# print(other_data)
# extract = Extract()
# # path = r'D:\四川报告\相关代码\四川报告之文本内容提取\data'
# path = "data/temp.docx"
# result = extract.extract_result(path)
# print(result)
# for file in os.listdir(path):
# if file[-4:] == 'docx':
# filepath = os.path.join(path, file)
# paras = get_text_from_docx(filepath)
# print(paras)
# result = extract.extract_result(paras)
# print(result)
\ No newline at end of file
extract_table.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/2/17 11:57
# @Author : ctt
# @File : extract_table
# @Project : 从word中提取指定表格
import re
import json
import pandas as pd
from docx import Document
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table, _Row
from docx.text.paragraph import Paragraph
def iter_block_items(parent):
"""
Generate a reference to each paragraph and table child within *parent*,
in document order. Each returned value is an instance of either Table or
Paragraph. *parent* would most commonly be a reference to a main
Document object, but also works for a _Cell object, which itself can
contain paragraphs and tables.
"""
if isinstance(parent, _Document):
parent_elm = parent.element.body
elif isinstance(parent, _Cell):
parent_elm = parent._tc
elif isinstance(parent, _Row):
parent_elm = parent._tr
else:
raise ValueError("something's not right")
for child in parent_elm.iterchildren():
if isinstance(child, CT_P):
yield Paragraph(child, parent)
elif isinstance(child, CT_Tbl):
yield Table(child, parent)
def parase_table(table):
# 先定义结果dataframe
out_df = pd.DataFrame()
for i, row in enumerate(table.rows[:]):
row_content = []
for cell in row.cells:
c = cell.text.strip()
# print(c)
row_content.append(c)
out_df = pd.concat([out_df, pd.DataFrame(row_content)], axis=1, ignore_index=True)
return out_df.T
def get_choose_table(document, table_names: list):
table_names_rule = '|'.join(table_names)
table_names_data = {}
[table_names_data.update({key: ''}) for key in table_names]
# {'资产负债表': '', '收入费用表（1）': '', '收入费用表（2）': '',}
dw_pattern = re.compile(r''+table_names_rule)
for block in iter_block_items(document):
# 处理段落
if isinstance(block, Paragraph):
dw = dw_pattern.findall(block.text)
# 通过字符串匹配找到目标表格位置，并解析相应的内容
elif isinstance(block, Table) and dw:
# dw[0]为“资产负债表”
table_df = parase_table(block.table)
if "编制单位" in table_names_data:
pass
else:
table_names_data.update({'时间': table_df.iloc[0, -2]})
table_names_data.update({'单位': table_df.iloc[0, -1].replace(":", "：").split("：")[1]})
table_names_data.update({'编制单位': table_df.iloc[0, 0].replace(":", "：").split("：")[1]})
# 表头为“编制单位：德阳市旌阳区发展和改革局 2021年12月31日 2021年12月31日 单位：万元”，待删除
table_df.drop([0], inplace=True)
# 待将列字段 "0 1 2 3" 转换成 “项目 附注 年末数 年初数”
# print(table_df.iloc[0]) # 为“项目 附注 年末数 年初数”
table_df.rename(columns=table_df.iloc[0], inplace=True)
# 去掉当前表头即 “项目 附注 年末数 年初数”，因为rename后会有一行重复的内容，
table_df.drop(table_df.index[0], inplace=True)
# print(table_df)
table_names_data[dw[0]] = table_df
# 处理有目标信息的表格（表头包含目标信息）
elif isinstance(block, Table):
table_df = parase_table(block.table)
if table_df[0][0] in table_names:
update_table_key = table_df[0][0]
# 去掉表的头两行
table_df.drop([0, 1], inplace=True)
# 更新表头，即列名
table_df.rename(columns=table_df.iloc[0], inplace=True)
# 去掉第一行的值
table_df.drop(table_df.index[0], inplace=True)
# 拼接原表和续表
concated_df = pd.concat([table_names_data[update_table_key], table_df], ignore_index=True)
# 更新表名对应的value
table_names_data.update({update_table_key: concated_df})
# print(table_names_data)
# 将df内容格式转换为JSON格式
for table_key, table_value in table_names_data.items():
if isinstance(table_value, pd.DataFrame):
table_names_data.update({table_key: json.loads(table_value.to_json(orient='records', force_ascii=False))})
return table_names_data
def get_other_table(document, table_names: list):
table_names_rule = '|'.join(table_names)
table_names_data = {}
[table_names_data.update({key: ''}) for key in table_names]
# {'（2）以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明。': ''}
dw_pattern = re.compile(r''+table_names_rule)
for block in iter_block_items(document):
# 处理段落
if isinstance(block, Paragraph):
dw = dw_pattern.findall(block.text)
# 通过字符串匹配找到目标表格位置，并解析相应的内容
elif isinstance(block, Table) and dw:
# dw[0]为“资产负债表”
table_df = parase_table(block.table)
table_df.drop([0, 1], inplace=True)
# 选择目标df，注意这里是将其复制一份数据，若直接修改会引起警告
select_df = table_df.iloc[:, [0, 3]].copy()
select_df.rename(columns=select_df.iloc[0], inplace=True)
select_df.drop(table_df.index[0], inplace=True)
table_names_data[dw[0]] = select_df
# print(table_names_data)
# 将df内容格式转换为JSON格式
for table_key, table_value in table_names_data.items():
if isinstance(table_value, pd.DataFrame):
table_names_data.update({table_key: json.loads(table_value.to_json(orient='records', force_ascii=False))})
return table_names_data
def get_other1_table(document, table_names: list):
table_names_rule = '|'.join(table_names)
table_names_data = {}
[table_names_data.update({key: ''}) for key in table_names]
# [{'货币资金明细信息如下'}]
dw_pattern = re.compile(r'' + table_names_rule)
for block in iter_block_items(document):
# 处理段落
if isinstance(block, Paragraph):
dw = dw_pattern.findall(block.text)
# 通过字符串匹配找到目标表格位置，并解析相应的内容
elif isinstance(block, Table) and dw:
# dw[0]为“资产负债表”
table_df = parase_table(block.table)
table_df.drop([0, 1, 2], inplace=True)
# print(table_df)
# 选择目标df，注意这里是将其复制一份数据，若直接修改会引起警告
# select_df = table_df.iloc[:, [0, 3]].copy()
select_df = table_df.copy()
select_df.rename(columns=select_df.iloc[0], inplace=True)
# print(select_df)
select_df.drop(table_df.index[0], inplace=True)
table_names_data[dw[0]] = select_df
# print(table_names_data)
# 将df内容格式转换为JSON格式
for table_key, table_value in table_names_data.items():
if isinstance(table_value, pd.DataFrame):
table_names_data.update({table_key: json.loads(table_value.to_json(orient='records', force_ascii=False))})
return table_names_data
if __name__ == '__main__':
docx_file = r'data/3月23测试半成品.docx'
document = Document(docx_file)
table_names = ['货币资金明细信息如下']
print(get_other1_table(document, table_names))
# import datetime
# start_time = datetime.datetime.now()
# docx_file = r'data/四川报告模板.docx'
# document = Document(docx_file)
# data = get_choose_table(document, ['资产负债表', '收入费用表（1）', '收入费用表（2）'])
# # 处理资产负债表
# temp_list = data["资产负债表"]
# temp_dict = {}
#
# for temp in temp_list:
# temp_text = re.sub(":", "：", temp["项目"])
# if temp_text.endswith("："):
# temp_dict.update({"temp_key": temp_text})
# continue
# else:
# temp["上级项目"] = temp_dict["temp_key"].strip("：")
#
#
# # 处理收入费用表（1）
# temp_list_0 = data["收入费用表（1）"]
# temp_dict_0 = {"temp_key": "收入合计"}
# # updata_list = ["收入合计", "本年盈余"]
# for temp_0 in temp_list_0:
# if temp_0["项目"].strip() == "收入合计":
# temp_dict_0.update({"temp_key": "本年盈余"})
# else:
# if temp_0["项目"].strip() == "本年盈余":
# continue
# else:
# temp_0["上级项目"] = temp_dict_0["temp_key"]
#
# # 处理收入费用表（2）
# temp_list_1 = data["收入费用表（2）"]
# temp_dict_1 = {"temp_key": "收入合计"}
# # updata_list = ["收入合计", "本年盈余"]
# for temp_1 in temp_list_1:
# if temp_1["项目"].strip() == "收入合计":
# temp_dict_1.update({"temp_key": "本年盈余"})
# else:
# if temp_1["项目"].strip() == "本年盈余":
# continue
# else:
# temp_1["上级项目"] = temp_dict_1["temp_key"]
# print(data)
# end_time = datetime.datetime.now()
# print(start_time)
# print(end_time)
# print("耗时: {}秒".format(end_time - start_time))
fdfs_client/__init__.py 0 → 100644
# __init__.py
__version__ = '2.2.0'
VERSION = tuple(map(int, __version__.split('.')))
fdfs_client/client.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: client.py
'''
Client module for Fastdfs 3.08
author: scott yuan scottzer8@gmail.com
date: 2012-06-21
'''
import os
import sys
from fdfs_client.utils import *
from fdfs_client.tracker_client import *
from fdfs_client.storage_client import *
from fdfs_client.exceptions import *
def get_tracker_conf(conf_path='client.conf'):
cf = Fdfs_ConfigParser()
tracker = {}
try:
cf.read(conf_path)
timeout = cf.getint('__config__', 'connect_timeout')
tracker_list = cf.get('__config__', 'tracker_server')
if isinstance(tracker_list, str):
tracker_list = [tracker_list]
tracker_ip_list = []
for tr in tracker_list:
tracker_ip, tracker_port = tr.split(':')
tracker_ip_list.append(tracker_ip)
tracker['host_tuple'] = tuple(tracker_ip_list)
tracker['port'] = int(tracker_port)
tracker['timeout'] = timeout
tracker['name'] = 'Tracker Pool'
except:
raise
return tracker
class Fdfs_client(object):
'''
Class Fdfs_client implemented Fastdfs client protol ver 3.08.
It's useful upload, download, delete file to or from fdfs server, etc. It's uses
connection pool to manage connection to server.
'''
def __init__(self, trackers, poolclass=ConnectionPool):
self.trackers = trackers
self.tracker_pool = poolclass(**self.trackers)
self.timeout = self.trackers['timeout']
return None
def __del__(self):
try:
self.pool.destroy()
self.pool = None
except:
pass
def upload_by_filename(self, filename, meta_dict=None):
'''
Upload a file to Storage server.
arguments:
@filename: string, name of file that will be uploaded
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
} meta_dict can be null
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : local_file_name,
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(uploading)')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_by_filename(tc, store_serv, filename, meta_dict)
def upload_by_file(self, filename, meta_dict=None):
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(uploading)')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_by_file(tc, store_serv, filename, meta_dict)
def upload_by_buffer(self, filebuffer, file_ext_name=None, meta_dict=None):
'''
Upload a buffer to Storage server.
arguments:
@filebuffer: string, buffer
@file_ext_name: string, file extend name
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
}
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : '',
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
if not filebuffer:
raise DataError('[-] Error: argument filebuffer can not be null.')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_by_buffer(tc, store_serv, filebuffer, file_ext_name, meta_dict)
def upload_slave_by_filename(self, filename, remote_file_id, prefix_name, meta_dict=None):
'''
Upload slave file to Storage server.
arguments:
@filename: string, local file name
@remote_file_id: string, remote file id
@prefix_name: string
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
}
@return dictionary {
'Status' : 'Upload slave successed.',
'Local file name' : local_filename,
'Uploaded size' : upload_size,
'Remote file id' : remote_file_id,
'Storage IP' : storage_ip
}
'''
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(uploading slave)')
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(uploading slave)')
if not prefix_name:
raise DataError('[-] Error: prefix_name can not be null.')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_with_group(group_name)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
try:
ret_dict = store.storage_upload_slave_by_filename(tc, store_serv, filename, prefix_name, remote_filename,
meta_dict=None)
except:
raise
ret_dict['Status'] = 'Upload slave file successed.'
return ret_dict
def upload_slave_by_file(self, filename, remote_file_id, prefix_name, meta_dict=None):
'''
Upload slave file to Storage server.
arguments:
@filename: string, local file name
@remote_file_id: string, remote file id
@prefix_name: string
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
}
@return dictionary {
'Status' : 'Upload slave successed.',
'Local file name' : local_filename,
'Uploaded size' : upload_size,
'Remote file id' : remote_file_id,
'Storage IP' : storage_ip
}
'''
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(uploading slave)')
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(uploading slave)')
if not prefix_name:
raise DataError('[-] Error: prefix_name can not be null.')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_with_group(group_name)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
try:
ret_dict = store.storage_upload_slave_by_file(tc, store_serv, filename, prefix_name, remote_filename,
meta_dict=None)
except:
raise
ret_dict['Status'] = 'Upload slave file successed.'
return ret_dict
def upload_slave_by_buffer(self, filebuffer, remote_file_id, meta_dict=None, file_ext_name=None):
'''
Upload slave file by buffer
arguments:
@filebuffer: string
@remote_file_id: string
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
}
@return dictionary {
'Status' : 'Upload slave successed.',
'Local file name' : local_filename,
'Uploaded size' : upload_size,
'Remote file id' : remote_file_id,
'Storage IP' : storage_ip
}
'''
if not filebuffer:
raise DataError('[-] Error: argument filebuffer can not be null.')
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(uploading slave)')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_slave_by_buffer(tc, store_serv, filebuffer, remote_filename, meta_dict,
file_ext_name)
def upload_appender_by_filename(self, local_filename, meta_dict=None):
'''
Upload an appender file by filename.
arguments:
@local_filename: string
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
} Notice: it can be null
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : '',
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
isfile, errmsg = fdfs_check_file(local_filename)
if not isfile:
raise DataError(errmsg + '(uploading appender)')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_appender_by_filename(tc, store_serv, local_filename, meta_dict)
def upload_appender_by_file(self, local_filename, meta_dict=None):
'''
Upload an appender file by file.
arguments:
@local_filename: string
@meta_dict: dictionary e.g.:{
'ext_name' : 'jpg',
'file_size' : '10240B',
'width' : '160px',
'hight' : '80px'
} Notice: it can be null
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : '',
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
isfile, errmsg = fdfs_check_file(local_filename)
if not isfile:
raise DataError(errmsg + '(uploading appender)')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_appender_by_file(tc, store_serv, local_filename, meta_dict)
def upload_appender_by_buffer(self, filebuffer, file_ext_name=None, meta_dict=None):
'''
Upload a buffer to Storage server.
arguments:
@filebuffer: string
@file_ext_name: string, can be null
@meta_dict: dictionary, can be null
@return dict {
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : 'Upload successed.',
'Local file name' : '',
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
} if success else None
'''
if not filebuffer:
raise DataError('[-] Error: argument filebuffer can not be null.')
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_stor_without_group()
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_upload_appender_by_buffer(tc, store_serv, filebuffer, meta_dict, file_ext_name)
def delete_file(self, remote_file_id):
'''
Delete a file from Storage server.
arguments:
@remote_file_id: string, file_id of file that is on storage server
@return tuple ('Delete file successed.', remote_file_id, storage_ip)
'''
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(in delete file)')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_delete_file(tc, store_serv, remote_filename)
def download_to_file(self, local_filename, remote_file_id, offset=0, down_bytes=0):
'''
Download a file from Storage server.
arguments:
@local_filename: string, local name of file
@remote_file_id: string, file_id of file that is on storage server
@offset: long
@downbytes: long
@return dict {
'Remote file_id' : remote_file_id,
'Content' : local_filename,
'Download size' : downloaded_size,
'Storage IP' : storage_ip
}
'''
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(in download file)')
group_name, remote_filename = tmp
if not offset:
file_offset = int(offset)
if not down_bytes:
download_bytes = int(down_bytes)
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_fetch(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_download_to_file(tc, store_serv, local_filename, file_offset, download_bytes,
remote_filename)
def download_to_buffer(self, remote_file_id, offset=0, down_bytes=0):
'''
Download a file from Storage server and store in buffer.
arguments:
@remote_file_id: string, file_id of file that is on storage server
@offset: long
@down_bytes: long
@return dict {
'Remote file_id' : remote_file_id,
'Content' : file_buffer,
'Download size' : downloaded_size,
'Storage IP' : storage_ip
}
'''
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(in download file)')
group_name, remote_filename = tmp
if not offset:
file_offset = int(offset)
if not down_bytes:
download_bytes = int(down_bytes)
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_fetch(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
file_buffer = None
return store.storage_download_to_buffer(tc, store_serv, file_buffer, file_offset, download_bytes,
remote_filename)
def list_one_group(self, group_name):
'''
List one group information.
arguments:
@group_name: string, group name will be list
@return Group_info, instance
'''
tc = Tracker_client(self.tracker_pool)
return tc.tracker_list_one_group(group_name)
def list_servers(self, group_name, storage_ip=None):
'''
List all storage servers information in a group
arguments:
@group_name: string
@return dictionary {
'Group name' : group_name,
'Servers' : server list,
}
'''
tc = Tracker_client(self.tracker_pool)
return tc.tracker_list_servers(group_name, storage_ip)
def list_all_groups(self):
'''
List all group information.
@return dictionary {
'Groups count' : group_count,
'Groups' : list of groups
}
'''
tc = Tracker_client(self.tracker_pool)
return tc.tracker_list_all_groups()
def get_meta_data(self, remote_file_id):
'''
Get meta data of remote file.
arguments:
@remote_fileid: string, remote file id
@return dictionary, meta data
'''
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(in get meta data)')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_get_metadata(tc, store_serv, remote_filename)
def set_meta_data(self, remote_file_id, meta_dict, op_flag=STORAGE_SET_METADATA_FLAG_OVERWRITE):
'''
Set meta data of remote file.
arguments:
@remote_file_id: string
@meta_dict: dictionary
@op_flag: char, 'O' for overwrite, 'M' for merge
@return dictionary {
'Status' : status,
'Storage IP' : storage_ip
}
'''
tmp = split_remote_fileid(remote_file_id)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(in set meta data)')
group_name, remote_filename = tmp
tc = Tracker_client(self.tracker_pool)
try:
store_serv = tc.tracker_query_storage_update(group_name, remote_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
status = store.storage_set_metadata(tc, store_serv, remote_filename, meta_dict)
except (ConnectionError, ResponseError, DataError):
raise
# if status == 2:
# raise DataError('[-] Error: remote file %s is not exist.' % remote_file_id)
if status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
ret_dict = {}
ret_dict['Status'] = 'Set meta data success.'
ret_dict['Storage IP'] = store_serv.ip_addr
return ret_dict
def append_by_filename(self, local_filename, remote_fileid):
isfile, errmsg = fdfs_check_file(local_filename)
if not isfile:
raise DataError(errmsg + '(append)')
tmp = split_remote_fileid(remote_fileid)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(append)')
group_name, appended_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appended_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_append_by_filename(tc, store_serv, local_filename, appended_filename)
def append_by_file(self, local_filename, remote_fileid):
isfile, errmsg = fdfs_check_file(local_filename)
if not isfile:
raise DataError(errmsg + '(append)')
tmp = split_remote_fileid(remote_fileid)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(append)')
group_name, appended_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appended_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_append_by_file(tc, store_serv, local_filename, appended_filename)
def append_by_buffer(self, file_buffer, remote_fileid):
if not file_buffer:
raise DataError('[-] Error: file_buffer can not be null.')
tmp = split_remote_fileid(remote_fileid)
if not tmp:
raise DataError('[-] Error: remote_file_id is invalid.(append)')
group_name, appended_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appended_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_append_by_buffer(tc, store_serv, file_buffer, appended_filename)
def truncate_file(self, truncated_filesize, appender_fileid):
'''
Truncate file in Storage server.
arguments:
@truncated_filesize: long
@appender_fileid: remote_fileid
@return: dictionary {
'Status' : 'Truncate successed.',
'Storage IP' : storage_ip
}
'''
trunc_filesize = int(truncated_filesize)
tmp = split_remote_fileid(appender_fileid)
if not tmp:
raise DataError('[-] Error: appender_fileid is invalid.(truncate)')
group_name, appender_filename = tmp
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appender_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_truncate_file(tc, store_serv, trunc_filesize, appender_filename)
def modify_by_filename(self, filename, appender_fileid, offset=0):
'''
Modify a file in Storage server by file.
arguments:
@filename: string, local file name
@offset: long, file offset
@appender_fileid: string, remote file id
@return: dictionary {
'Status' : 'Modify successed.',
'Storage IP' : storage_ip
}
'''
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(modify)')
filesize = os.stat(filename).st_size
tmp = split_remote_fileid(appender_fileid)
if not tmp:
raise DataError('[-] Error: remote_fileid is invalid.(modify)')
group_name, appender_filename = tmp
if not offset:
file_offset = int(offset)
else:
file_offset = 0
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appender_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_modify_by_filename(tc, store_serv, filename, file_offset, filesize, appender_filename)
def modify_by_file(self, filename, appender_fileid, offset=0):
'''
Modify a file in Storage server by file.
arguments:
@filename: string, local file name
@offset: long, file offset
@appender_fileid: string, remote file id
@return: dictionary {
'Status' : 'Modify successed.',
'Storage IP' : storage_ip
}
'''
isfile, errmsg = fdfs_check_file(filename)
if not isfile:
raise DataError(errmsg + '(modify)')
filesize = os.stat(filename).st_size
tmp = split_remote_fileid(appender_fileid)
if not tmp:
raise DataError('[-] Error: remote_fileid is invalid.(modify)')
group_name, appender_filename = tmp
if not offset:
file_offset = int(offset)
else:
file_offset = 0
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appender_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_modify_by_file(tc, store_serv, filename, file_offset, filesize, appender_filename)
def modify_by_buffer(self, filebuffer, appender_fileid, offset=0):
'''
Modify a file in Storage server by buffer.
arguments:
@filebuffer: string, file buffer
@offset: long, file offset
@appender_fileid: string, remote file id
@return: dictionary {
'Status' : 'Modify successed.',
'Storage IP' : storage_ip
}
'''
if not filebuffer:
raise DataError('[-] Error: filebuffer can not be null.(modify)')
filesize = len(filebuffer)
tmp = split_remote_fileid(appender_fileid)
if not tmp:
raise DataError('[-] Error: remote_fileid is invalid.(modify)')
group_name, appender_filename = tmp
if not offset:
file_offset = int(offset)
else:
file_offset = 0
tc = Tracker_client(self.tracker_pool)
store_serv = tc.tracker_query_storage_update(group_name, appender_filename)
store = Storage_client(store_serv.ip_addr, store_serv.port, self.timeout)
return store.storage_modify_by_buffer(tc, store_serv, filebuffer, file_offset, filesize, appender_filename)
fdfs_client/connection.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: connection.py
import socket
import os
import sys
import time
import random
from itertools import chain
from fdfs_client.exceptions import (
FDFSError,
ConnectionError,
ResponseError,
InvaildResponse,
DataError
)
# start class Connection
class Connection(object):
'''Manage TCP comunication to and from Fastdfs Server.'''
def __init__(self, **conn_kwargs):
self.pid = os.getpid()
self.host_tuple = conn_kwargs['host_tuple']
self.remote_port = conn_kwargs['port']
self.remote_addr = None
self.timeout = conn_kwargs['timeout']
self._sock = None
def __del__(self):
try:
self.disconnect()
except:
pass
def connect(self):
'''Connect to fdfs server.'''
if self._sock:
return
try:
sock = self._connect()
except socket.error as e:
raise ConnectionError(self._errormessage(e))
self._sock = sock
# print '[+] Create a connection success.'
# print '\tLocal address is %s:%s.' % self._sock.getsockname()
# print '\tRemote address is %s:%s' % (self.remote_addr, self.remote_port)
def _connect(self):
'''Create TCP socket. The host is random one of host_tuple.'''
self.remote_addr = random.choice(self.host_tuple)
# print '[+] Connecting... remote: %s:%s' % (self.remote_addr, self.remote_port)
# sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# sock.settimeout(self.timeout)
sock = socket.create_connection((self.remote_addr, self.remote_port), self.timeout)
return sock
def disconnect(self):
'''Disconnect from fdfs server.'''
if self._sock is None:
return
try:
self._sock.close()
except socket.error as e:
raise ConnectionError(self._errormessage(e))
self._sock = None
def get_sock(self):
return self._sock
def _errormessage(self, exception):
# args for socket.error can either be (errno, "message")
# or just "message" '''
if len(exception.args) == 1:
return "[-] Error: connect to %s:%s. %s." % (self.remote_addr, self.remote_port, exception.args[0])
else:
return "[-] Error: %s connect to %s:%s. %s." % \
(exception.args[0], self.remote_addr, self.remote_port, exception.args[1])
# end class Connection
# start ConnectionPool
class ConnectionPool(object):
'''Generic Connection Pool'''
def __init__(self, name='', conn_class=Connection,
max_conn=None, **conn_kwargs):
self.pool_name = name
self.pid = os.getpid()
self.conn_class = conn_class
self.max_conn = max_conn or 2 ** 31
self.conn_kwargs = conn_kwargs
self._conns_created = 0
self._conns_available = []
self._conns_inuse = set()
# print '[+] Create a connection pool success, name: %s.' % self.pool_name
def _check_pid(self):
if self.pid != os.getpid():
self.destroy()
self.__init__(self.conn_class, self.max_conn, **self.conn_kwargs)
def make_conn(self):
'''Create a new connection.'''
if self._conns_created >= self.max_conn:
raise ConnectionError('[-] Error: Too many connections.')
num_try = 10
while True:
try:
if num_try <= 0:
sys.exit()
conn_instance = self.conn_class(**self.conn_kwargs)
conn_instance.connect()
self._conns_created += 1
break
except ConnectionError as e:
print(e)
num_try -= 1
conn_instance = None
return conn_instance
def get_connection(self):
'''Get a connection from pool.'''
self._check_pid()
try:
conn = self._conns_available.pop()
# print '[+] Get a connection from pool %s.' % self.pool_name
# print '\tLocal address is %s:%s.' % conn._sock.getsockname()
# print '\tRemote address is %s:%s' % (conn.remote_addr, conn.remote_port)
except IndexError:
conn = self.make_conn()
self._conns_inuse.add(conn)
return conn
def remove(self, conn):
'''Remove connection from pool.'''
if conn in self._conns_inuse:
self._conns_inuse.remove(conn)
self._conns_created -= 1
if conn in self._conns_available:
self._conns_available.remove(conn)
self._conns_created -= 1
def destroy(self):
'''Disconnect all connections in the pool.'''
all_conns = chain(self._conns_inuse, self._conns_available)
for conn in all_conns:
conn.disconnect()
# print '[-] Destroy connection pool %s.' % self.pool_name
def release(self, conn):
'''Release the connection back to the pool.'''
self._check_pid()
if conn.pid == self.pid:
self._conns_inuse.remove(conn)
self._conns_available.append(conn)
# print '[-] Release connection back to pool %s.' % self.pool_name
# end ConnectionPool class
def tcp_recv_response(conn, bytes_size, buffer_size=4096):
'''Receive response from server.
It is not include tracker header.
arguments:
@conn: connection
@bytes_size: int, will be received byte_stream size
@buffer_size: int, receive buffer size
@Return: tuple,(response, received_size)
'''
recv_buff = []
total_size = 0
try:
while bytes_size > 0:
resp = conn._sock.recv(buffer_size)
recv_buff.append(resp)
total_size += len(resp)
bytes_size -= len(resp)
except (socket.error, socket.timeout) as e:
raise ConnectionError('[-] Error: while reading from socket: (%s)' % e.args)
return (b''.join(recv_buff), total_size)
def tcp_send_data(conn, bytes_stream):
'''Send buffer to server.
It is not include tracker header.
arguments:
@conn: connection
@bytes_stream: trasmit buffer
@Return bool
'''
try:
conn._sock.sendall(bytes_stream)
except (socket.error, socket.timeout) as e:
raise ConnectionError('[-] Error: while writting to socket: (%s)' % e.args)
fdfs_client/exceptions.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: exceptions.py
'''Core exceptions raised by fdfs client'''
class FDFSError(Exception):
pass
class ConnectionError(FDFSError):
pass
class ResponseError(FDFSError):
pass
class InvaildResponse(FDFSError):
pass
class DataError(FDFSError):
pass
fdfs_client/fdfs_protol.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: fdfs_protol.py
import struct
import socket
from fdfs_client.exceptions import (
FDFSError,
ConnectionError,
ResponseError,
InvaildResponse,
DataError
)
# define FDFS protol constans
TRACKER_PROTO_CMD_STORAGE_JOIN = 81
FDFS_PROTO_CMD_QUIT = 82
TRACKER_PROTO_CMD_STORAGE_BEAT = 83 # storage heart beat
TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE = 84 # report disk usage
TRACKER_PROTO_CMD_STORAGE_REPLICA_CHG = 85 # repl new storage servers
TRACKER_PROTO_CMD_STORAGE_SYNC_SRC_REQ = 86 # src storage require sync
TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ = 87 # dest storage require sync
TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY = 88 # sync done notify
TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT = 89 # report src last synced time as dest server
TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_QUERY = 79 # dest storage query sync src storage server
TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED = 78 # storage server report it's ip changed
TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ = 77 # storage server request storage server's changelog
TRACKER_PROTO_CMD_STORAGE_REPORT_STATUS = 76 # report specified storage server status
TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ = 75 # storage server request parameters
TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FREE = 74 # storage report trunk free space
TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID = 73 # storage report current trunk file id
TRACKER_PROTO_CMD_STORAGE_FETCH_TRUNK_FID = 72 # storage get current trunk file id
TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_START = 61 # start of tracker get system data files
TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_END = 62 # end of tracker get system data files
TRACKER_PROTO_CMD_TRACKER_GET_ONE_SYS_FILE = 63 # tracker get a system data file
TRACKER_PROTO_CMD_TRACKER_GET_STATUS = 64 # tracker get status of other tracker
TRACKER_PROTO_CMD_TRACKER_PING_LEADER = 65 # tracker ping leader
TRACKER_PROTO_CMD_TRACKER_NOTIFY_NEXT_LEADER = 66 # notify next leader to other trackers
TRACKER_PROTO_CMD_TRACKER_COMMIT_NEXT_LEADER = 67 # commit next leader to other trackers
TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP = 90
TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS = 91
TRACKER_PROTO_CMD_SERVER_LIST_STORAGE = 92
TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE = 93
TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE = 101
TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE = 102
TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE = 103
TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE = 104
TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL = 105
TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL = 106
TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL = 107
TRACKER_PROTO_CMD_RESP = 100
FDFS_PROTO_CMD_ACTIVE_TEST = 111 # active test, tracker and storage both support since V1.28
STORAGE_PROTO_CMD_REPORT_CLIENT_IP = 9 # ip as tracker client
STORAGE_PROTO_CMD_UPLOAD_FILE = 11
STORAGE_PROTO_CMD_DELETE_FILE = 12
STORAGE_PROTO_CMD_SET_METADATA = 13
STORAGE_PROTO_CMD_DOWNLOAD_FILE = 14
STORAGE_PROTO_CMD_GET_METADATA = 15
STORAGE_PROTO_CMD_SYNC_CREATE_FILE = 16
STORAGE_PROTO_CMD_SYNC_DELETE_FILE = 17
STORAGE_PROTO_CMD_SYNC_UPDATE_FILE = 18
STORAGE_PROTO_CMD_SYNC_CREATE_LINK = 19
STORAGE_PROTO_CMD_CREATE_LINK = 20
STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE = 21
STORAGE_PROTO_CMD_QUERY_FILE_INFO = 22
STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE = 23 # create appender file
STORAGE_PROTO_CMD_APPEND_FILE = 24 # append file
STORAGE_PROTO_CMD_SYNC_APPEND_FILE = 25
STORAGE_PROTO_CMD_FETCH_ONE_PATH_BINLOG = 26 # fetch binlog of one store path
STORAGE_PROTO_CMD_RESP = TRACKER_PROTO_CMD_RESP
STORAGE_PROTO_CMD_UPLOAD_MASTER_FILE = STORAGE_PROTO_CMD_UPLOAD_FILE
STORAGE_PROTO_CMD_TRUNK_ALLOC_SPACE = 27 # since V3.00
STORAGE_PROTO_CMD_TRUNK_ALLOC_CONFIRM = 28 # since V3.00
STORAGE_PROTO_CMD_TRUNK_FREE_SPACE = 29 # since V3.00
STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG = 30 # since V3.00
STORAGE_PROTO_CMD_TRUNK_GET_BINLOG_SIZE = 31 # since V3.07
STORAGE_PROTO_CMD_TRUNK_DELETE_BINLOG_MARKS = 32 # since V3.07
STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE = 33 # since V3.07
STORAGE_PROTO_CMD_MODIFY_FILE = 34 # since V3.08
STORAGE_PROTO_CMD_SYNC_MODIFY_FILE = 35 # since V3.08
STORAGE_PROTO_CMD_TRUNCATE_FILE = 36 # since V3.08
STORAGE_PROTO_CMD_SYNC_TRUNCATE_FILE = 37 # since V3.08
# for overwrite all old metadata
STORAGE_SET_METADATA_FLAG_OVERWRITE = 'O'
STORAGE_SET_METADATA_FLAG_OVERWRITE_STR = "O"
# for replace, insert when the meta item not exist, otherwise update it
STORAGE_SET_METADATA_FLAG_MERGE = 'M'
STORAGE_SET_METADATA_FLAG_MERGE_STR = "M"
FDFS_RECORD_SEPERATOR = '\x01'
FDFS_FIELD_SEPERATOR = '\x02'
# common constants
FDFS_GROUP_NAME_MAX_LEN = 16
IP_ADDRESS_SIZE = 16
FDFS_PROTO_PKG_LEN_SIZE = 8
FDFS_PROTO_CMD_SIZE = 1
FDFS_PROTO_STATUS_SIZE = 1
FDFS_PROTO_IP_PORT_SIZE = (IP_ADDRESS_SIZE + 6)
FDFS_MAX_SERVERS_EACH_GROUP = 32
FDFS_MAX_GROUPS = 512
FDFS_MAX_TRACKERS = 16
FDFS_DOMAIN_NAME_MAX_LEN = 128
FDFS_MAX_META_NAME_LEN = 64
FDFS_MAX_META_VALUE_LEN = 256
FDFS_FILE_PREFIX_MAX_LEN = 16
FDFS_LOGIC_FILE_PATH_LEN = 10
FDFS_TRUE_FILE_PATH_LEN = 6
FDFS_FILENAME_BASE64_LENGTH = 27
FDFS_TRUNK_FILE_INFO_LEN = 16
FDFS_FILE_EXT_NAME_MAX_LEN = 6
FDFS_SPACE_SIZE_BASE_INDEX = 2 # storage space size based (MB)
FDFS_UPLOAD_BY_BUFFER = 1
FDFS_UPLOAD_BY_FILENAME = 2
FDFS_UPLOAD_BY_FILE = 3
FDFS_DOWNLOAD_TO_BUFFER = 1
FDFS_DOWNLOAD_TO_FILE = 2
FDFS_NORMAL_LOGIC_FILENAME_LENGTH = (
FDFS_LOGIC_FILE_PATH_LEN + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1)
FDFS_TRUNK_FILENAME_LENGTH = (
FDFS_TRUE_FILE_PATH_LEN + FDFS_FILENAME_BASE64_LENGTH + FDFS_TRUNK_FILE_INFO_LEN + 1 + FDFS_FILE_EXT_NAME_MAX_LEN)
FDFS_TRUNK_LOGIC_FILENAME_LENGTH = (FDFS_TRUNK_FILENAME_LENGTH + (FDFS_LOGIC_FILE_PATH_LEN - FDFS_TRUE_FILE_PATH_LEN))
FDFS_VERSION_SIZE = 6
TRACKER_QUERY_STORAGE_FETCH_BODY_LEN = (FDFS_GROUP_NAME_MAX_LEN + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE)
TRACKER_QUERY_STORAGE_STORE_BODY_LEN = (FDFS_GROUP_NAME_MAX_LEN + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE + 1)
# status code, order is important!
FDFS_STORAGE_STATUS_INIT = 0
FDFS_STORAGE_STATUS_WAIT_SYNC = 1
FDFS_STORAGE_STATUS_SYNCING = 2
FDFS_STORAGE_STATUS_IP_CHANGED = 3
FDFS_STORAGE_STATUS_DELETED = 4
FDFS_STORAGE_STATUS_OFFLINE = 5
FDFS_STORAGE_STATUS_ONLINE = 6
FDFS_STORAGE_STATUS_ACTIVE = 7
FDFS_STORAGE_STATUS_RECOVERY = 9
FDFS_STORAGE_STATUS_NONE = 99
class Storage_server(object):
'''Class storage server for upload.'''
def __init__(self):
self.ip_addr = None
self.port = None
self.group_name = ''
self.store_path_index = 0
# Class tracker_header
class Tracker_header(object):
'''
Class for Pack or Unpack tracker header
struct tracker_header{
char pkg_len[FDFS_PROTO_PKG_LEN_SIZE],
char cmd,
char status,
}
'''
def __init__(self):
self.fmt = '!QBB' # pkg_len[FDFS_PROTO_PKG_LEN_SIZE] + cmd + status
self.st = struct.Struct(self.fmt)
self.pkg_len = 0
self.cmd = 0
self.status = 0
def _pack(self, pkg_len=0, cmd=0, status=0):
return self.st.pack(pkg_len, cmd, status)
def _unpack(self, bytes_stream):
self.pkg_len, self.cmd, self.status = self.st.unpack(bytes_stream)
return True
def header_len(self):
return self.st.size
def send_header(self, conn):
'''Send Tracker header to server.'''
header = self._pack(self.pkg_len, self.cmd, self.status)
try:
conn._sock.sendall(header)
except (socket.error, socket.timeout) as e:
raise ConnectionError('[-] Error: while writting to socket: %s' % (e.args,))
def recv_header(self, conn):
'''Receive response from server.
if sucess, class member (pkg_len, cmd, status) is response.
'''
try:
header = conn._sock.recv(self.header_len())
except (socket.error, socket.timeout) as e:
raise ConnectionError('[-] Error: while reading from socket: %s' % (e.args,))
self._unpack(header)
def fdfs_pack_metadata(meta_dict):
ret = ''
for key in meta_dict:
ret += '%s%c%s%c' % (key, FDFS_FIELD_SEPERATOR, meta_dict[key], FDFS_RECORD_SEPERATOR)
return ret[0:-1]
def fdfs_unpack_metadata(bytes_stream):
li = bytes_stream.split(FDFS_RECORD_SEPERATOR)
return dict([item.split(FDFS_FIELD_SEPERATOR) for item in li])
fdfs_client/fdfs_test.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: fdfs_test.py
import os
import sys
import time
try:
from fdfs_client.client import *
from fdfs_client.exceptions import *
except ImportError:
import_path = os.path.abspath('../')
sys.path.append(import_path)
from fdfs_client.client import *
from fdfs_client.exceptions import *
def usage():
s = 'Usage: python fdfs_test.py {options} [{local_filename} [{remote_file_id}]]\n'
s += 'options: upfile, upbuffer, downfile, downbuffer, delete, listgroup, listserv\n'
s += ' upslavefile, upslavebuffer, upappendfile, upappendbuffer\n'
s += '\tupfile {local_filename}\n'
s += '\tupbuffer {local_filename}\n'
s += '\tdownfile {local_filename} {remote_file_id}\n'
s += '\tdownbuffer {remote_file_id}\n'
s += '\tdelete {remote_file_id}\n'
s += '\tlistgroup {group_name}\n'
s += '\tlistall \n'
s += '\tlistsrv {group_name} [storage_ip]\n'
s += '\tsetmeta {remote_file_id}\n'
s += '\tgetmeta {remote_file_id}\n'
s += '\tupslavefile {local_filename} {remote_fileid} {prefix_name}\n'
s += '\tupappendfile {local_filename}\n'
s += '\ttruncate {truncate_filesize} {remote_fileid}\n'
s += '\tmodifyfile {local_filename} {remote_fileid} {file_offset}\n'
s += '\tmodifybuffer {local_filename} {remote_fileid} {file_offset}\n'
s += 'e.g.: python fdfs_test.py upfile test'
print(s)
sys.exit(0)
if len(sys.argv) < 2:
usage()
client = Fdfs_client('client.conf')
def upfile_func():
# Upload by filename
# usage: python fdfs_test.py upfile {local_filename}
if len(sys.argv) < 3:
usage()
return None
try:
local_filename = sys.argv[2]
file_size = os.stat(local_filename).st_size
# meta_buffer can be null.
meta_dict = {
'ext_name': 'py',
'file_size': str(file_size) + 'B'
}
t1 = time.time()
ret_dict = client.upload_by_filename(local_filename, meta_dict)
t2 = time.time()
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
print('[+] time consume: %fs' % (t2 - t1))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upfileex_func():
# Upload by file
# usage: python fdfs_test.py upfileex {local_filename}
if len(sys.argv) < 3:
usage()
return None
try:
local_filename = sys.argv[2]
t1 = time.time()
ret_dict = client.upload_by_file(local_filename)
t2 = time.time()
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
print('[+] time consume: %fs' % (t2 - t1))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upslavefile_func():
# upload slave file
# usage: python fdfs_test.py upslavefile {local_filename} {remote_fileid} {prefix_name}
if len(sys.argv) < 5:
usage()
return None
try:
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
prefix_name = sys.argv[4]
ret_dict = client.upload_slave_by_file(local_filename, remote_fileid, \
prefix_name)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upslavebuffer_func():
# upload slave by buffer
# usage: python fdfs_test.py upslavebuffer {local_filename} {remote_fileid} {prefix_name}
if len(sys.argv) < 5:
usage()
return None
try:
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
prefix_name = sys.argv[4]
with open(local_filename, 'rb') as f:
filebuffer = f.read()
ret_dict = client.upload_slave_by_buffer(local_filename, \
remote_fileid, prefix_name)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def del_func():
# delete file
# usage: python fdfs_test.py delete {remote_fileid}
if len(sys.argv) < 3:
usage()
return None
try:
remote_file_id = sys.argv[2]
ret_tuple = client.delete_file(remote_file_id)
print('[+] %s' % ret_tuple[0])
print('[+] remote_fileid: %s' % ret_tuple[1])
print('[+] Storage IP: %s' % ret_tuple[2])
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def downfile_func():
# Download to file
# usage: python fdfs_test.py downfile {local_filename} {remote_fileid}
if len(sys.argv) < 3:
usage()
return None
try:
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
ret_dict = client.download_to_file(local_filename, remote_fileid)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def list_group_func():
# List one group info
# usage: python fdfs_test.py listgroup {group_name}
if len(sys.argv) < 3:
usage()
return None
try:
group_name = sys.argv[2]
ret = client.list_one_group(group_name)
print(ret)
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def listall_func():
# List all group info
# usage: python fdfs_test.py listall
if len(sys.argv) < 2:
usage()
return None
try:
ret_dict = client.list_all_groups()
print('=' * 80)
print('Groups count:', ret_dict['Groups count'])
for li in ret_dict['Groups']:
print('-' * 80)
print(li)
print('-' * 80)
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def list_server_func():
# List all servers info of group
# usage: python fdfs_test.py listsrv {group_name} [storage_ip]
if len(sys.argv) < 3:
usage()
return None
try:
group_name = sys.argv[2]
if len(sys.argv) > 3:
storage_ip = sys.argv[3]
else:
storage_ip = None
ret_dict = client.list_servers(group_name, storage_ip)
print('=' * 80)
print('Group name: %s' % ret_dict['Group name'])
print('=' * 80)
i = 1
for serv in ret_dict['Servers']:
print('Storage server %d:' % i)
print('=' * 80)
print(serv)
i += 1
print('=' * 80)
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upbuffer_func():
# Upload by buffer
# usage: python fdfs_test.py upbuffer {local_filename} [remote_file_ext_name]
if len(sys.argv) < 3:
usage()
return None
local_filename = sys.argv[2]
if len(sys.argv) > 3:
ext_name = sys.argv[3]
else:
ext_name = None
# meta_buffer can be null.
meta_buffer = {
'ext_name': 'gif',
'width': '150px',
'height': '80px'
}
try:
with open(local_filename, 'rb') as f:
file_buffer = f.read()
ret_dict = client.upload_by_buffer(file_buffer, ext_name, meta_buffer)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def downbuffer_func():
# Download to buffer
# usage: python fdfs_test.py downbuffer {remote_file_id}
# e.g.: 'group1/M00/00/00/wKjzhU_rLNmjo2-1AAAamGDONEA5818.py'
if len(sys.argv) < 3:
usage()
return None
remote_fileid = sys.argv[2]
try:
ret_dict = client.download_to_buffer(remote_fileid)
print('Downloaded content:')
print(ret_dict['Content'])
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def get_meta_data_func():
# Get meta data of remote file
# usage python fdfs_test.py getmeta {remote_file_id}
if len(sys.argv) < 3:
usage()
return None
remote_fileid = sys.argv[2]
try:
ret_dict = client.get_meta_data(remote_fileid)
print(ret_dict)
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def set_meta_data_func():
# Set meta data of remote file
# usage python fdfs_test.py setmeta {remote_file_id}
if len(sys.argv) < 3:
usage()
return None
remote_fileid = sys.argv[2]
meta_dict = {
'ext_name': 'jgp',
'width': '160px',
'hight': '80px',
}
try:
ret_dict = client.set_meta_data(remote_fileid, meta_dict)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upappendfile_func():
# Upload an appender file by filename
# usage: python fdfs_test.py upappendfile {local_filename}
if len(sys.argv) < 3:
usage()
return None
local_filename = sys.argv[2]
try:
ret_dict = client.upload_appender_by_file(local_filename)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def upappendbuffer_func():
# Upload an appender file by buffer
# usage: python fdfs_test.py upappendbuffer {local_filename}
if len(sys.argv) < 3:
usage()
return None
local_filename = sys.argv[2]
try:
with open(local_filename, 'rb') as f:
file_buffer = f.read()
ret_dict = client.upload_appender_by_buffer(file_buffer)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def appendfile_func():
# Append a remote file
# usage: python fdfs_test.py appendfile {local_filename} {remote_file_id}
if len(sys.argv) < 4:
usage()
return None
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
try:
ret_dict = client.append_by_file(local_filename, remote_fileid)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def appendbuffer_func():
# Append a remote file by buffer
# usage: python fdfs_test.py appendbuffer {local_filename} {remote_file_id}
if len(sys.argv) < 4:
usage()
return None
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
try:
with open(local_filename, 'rb') as f:
filebuffer = f.read()
ret_dict = client.append_by_buffer(filebuffer, remote_fileid)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def truncate_func():
# Truncate file
# usage: python fdfs_test.py truncate {truncate_filesize} {remote_file_id}
if len(sys.argv) < 4:
usage()
return None
truncate_filesize = int(sys.argv[2])
remote_fileid = sys.argv[3]
try:
ret_dict = client.truncate_file(truncate_filesize, remote_fileid)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def modifyfile_func():
# Modify file by filename
# usage: python fdfs_test.py modifyfile {local_filename} {remote_fileid} [file_offset]
if len(sys.argv) < 4:
usage()
return None
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
if len(sys.argv) > 4:
file_offset = int(sys.argv[4])
else:
file_offset = 0
try:
ret_dict = client.modify_by_filename(local_filename, remote_fileid, file_offset)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
def modifybuffer_func():
# Modify file by buffer
# usage: python fdfs_test.py modifybuffer {local_filename} {remote_fileid} [file_offset]
if len(sys.argv) < 4:
usage()
return None
local_filename = sys.argv[2]
remote_fileid = sys.argv[3]
if len(sys.argv) > 4:
file_offset = int(sys.argv[4])
else:
file_offset = 0
try:
with open(local_filename, 'rb') as f:
filebuffer = f.read()
ret_dict = client.modify_by_buffer(filebuffer, remote_fileid, file_offset)
for key in ret_dict:
print('[+] %s : %s' % (key, ret_dict[key]))
except (ConnectionError, ResponseError, DataError) as e:
print(e)
result = {
'upfile': lambda: upfile_func(),
'upfileex': lambda: upfileex_func(),
'upbuffer': lambda: upbuffer_func(),
'delete': lambda: del_func(),
'downfile': lambda: downfile_func(),
'downbuffer': lambda: downbuffer_func(),
'listgroup': lambda: list_group_func(),
'listall': lambda: listall_func(),
'listsrv': lambda: list_server_func(),
'getmeta': lambda: get_meta_data_func(),
'setmeta': lambda: set_meta_data_func(),
'upslavefile': lambda: upslavefile_func(),
'upappendfile': lambda: upappendfile_func(),
'upappendbuffer': lambda: upappendbuffer_func(),
'appendfile': lambda: appendfile_func(),
'appendbuffer': lambda: appendbuffer_func(),
'truncate': lambda: truncate_func(),
'modifyfile': lambda: modifyfile_func(),
'modifybuffer': lambda: modifybuffer_func(),
'-h': lambda: usage(),
}[sys.argv[1].lower()]()
fdfs_client/storage_client.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: storage_client.py
import os
import stat
import errno
import struct
import socket
import datetime
import platform
from fdfs_client.fdfs_protol import *
from fdfs_client.connection import *
# from test_fdfs.sendfile import *
from fdfs_client.exceptions import (
FDFSError,
ConnectionError,
ResponseError,
InvaildResponse,
DataError
)
from fdfs_client.utils import *
__os_sep__ = "/" if platform.system() == 'Windows' else os.sep
def tcp_send_file(conn, filename, buffer_size=1024):
'''
Send file to server, and split into multiple pkgs while sending.
arguments:
@conn: connection
@filename: string
@buffer_size: int ,send buffer size
@Return int: file size if success else raise ConnectionError.
'''
file_size = 0
with open(filename, 'rb') as f:
while 1:
try:
send_buffer = f.read(buffer_size)
send_size = len(send_buffer)
if send_size == 0:
break
tcp_send_data(conn, send_buffer)
file_size += send_size
except ConnectionError as e:
raise ConnectionError('[-] Error while uploading file(%s).' % e.args)
except IOError as e:
raise DataError('[-] Error while reading local file(%s).' % e.args)
return file_size
def tcp_send_file_ex(conn, filename, buffer_size=4096):
'''
Send file to server. Using linux system call 'sendfile'.
arguments:
@conn: connection
@filename: string
@return long, sended size
'''
if 'linux' not in sys.platform.lower():
raise DataError('[-] Error: \'sendfile\' system call only available on linux.')
nbytes = 0
offset = 0
sock_fd = conn.get_sock().fileno()
with open(filename, 'rb') as f:
in_fd = f.fileno()
while 1:
try:
pass
# sent = sendfile(sock_fd, in_fd, offset, buffer_size)
# if 0 == sent:
# break
# nbytes += sent
# offset += sent
except OSError as e:
if e.errno == errno.EAGAIN:
continue
raise
return nbytes
def tcp_recv_file(conn, local_filename, file_size, buffer_size=1024):
'''
Receive file from server, fragmented it while receiving and write to disk.
arguments:
@conn: connection
@local_filename: string
@file_size: int, remote file size
@buffer_size: int, receive buffer size
@Return int: file size if success else raise ConnectionError.
'''
total_file_size = 0
flush_size = 0
remain_bytes = file_size
with open(local_filename, 'wb+') as f:
while remain_bytes > 0:
try:
if remain_bytes >= buffer_size:
file_buffer, recv_size = tcp_recv_response(conn, buffer_size, buffer_size)
else:
file_buffer, recv_size = tcp_recv_response(conn, remain_bytes, buffer_size)
f.write(file_buffer)
remain_bytes -= buffer_size
total_file_size += recv_size
flush_size += recv_size
if flush_size >= 4096:
f.flush()
flush_size = 0
except ConnectionError as e:
raise ConnectionError('[-] Error: while downloading file(%s).' % e.args)
except IOError as e:
raise DataError('[-] Error: while writting local file(%s).' % e.args)
return total_file_size
class Storage_client(object):
'''
The Class Storage_client for storage server.
Note: argument host_tuple of storage server ip address, that should be a single element.
'''
def __init__(self, *kwargs):
conn_kwargs = {
'name': 'Storage Pool',
'host_tuple': (kwargs[0],),
'port': kwargs[1],
'timeout': kwargs[2]
}
self.pool = ConnectionPool(**conn_kwargs)
return None
def __del__(self):
try:
self.pool.destroy()
self.pool = None
except:
pass
def update_pool(self, old_store_serv, new_store_serv, timeout=30):
'''
Update connection pool of storage client.
We need update connection pool of storage client, while storage server is changed.
but if server not changed, we do nothing.
'''
if old_store_serv.ip_addr == new_store_serv.ip_addr:
return None
self.pool.destroy()
conn_kwargs = {
'name': 'Storage_pool',
'host_tuple': (new_store_serv.ip_addr,),
'port': new_store_serv.port,
'timeout': timeout
}
self.pool = ConnectionPool(**conn_kwargs)
return True
def _storage_do_upload_file(self, tracker_client, store_serv, file_buffer, file_size=None, upload_type=None,
meta_dict=None, cmd=None, master_filename=None, prefix_name=None, file_ext_name=None):
'''
core of upload file.
arguments:
@tracker_client: Tracker_client, it is useful connect to tracker server
@store_serv: Storage_server, it is return from query tracker server
@file_buffer: string, file name or file buffer for send
@file_size: int
@upload_type: int, optional: FDFS_UPLOAD_BY_FILE, FDFS_UPLOAD_BY_FILENAME,
FDFS_UPLOAD_BY_BUFFER
@meta_dic: dictionary, store metadata in it
@cmd: int, reference fdfs protol
@master_filename: string, useful upload slave file
@prefix_name: string
@file_ext_name: string
@Return dictionary
{
'Group name' : group_name,
'Remote file_id' : remote_file_id,
'Status' : status,
'Local file name' : local_filename,
'Uploaded size' : upload_size,
'Storage IP' : storage_ip
}
'''
store_conn = self.pool.get_connection()
th = Tracker_header()
master_filename_len = len(master_filename) if master_filename else 0
prefix_name_len = len(prefix_name) if prefix_name else 0
upload_slave = len(store_serv.group_name) and master_filename_len
file_ext_name = str(file_ext_name) if file_ext_name else ''
# non_slave_fmt |-store_path_index(1)-file_size(8)-file_ext_name(6)-|
non_slave_fmt = '!B Q %ds' % FDFS_FILE_EXT_NAME_MAX_LEN
# slave_fmt |-master_len(8)-file_size(8)-prefix_name(16)-file_ext_name(6)
# -master_name(master_filename_len)-|
slave_fmt = '!Q Q %ds %ds %ds' % (FDFS_FILE_PREFIX_MAX_LEN, FDFS_FILE_EXT_NAME_MAX_LEN, master_filename_len)
th.pkg_len = struct.calcsize(slave_fmt) if upload_slave else struct.calcsize(non_slave_fmt)
th.pkg_len += file_size
th.cmd = cmd
th.send_header(store_conn)
if upload_slave:
send_buffer = struct.pack(
slave_fmt, master_filename_len, file_size, prefix_name, file_ext_name, master_filename)
else:
send_buffer = struct.pack(non_slave_fmt, store_serv.store_path_index, file_size, file_ext_name.encode())
try:
tcp_send_data(store_conn, send_buffer)
if upload_type == FDFS_UPLOAD_BY_FILENAME:
send_file_size = tcp_send_file(store_conn, file_buffer)
elif upload_type == FDFS_UPLOAD_BY_BUFFER:
tcp_send_data(store_conn, file_buffer)
elif upload_type == FDFS_UPLOAD_BY_FILE:
send_file_size = tcp_send_file_ex(store_conn, file_buffer)
th.recv_header(store_conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(store_conn, th.pkg_len)
if recv_size <= FDFS_GROUP_NAME_MAX_LEN:
errmsg = '[-] Error: Storage response length is not match, '
errmsg += 'expect: %d, actual: %d' % (th.pkg_len, recv_size)
raise ResponseError(errmsg)
# recv_fmt: |-group_name(16)-remote_file_name(recv_size - 16)-|
recv_fmt = '!%ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, th.pkg_len - FDFS_GROUP_NAME_MAX_LEN)
(group_name, remote_name) = struct.unpack(recv_fmt, recv_buffer)
remote_filename = remote_name.strip(b'\x00')
if meta_dict and len(meta_dict) > 0:
status = self.storage_set_metadata(tracker_client, store_serv, remote_filename, meta_dict)
if status != 0:
# rollback
self.storage_delete_file(tracker_client, store_serv, remote_filename)
raise DataError('[-] Error: %d, %s' % (status, os.strerror(status)))
except:
raise
finally:
self.pool.release(store_conn)
ret_dic = {
'Group name': group_name.strip(b'\x00'),
'Remote file_id': group_name.strip(b'\x00') + __os_sep__.encode() + remote_filename,
'Status': 'Upload successed.',
'Local file name': file_buffer if (upload_type == FDFS_UPLOAD_BY_FILENAME
or upload_type == FDFS_UPLOAD_BY_FILE
) else '',
'Uploaded size': appromix(send_file_size) if (upload_type == FDFS_UPLOAD_BY_FILENAME
or upload_type == FDFS_UPLOAD_BY_FILE
) else appromix(len(file_buffer)),
'Storage IP': store_serv.ip_addr
}
return ret_dic
def storage_upload_by_filename(self, tracker_client, store_serv, filename, meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILENAME,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_FILE, None, None, file_ext_name)
def storage_upload_by_file(self, tracker_client, store_serv, filename, meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILE,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_FILE, None, None, file_ext_name)
def storage_upload_by_buffer(self, tracker_client, store_serv, file_buffer, file_ext_name=None, meta_dict=None):
buffer_size = len(file_buffer)
return self._storage_do_upload_file(tracker_client, store_serv, file_buffer, buffer_size, FDFS_UPLOAD_BY_BUFFER,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_FILE, None, None, file_ext_name)
def storage_upload_slave_by_filename(self, tracker_client, store_serv, filename, prefix_name, remote_filename,
meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILENAME,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, remote_filename,
prefix_name, file_ext_name)
def storage_upload_slave_by_file(self, tracker_client, store_serv, filename, prefix_name, remote_filename,
meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILE,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, remote_filename,
prefix_name, file_ext_name)
def storage_upload_slave_by_buffer(self, tracker_client, store_serv, filebuffer, remote_filename, meta_dict,
file_ext_name):
file_size = len(filebuffer)
return self._storage_do_upload_file(tracker_client, store_serv, filebuffer, file_size, FDFS_UPLOAD_BY_BUFFER,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, None, remote_filename,
file_ext_name)
def storage_upload_appender_by_filename(self, tracker_client, store_serv, filename, meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILENAME,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, None, None,
file_ext_name)
def storage_upload_appender_by_file(self, tracker_client, store_serv, filename, meta_dict=None):
file_size = os.stat(filename).st_size
file_ext_name = get_file_ext_name(filename)
return self._storage_do_upload_file(tracker_client, store_serv, filename, file_size, FDFS_UPLOAD_BY_FILE,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, None, None,
file_ext_name)
def storage_upload_appender_by_buffer(self, tracker_client, store_serv, file_buffer, meta_dict=None,
file_ext_name=None):
file_size = len(file_buffer)
return self._storage_do_upload_file(tracker_client, store_serv, file_buffer, file_size, FDFS_UPLOAD_BY_BUFFER,
meta_dict, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, None, None,
file_ext_name)
def storage_delete_file(self, tracker_client, store_serv, remote_filename):
'''
Delete file from storage server.
'''
store_conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = STORAGE_PROTO_CMD_DELETE_FILE
file_name_len = len(remote_filename)
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN + file_name_len
try:
th.send_header(store_conn)
# del_fmt: |-group_name(16)-filename(len)-|
del_fmt = '!%ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, file_name_len)
send_buffer = struct.pack(del_fmt, store_serv.group_name, remote_filename)
tcp_send_data(store_conn, send_buffer)
th.recv_header(store_conn)
# if th.status == 2:
# raise DataError('[-] Error: remote file %s is not exist.'
# % (store_serv.group_name + __os_sep__.encode() + remote_filename))
if th.status != 0:
raise DataError('Error: %d, %s' % (th.status, os.strerror(th.status)))
# recv_buffer, recv_size = tcp_recv_response(store_conn, th.pkg_len)
except:
raise
finally:
self.pool.release(store_conn)
remote_filename = store_serv.group_name + __os_sep__.encode() + remote_filename
return ('Delete file successed.', remote_filename, store_serv.ip_addr)
def _storage_do_download_file(self, tracker_client, store_serv, file_buffer, offset, download_size,
download_type, remote_filename):
'''
Core of download file from storage server.
You can choice download type, optional FDFS_DOWNLOAD_TO_FILE or
FDFS_DOWNLOAD_TO_BUFFER. And you can choice file offset.
@Return dictionary
'Remote file name' : remote_filename,
'Content' : local_filename or buffer,
'Download size' : download_size,
'Storage IP' : storage_ip
'''
store_conn = self.pool.get_connection()
th = Tracker_header()
remote_filename_len = len(remote_filename)
th.pkg_len = FDFS_PROTO_PKG_LEN_SIZE * 2 + FDFS_GROUP_NAME_MAX_LEN + remote_filename_len
th.cmd = STORAGE_PROTO_CMD_DOWNLOAD_FILE
try:
th.send_header(store_conn)
# down_fmt: |-offset(8)-download_bytes(8)-group_name(16)-remote_filename(len)-|
down_fmt = '!Q Q %ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, remote_filename_len)
send_buffer = struct.pack(down_fmt, offset, download_size, store_serv.group_name, remote_filename)
tcp_send_data(store_conn, send_buffer)
th.recv_header(store_conn)
# if th.status == 2:
# raise DataError('[-] Error: remote file %s is not exist.' %
# (store_serv.group_name + __os_sep__.encode() + remote_filename))
if th.status != 0:
raise DataError('Error: %d %s' % (th.status, os.strerror(th.status)))
if download_type == FDFS_DOWNLOAD_TO_FILE:
total_recv_size = tcp_recv_file(store_conn, file_buffer, th.pkg_len)
elif download_type == FDFS_DOWNLOAD_TO_BUFFER:
recv_buffer, total_recv_size = tcp_recv_response(store_conn, th.pkg_len)
except:
raise
finally:
self.pool.release(store_conn)
ret_dic = {
'Remote file_id': store_serv.group_name + __os_sep__.encode() + remote_filename,
'Content': file_buffer if download_type == FDFS_DOWNLOAD_TO_FILE else recv_buffer,
'Download size': appromix(total_recv_size),
'Storage IP': store_serv.ip_addr
}
return ret_dic
def storage_download_to_file(self, tracker_client, store_serv, local_filename, file_offset, download_bytes,
remote_filename):
return self._storage_do_download_file(tracker_client, store_serv, local_filename, file_offset, download_bytes,
FDFS_DOWNLOAD_TO_FILE, remote_filename)
def storage_download_to_buffer(self, tracker_client, store_serv, file_buffer, file_offset, download_bytes,
remote_filename):
return self._storage_do_download_file(tracker_client, store_serv, file_buffer, file_offset, download_bytes,
FDFS_DOWNLOAD_TO_BUFFER, remote_filename)
def storage_set_metadata(self, tracker_client, store_serv, remote_filename, meta_dict,
op_flag=STORAGE_SET_METADATA_FLAG_OVERWRITE):
ret = 0
conn = self.pool.get_connection()
remote_filename_len = len(remote_filename)
meta_buffer = fdfs_pack_metadata(meta_dict)
meta_len = len(meta_buffer)
th = Tracker_header()
th.pkg_len = FDFS_PROTO_PKG_LEN_SIZE * 2 + 1 + FDFS_GROUP_NAME_MAX_LEN + remote_filename_len + meta_len
th.cmd = STORAGE_PROTO_CMD_SET_METADATA
try:
th.send_header(conn)
# meta_fmt: |-filename_len(8)-meta_len(8)-op_flag(1)-group_name(16)
# -filename(remote_filename_len)-meta(meta_len)|
meta_fmt = '!Q Q c %ds %ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, remote_filename_len, meta_len)
send_buffer = struct.pack(meta_fmt, remote_filename_len, meta_len, op_flag, store_serv.group_name,
remote_filename, meta_buffer)
tcp_send_data(conn, send_buffer)
th.recv_header(conn)
if th.status != 0:
ret = th.status
except:
raise
finally:
self.pool.release(conn)
return ret
def storage_get_metadata(self, tracker_client, store_serv, remote_file_name):
store_conn = self.pool.get_connection()
th = Tracker_header()
remote_filename_len = len(remote_file_name)
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN + remote_filename_len
th.cmd = STORAGE_PROTO_CMD_GET_METADATA
try:
th.send_header(store_conn)
# meta_fmt: |-group_name(16)-filename(remote_filename_len)-|
meta_fmt = '!%ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, remote_filename_len)
send_buffer = struct.pack(meta_fmt, store_serv.group_name, remote_file_name.encode())
tcp_send_data(store_conn, send_buffer)
th.recv_header(store_conn)
# if th.status == 2:
# raise DataError('[-] Error: Remote file %s has no meta data.'
# % (store_serv.group_name + __os_sep__.encode() + remote_file_name))
if th.status != 0:
raise DataError('[-] Error:%d, %s' % (th.status, os.strerror(th.status)))
if th.pkg_len == 0:
ret_dict = {}
meta_buffer, recv_size = tcp_recv_response(store_conn, th.pkg_len)
except:
raise
finally:
self.pool.release(store_conn)
ret_dict = fdfs_unpack_metadata(meta_buffer)
return ret_dict
def _storage_do_append_file(self, tracker_client, store_serv, file_buffer, file_size, upload_type,
appended_filename):
store_conn = self.pool.get_connection()
th = Tracker_header()
appended_filename_len = len(appended_filename)
th.pkg_len = FDFS_PROTO_PKG_LEN_SIZE * 2 + appended_filename_len + file_size
th.cmd = STORAGE_PROTO_CMD_APPEND_FILE
try:
th.send_header(store_conn)
# append_fmt: |-appended_filename_len(8)-file_size(8)-appended_filename(len)
# -filecontent(filesize)-|
append_fmt = '!Q Q %ds' % appended_filename_len
send_buffer = struct.pack(append_fmt, appended_filename_len, file_size, appended_filename)
tcp_send_data(store_conn, send_buffer)
if upload_type == FDFS_UPLOAD_BY_FILENAME:
tcp_send_file(store_conn, file_buffer)
elif upload_type == FDFS_UPLOAD_BY_BUFFER:
tcp_send_data(store_conn, file_buffer)
elif upload_type == FDFS_UPLOAD_BY_FILE:
tcp_send_file_ex(store_conn, file_buffer)
th.recv_header(store_conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
except:
raise
finally:
self.pool.release(store_conn)
ret_dict = {}
ret_dict['Status'] = 'Append file successed.'
ret_dict['Appender file name'] = store_serv.group_name + __os_sep__.encode() + appended_filename
ret_dict['Appended size'] = appromix(file_size)
ret_dict['Storage IP'] = store_serv.ip_addr
return ret_dict
def storage_append_by_filename(self, tracker_client, store_serv, local_filename, appended_filename):
file_size = os.stat(local_filename).st_size
return self._storage_do_append_file(tracker_client, store_serv, local_filename, file_size,
FDFS_UPLOAD_BY_FILENAME, appended_filename)
def storage_append_by_file(self, tracker_client, store_serv, local_filename, appended_filename):
file_size = os.stat(local_filename).st_size
return self._storage_do_append_file(tracker_client, store_serv, local_filename, file_size, FDFS_UPLOAD_BY_FILE,
appended_filename)
def storage_append_by_buffer(self, tracker_client, store_serv, file_buffer, appended_filename):
file_size = len(file_buffer)
return self._storage_do_append_file(tracker_client, store_serv, file_buffer, file_size, FDFS_UPLOAD_BY_BUFFER,
appended_filename)
def _storage_do_truncate_file(self, tracker_client, store_serv, truncated_filesize, appender_filename):
store_conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = STORAGE_PROTO_CMD_TRUNCATE_FILE
appender_filename_len = len(appender_filename)
th.pkg_len = FDFS_PROTO_PKG_LEN_SIZE * 2 + appender_filename_len
try:
th.send_header(store_conn)
# truncate_fmt:|-appender_filename_len(8)-truncate_filesize(8)
# -appender_filename(len)-|
truncate_fmt = '!Q Q %ds' % appender_filename_len
send_buffer = struct.pack(truncate_fmt, appender_filename_len, truncated_filesize, appender_filename)
tcp_send_data(store_conn, send_buffer)
th.recv_header(store_conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
except:
raise
finally:
self.pool.release(store_conn)
ret_dict = {}
ret_dict['Status'] = 'Truncate successed.'
ret_dict['Storage IP'] = store_serv.ip_addr
return ret_dict
def storage_truncate_file(self, tracker_client, store_serv, truncated_filesize, appender_filename):
return self._storage_do_truncate_file(tracker_client, store_serv, truncated_filesize, appender_filename)
def _storage_do_modify_file(self, tracker_client, store_serv, upload_type, filebuffer, offset, filesize,
appender_filename):
store_conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = STORAGE_PROTO_CMD_MODIFY_FILE
appender_filename_len = len(appender_filename)
th.pkg_len = FDFS_PROTO_PKG_LEN_SIZE * 3 + appender_filename_len + filesize
try:
th.send_header(store_conn)
# modify_fmt: |-filename_len(8)-offset(8)-filesize(8)-filename(len)-|
modify_fmt = '!Q Q Q %ds' % appender_filename_len
send_buffer = struct.pack(modify_fmt, appender_filename_len, offset, filesize, appender_filename)
tcp_send_data(store_conn, send_buffer)
if upload_type == FDFS_UPLOAD_BY_FILENAME:
upload_size = tcp_send_file(store_conn, filebuffer)
elif upload_type == FDFS_UPLOAD_BY_BUFFER:
tcp_send_data(store_conn, filebuffer)
elif upload_type == FDFS_UPLOAD_BY_FILE:
upload_size = tcp_send_file_ex(store_conn, filebuffer)
th.recv_header(store_conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
except:
raise
finally:
self.pool.release(store_conn)
ret_dict = {}
ret_dict['Status'] = 'Modify successed.'
ret_dict['Storage IP'] = store_serv.ip_addr
return ret_dict
def storage_modify_by_filename(self, tracker_client, store_serv, filename, offset, filesize, appender_filename):
return self._storage_do_modify_file(tracker_client, store_serv, FDFS_UPLOAD_BY_FILENAME, filename, offset,
filesize, appender_filename)
def storage_modify_by_file(self, tracker_client, store_serv, filename, offset, filesize, appender_filename):
return self._storage_do_modify_file(tracker_client, store_serv, FDFS_UPLOAD_BY_FILE, filename, offset, filesize,
appender_filename)
def storage_modify_by_buffer(self, tracker_client, store_serv, filebuffer, offset, filesize, appender_filename):
return self._storage_do_modify_file(tracker_client, store_serv, FDFS_UPLOAD_BY_BUFFER, filebuffer, offset,
filesize, appender_filename)
fdfs_client/tracker_client.py 0 → 100644
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# filename: tracker_client.py
import struct
import socket
from datetime import datetime
from fdfs_client.fdfs_protol import *
from fdfs_client.connection import *
from fdfs_client.exceptions import (
FDFSError,
ConnectionError,
ResponseError,
InvaildResponse,
DataError
)
from fdfs_client.utils import *
def parse_storage_status(status_code):
try:
ret = {
FDFS_STORAGE_STATUS_INIT: lambda: 'INIT',
FDFS_STORAGE_STATUS_WAIT_SYNC: lambda: 'WAIT_SYNC',
FDFS_STORAGE_STATUS_SYNCING: lambda: 'SYNCING',
FDFS_STORAGE_STATUS_IP_CHANGED: lambda: 'IP_CHANGED',
FDFS_STORAGE_STATUS_DELETED: lambda: 'DELETED',
FDFS_STORAGE_STATUS_OFFLINE: lambda: 'OFFLINE',
FDFS_STORAGE_STATUS_ONLINE: lambda: 'ONLINE',
FDFS_STORAGE_STATUS_ACTIVE: lambda: 'ACTIVE',
FDFS_STORAGE_STATUS_RECOVERY: lambda: 'RECOVERY'
}[status_code]()
except KeyError:
ret = 'UNKNOW'
return ret
class Storage_info(object):
def __init__(self):
self.status = 0
self.id = ''
self.ip_addr = ''
self.domain_name = ''
self.src_id = ''
self.version = ''
self.join_time = datetime.fromtimestamp(0).isoformat()
self.up_time = datetime.fromtimestamp(0).isoformat()
self.totalMB = ''
self.freeMB = ''
self.upload_prio = 0
self.store_path_count = 0
self.subdir_count_per_path = 0
self.curr_write_path = 0
self.storage_port = 23000
self.storage_http_port = 80
self.alloc_count = 0
self.current_count = 0
self.max_count = 0
self.total_upload_count = 0
self.success_upload_count = 0
self.total_append_count = 0
self.success_append_count = 0
self.total_modify_count = 0
self.success_modify_count = 0
self.total_truncate_count = 0
self.success_truncate_count = 0
self.total_setmeta_count = 0
self.success_setmeta_count = 0
self.total_del_count = 0
self.success_del_count = 0
self.total_download_count = 0
self.success_download_count = 0
self.total_getmeta_count = 0
self.success_getmeta_count = 0
self.total_create_link_count = 0
self.success_create_link_count = 0
self.total_del_link_count = 0
self.success_del_link_count = 0
self.total_upload_bytes = 0
self.success_upload_bytes = 0
self.total_append_bytes = 0
self.success_append_bytes = 0
self.total_modify_bytes = 0
self.success_modify_bytes = 0
self.total_download_bytes = 0
self.success_download_bytes = 0
self.total_sync_in_bytes = 0
self.success_sync_in_bytes = 0
self.total_sync_out_bytes = 0
self.success_sync_out_bytes = 0
self.total_file_open_count = 0
self.success_file_open_count = 0
self.total_file_read_count = 0
self.success_file_read_count = 0
self.total_file_write_count = 0
self.success_file_write_count = 0
self.last_source_sync = datetime.fromtimestamp(0).isoformat()
self.last_sync_update = datetime.fromtimestamp(0).isoformat()
self.last_synced_time = datetime.fromtimestamp(0).isoformat()
self.last_heartbeat_time = datetime.fromtimestamp(0).isoformat()
self.if_trunk_server = ''
# fmt = |-status(1)-ipaddr(16)-domain(128)-srcipaddr(16)-ver(6)-52*8-|
self.fmt = '!B 16s 16s 128s 16s 6s 10Q 4s4s4s 42Q?'
def set_info(self, bytes_stream):
(self.status, self.id, ip_addr, domain_name, self.src_id, version, join_time, up_time, totalMB, freeMB,
self.upload_prio, self.store_path_count, self.subdir_count_per_path, self.curr_write_path, self.storage_port,
self.storage_http_port, self.alloc_count, self.current_count, self.max_count, self.total_upload_count,
self.success_upload_count, self.total_append_count, self.success_append_count, self.total_modify_count,
self.success_modify_count, self.total_truncate_count, self.success_truncate_count, self.total_setmeta_count,
self.success_setmeta_count, self.total_del_count, self.success_del_count, self.total_download_count,
self.success_download_count, self.total_getmeta_count, self.success_getmeta_count,
self.total_create_link_count, self.success_create_link_count, self.total_del_link_count,
self.success_del_link_count, self.total_upload_bytes, self.success_upload_bytes, self.total_append_bytes,
self.total_append_bytes, self.total_modify_bytes, self.success_modify_bytes, self.total_download_bytes,
self.success_download_bytes, self.total_sync_in_bytes, self.success_sync_in_bytes, self.total_sync_out_bytes,
self.success_sync_out_bytes, self.total_file_open_count, self.success_file_open_count,
self.total_file_read_count, self.success_file_read_count, self.total_file_write_count,
self.success_file_write_count, last_source_sync, last_sync_update, last_synced_time, last_heartbeat_time,
self.if_trunk_server,) = struct.unpack(self.fmt, bytes_stream)
try:
self.ip_addr = ip_addr.strip(b'\x00')
self.domain_name = domain_name.strip(b'\x00')
self.version = version.strip(b'\x00')
self.totalMB = appromix(totalMB, FDFS_SPACE_SIZE_BASE_INDEX)
self.freeMB = appromix(freeMB, FDFS_SPACE_SIZE_BASE_INDEX)
except ValueError as e:
raise ResponseError('[-] Error: disk space overrun, can not represented it.')
self.join_time = datetime.fromtimestamp(join_time).isoformat()
self.up_time = datetime.fromtimestamp(up_time).isoformat()
self.last_source_sync = datetime.fromtimestamp(last_source_sync).isoformat()
self.last_sync_update = datetime.fromtimestamp(last_sync_update).isoformat()
self.last_synced_time = datetime.fromtimestamp(last_synced_time).isoformat()
self.last_heartbeat_time = datetime.fromtimestamp(last_heartbeat_time).isoformat()
return True
def __str__(self):
'''Transform to readable string.'''
s = 'Storage information:\n'
s += '\tip_addr = %s (%s)\n' % (self.ip_addr, parse_storage_status(self.status))
s += '\thttp domain = %s\n' % self.domain_name
s += '\tversion = %s\n' % self.version
s += '\tjoin time = %s\n' % self.join_time
s += '\tup time = %s\n' % self.up_time
s += '\ttotal storage = %s\n' % self.totalMB
s += '\tfree storage = %s\n' % self.freeMB
s += '\tupload priority = %d\n' % self.upload_prio
s += '\tstore path count = %d\n' % self.store_path_count
s += '\tsubdir count per path = %d\n' % self.subdir_count_per_path
s += '\tstorage port = %d\n' % self.storage_port
s += '\tstorage HTTP port = %d\n' % self.storage_http_port
s += '\tcurrent write path = %d\n' % self.curr_write_path
s += '\tsource ip_addr = %s\n' % self.ip_addr
s += '\tif_trunk_server = %d\n' % self.if_trunk_server
s += '\ttotal upload count = %ld\n' % self.total_upload_count
s += '\tsuccess upload count = %ld\n' % self.success_upload_count
s += '\ttotal download count = %ld\n' % self.total_download_count
s += '\tsuccess download count = %ld\n' % self.success_download_count
s += '\ttotal append count = %ld\n' % self.total_append_count
s += '\tsuccess append count = %ld\n' % self.success_append_count
s += '\ttotal modify count = %ld\n' % self.total_modify_count
s += '\tsuccess modify count = %ld\n' % self.success_modify_count
s += '\ttotal truncate count = %ld\n' % self.total_truncate_count
s += '\tsuccess truncate count = %ld\n' % self.success_truncate_count
s += '\ttotal delete count = %ld\n' % self.total_del_count
s += '\tsuccess delete count = %ld\n' % self.success_del_count
s += '\ttotal set_meta count = %ld\n' % self.total_setmeta_count
s += '\tsuccess set_meta count = %ld\n' % self.success_setmeta_count
s += '\ttotal get_meta count = %ld\n' % self.total_getmeta_count
s += '\tsuccess get_meta count = %ld\n' % self.success_getmeta_count
s += '\ttotal create link count = %ld\n' % self.total_create_link_count
s += '\tsuccess create link count = %ld\n' % self.success_create_link_count
s += '\ttotal delete link count = %ld\n' % self.total_del_link_count
s += '\tsuccess delete link count = %ld\n' % self.success_del_link_count
s += '\ttotal upload bytes = %ld\n' % self.total_upload_bytes
s += '\tsuccess upload bytes = %ld\n' % self.success_upload_bytes
s += '\ttotal download bytes = %ld\n' % self.total_download_bytes
s += '\tsuccess download bytes = %ld\n' % self.success_download_bytes
s += '\ttotal append bytes = %ld\n' % self.total_append_bytes
s += '\tsuccess append bytes = %ld\n' % self.success_append_bytes
s += '\ttotal modify bytes = %ld\n' % self.total_modify_bytes
s += '\tsuccess modify bytes = %ld\n' % self.success_modify_bytes
s += '\ttotal sync_in bytes = %ld\n' % self.total_sync_in_bytes
s += '\tsuccess sync_in bytes = %ld\n' % self.success_sync_in_bytes
s += '\ttotal sync_out bytes = %ld\n' % self.total_sync_out_bytes
s += '\tsuccess sync_out bytes = %ld\n' % self.success_sync_out_bytes
s += '\ttotal file open count = %ld\n' % self.total_file_open_count
s += '\tsuccess file open count = %ld\n' % self.success_file_open_count
s += '\ttotal file read count = %ld\n' % self.total_file_read_count
s += '\tsuccess file read count = %ld\n' % self.success_file_read_count
s += '\ttotal file write count = %ld\n' % self.total_file_write_count
s += '\tsucess file write count = %ld\n' % self.success_file_write_count
s += '\tlast heartbeat time = %s\n' % self.last_heartbeat_time
s += '\tlast source update = %s\n' % self.last_source_sync
s += '\tlast sync update = %s\n' % self.last_sync_update
s += '\tlast synced time = %s\n' % self.last_synced_time
return s
def get_fmt_size(self):
return struct.calcsize(self.fmt)
class Group_info(object):
def __init__(self):
self.group_name = ''
self.totalMB = ''
self.freeMB = ''
self.trunk_freeMB = ''
self.count = 0
self.storage_port = 0
self.store_http_port = 0
self.active_count = 0
self.curr_write_server = 0
self.store_path_count = 0
self.subdir_count_per_path = 0
self.curr_trunk_file_id = 0
self.fmt = '!%ds 11Q' % (FDFS_GROUP_NAME_MAX_LEN + 1)
return None
def __str__(self):
s = 'Group information:\n'
s += '\tgroup name = %s\n' % self.group_name
s += '\ttotal disk space = %s\n' % self.totalMB
s += '\tdisk free space = %s\n' % self.freeMB
s += '\ttrunk free space = %s\n' % self.trunk_freeMB
s += '\tstorage server count = %d\n' % self.count
s += '\tstorage port = %d\n' % self.storage_port
s += '\tstorage HTTP port = %d\n' % self.store_http_port
s += '\tactive server count = %d\n' % self.active_count
s += '\tcurrent write server index = %d\n' % self.curr_write_server
s += '\tstore path count = %d\n' % self.store_path_count
s += '\tsubdir count per path = %d\n' % self.subdir_count_per_path
s += '\tcurrent trunk file id = %d\n' % self.curr_trunk_file_id
return s
def set_info(self, bytes_stream):
(group_name, totalMB, freeMB, trunk_freeMB, self.count, self.storage_port, self.store_http_port,
self.active_count, self.curr_write_server, self.store_path_count, self.subdir_count_per_path,
self.curr_trunk_file_id) = struct.unpack(self.fmt, bytes_stream)
try:
self.group_name = group_name.strip(b'\x00')
self.freeMB = appromix(freeMB, FDFS_SPACE_SIZE_BASE_INDEX)
self.totalMB = appromix(totalMB, FDFS_SPACE_SIZE_BASE_INDEX)
self.trunk_freeMB = appromix(trunk_freeMB, FDFS_SPACE_SIZE_BASE_INDEX)
except ValueError:
raise DataError('[-] Error disk space overrun, can not represented it.')
def get_fmt_size(self):
return struct.calcsize(self.fmt)
class Tracker_client(object):
'''Class Tracker client.'''
def __init__(self, pool):
self.pool = pool
def tracker_list_servers(self, group_name, storage_ip=None):
'''
List servers in a storage group
'''
conn = self.pool.get_connection()
th = Tracker_header()
ip_len = len(storage_ip) if storage_ip else 0
if ip_len >= IP_ADDRESS_SIZE:
ip_len = IP_ADDRESS_SIZE - 1
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN + ip_len
th.cmd = TRACKER_PROTO_CMD_SERVER_LIST_STORAGE
group_fmt = '!%ds' % FDFS_GROUP_NAME_MAX_LEN
store_ip_addr = storage_ip or ''
storage_ip_fmt = '!%ds' % ip_len
try:
th.send_header(conn)
send_buffer = struct.pack(group_fmt, group_name) + struct.pack(storage_ip_fmt, store_ip_addr)
tcp_send_data(conn, send_buffer)
th.recv_header(conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
si = Storage_info()
si_fmt_size = si.get_fmt_size()
recv_size = len(recv_buffer)
if recv_size % si_fmt_size != 0:
errinfo = '[-] Error: response size not match, expect: %d, actual: %d' % (th.pkg_len, recv_size)
raise ResponseError(errinfo)
except ConnectionError:
raise
finally:
self.pool.release(conn)
num_storage = recv_size / si_fmt_size
si_list = []
i = 0
while num_storage:
si.set_info(recv_buffer[(i * si_fmt_size): ((i + 1) * si_fmt_size)])
si_list.append(si)
si = Storage_info()
num_storage -= 1
i += 1
ret_dict = {}
ret_dict['Group name'] = group_name
ret_dict['Servers'] = si_list
return ret_dict
def tracker_list_one_group(self, group_name):
conn = self.pool.get_connection()
th = Tracker_header()
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN
th.cmd = TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP
# group_fmt: |-group_name(16)-|
group_fmt = '!%ds' % FDFS_GROUP_NAME_MAX_LEN
try:
th.send_header(conn)
send_buffer = struct.pack(group_fmt, group_name)
tcp_send_data(conn, send_buffer)
th.recv_header(conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
group_info = Group_info()
group_info.set_info(recv_buffer)
except ConnectionError:
raise
finally:
self.pool.release(conn)
return group_info
def tracker_list_all_groups(self):
conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS
try:
th.send_header(conn)
th.recv_header(conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
except:
raise
finally:
self.pool.release(conn)
gi = Group_info()
gi_fmt_size = gi.get_fmt_size()
if recv_size % gi_fmt_size != 0:
errmsg = '[-] Error: Response size is mismatch, except: %d, actul: %d' % (th.pkg_len, recv_size)
raise ResponseError(errmsg)
num_groups = recv_size / gi_fmt_size
ret_dict = {}
ret_dict['Groups count'] = num_groups
gi_list = []
i = 0
while num_groups:
gi.set_info(recv_buffer[i * gi_fmt_size: (i + 1) * gi_fmt_size])
gi_list.append(gi)
gi = Group_info()
i += 1
num_groups -= 1
ret_dict['Groups'] = gi_list
return ret_dict
def tracker_query_storage_stor_without_group(self):
'''Query storage server for upload, without group name.
Return: Storage_server object'''
conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE
try:
th.send_header(conn)
th.recv_header(conn)
if th.status != 0:
raise DataError('[-] Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
if recv_size != TRACKER_QUERY_STORAGE_STORE_BODY_LEN:
errmsg = '[-] Error: Tracker response length is invaild, '
errmsg += 'expect: %d, actual: %d' % (TRACKER_QUERY_STORAGE_STORE_BODY_LEN, recv_size)
raise ResponseError(errmsg)
except ConnectionError:
raise
finally:
self.pool.release(conn)
# recv_fmt |-group_name(16)-ipaddr(16-1)-port(8)-store_path_index(1)|
recv_fmt = '!%ds %ds Q B' % (FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE - 1)
store_serv = Storage_server()
(group_name, ip_addr, store_serv.port, store_serv.store_path_index) = struct.unpack(recv_fmt, recv_buffer)
store_serv.group_name = group_name.strip(b'\x00')
store_serv.ip_addr = ip_addr.strip(b'\x00')
return store_serv
def tracker_query_storage_stor_with_group(self, group_name):
'''Query storage server for upload, based group name.
arguments:
@group_name: string
@Return Storage_server object
'''
conn = self.pool.get_connection()
th = Tracker_header()
th.cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN
th.send_header(conn)
group_fmt = '!%ds' % FDFS_GROUP_NAME_MAX_LEN
send_buffer = struct.pack(group_fmt, group_name)
try:
tcp_send_data(conn, send_buffer)
th.recv_header(conn)
if th.status != 0:
raise DataError('Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
if recv_size != TRACKER_QUERY_STORAGE_STORE_BODY_LEN:
errmsg = '[-] Error: Tracker response length is invaild, '
errmsg += 'expect: %d, actual: %d' % (TRACKER_QUERY_STORAGE_STORE_BODY_LEN, recv_size)
raise ResponseError(errmsg)
except ConnectionError:
raise
finally:
self.pool.release(conn)
# recv_fmt: |-group_name(16)-ipaddr(16-1)-port(8)-store_path_index(1)-|
recv_fmt = '!%ds %ds Q B' % (FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE - 1)
store_serv = Storage_server()
(group, ip_addr, store_serv.port, store_serv.store_path_index) = struct.unpack(recv_fmt, recv_buffer)
store_serv.group_name = group.strip(b'\x00')
store_serv.ip_addr = ip_addr.strip(b'\x00')
return store_serv
def _tracker_do_query_storage(self, group_name, filename, cmd):
'''
core of query storage, based group name and filename.
It is useful download, delete and set meta.
arguments:
@group_name: string
@filename: string. remote file_id
@Return: Storage_server object
'''
conn = self.pool.get_connection()
th = Tracker_header()
file_name_len = len(filename)
th.pkg_len = FDFS_GROUP_NAME_MAX_LEN + file_name_len
th.cmd = cmd
th.send_header(conn)
# query_fmt: |-group_name(16)-filename(file_name_len)-|
query_fmt = '!%ds %ds' % (FDFS_GROUP_NAME_MAX_LEN, file_name_len)
send_buffer = struct.pack(query_fmt, group_name, filename)
try:
tcp_send_data(conn, send_buffer)
th.recv_header(conn)
if th.status != 0:
raise DataError('Error: %d, %s' % (th.status, os.strerror(th.status)))
recv_buffer, recv_size = tcp_recv_response(conn, th.pkg_len)
if recv_size != TRACKER_QUERY_STORAGE_FETCH_BODY_LEN:
errmsg = '[-] Error: Tracker response length is invaild, '
errmsg += 'expect: %d, actual: %d' % (th.pkg_len, recv_size)
raise ResponseError(errmsg)
except ConnectionError:
raise
finally:
self.pool.release(conn)
# recv_fmt: |-group_name(16)-ip_addr(16)-port(8)-|
recv_fmt = '!%ds %ds Q' % (FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE - 1)
store_serv = Storage_server()
(group_name, ipaddr, store_serv.port) = struct.unpack(recv_fmt, recv_buffer)
store_serv.group_name = group_name.strip(b'\x00')
store_serv.ip_addr = ipaddr.strip(b'\x00')
return store_serv
def tracker_query_storage_update(self, group_name, filename):
'''
Query storage server to update(delete and set_meta).
'''
return self._tracker_do_query_storage(group_name, filename, TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE)
def tracker_query_storage_fetch(self, group_name, filename):
'''
Query storage server to download.
'''
return self._tracker_do_query_storage(group_name, filename, TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE)
fdfs_client/utils.py 0 → 100644
#!/usr/bin/env python
# -*- coding = utf-8 -*-
# filename: utils.py
import io
import os
import sys
import stat
import platform
import configparser
SUFFIX = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
__os_sep__ = "/" if platform.system() == 'Windows' else os.sep
def appromix(size, base=0):
'''Conver bytes stream size to human-readable format.
Keyword arguments:
size: int, bytes stream size
base: int, suffix index
Return: string
'''
multiples = 1024
if size < 0:
raise ValueError('[-] Error: number must be non-negative.')
if size < multiples:
return '{0:d}{1}'.format(size, SUFFIX[base])
for suffix in SUFFIX[base:]:
if size < multiples:
return '{0:.2f}{1}'.format(size, suffix)
size = size / float(multiples)
raise ValueError('[-] Error: number too big.')
def get_file_ext_name(filename, double_ext=True):
li = filename.split(os.extsep)
if len(li) <= 1:
return ''
else:
if li[-1].find(__os_sep__) != -1:
return ''
if double_ext:
if len(li) > 2:
if li[-2].find(__os_sep__) == -1:
return '%s.%s' % (li[-2], li[-1])
return li[-1]
class Fdfs_ConfigParser(configparser.RawConfigParser):
"""
Extends ConfigParser to allow files without sections.
This is done by wrapping read files and prepending them with a placeholder
section, which defaults to '__config__'
"""
def __init__(self, default_section=None, *args, **kwargs):
configparser.RawConfigParser.__init__(self, *args, **kwargs)
self._default_section = None
self.set_default_section(default_section or '__config__')
def get_default_section(self):
return self._default_section
def set_default_section(self, section):
self.add_section(section)
# move all values from the previous default section to the new one
try:
default_section_items = self.items(self._default_section)
self.remove_section(self._default_section)
except configparser.NoSectionError:
pass
else:
for (key, value) in default_section_items:
self.set(section, key, value)
self._default_section = section
def read(self, filenames):
if isinstance(filenames, str):
filenames = [filenames]
read_ok = []
for filename in filenames:
try:
with open(filename) as fp:
self.readfp(fp)
except IOError:
continue
else:
read_ok.append(filename)
return read_ok
def readfp(self, fp, *args, **kwargs):
stream = io.StringIO()
try:
stream.name = fp.name
except AttributeError:
pass
stream.write('[' + self._default_section + ']\n')
stream.write(fp.read())
stream.seek(0, 0)
return self._read(stream, stream.name)
def write(self, fp):
# Write the items from the default section manually and then remove them
# from the data. They'll be re-added later.
try:
default_section_items = self.items(self._default_section)
self.remove_section(self._default_section)
for (key, value) in default_section_items:
fp.write("{0} = {1}\n".format(key, value))
fp.write("\n")
except configparser.NoSectionError:
pass
configparser.RawConfigParser.write(self, fp)
self.add_section(self._default_section)
for (key, value) in default_section_items:
self.set(self._default_section, key, value)
def _read(self, fp, fpname):
"""Parse a sectioned setup file.
The sections in setup file contains a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored.
"""
cursect = None # None, or a dictionary
optname = None
lineno = 0
e = None # None, or an exception
while True:
line = fp.readline()
if not line:
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
continue
# continuation line?
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
cursect[optname] = "%s\n%s" % (cursect[optname], value)
# a section header or option header?
else:
# is it a section header?
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == DEFAULTSECT:
cursect = self._defaults
else:
cursect = self._dict()
cursect['__name__'] = sectname
self._sections[sectname] = cursect
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise MissingSectionHeaderError(fpname, lineno, line)
# an option line?
else:
mo = self.OPTCRE.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if vi in ('=', ':') and ';' in optval:
# ';' is a comment delimiter only if it follows
# a spacing character
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
# allow empty values
if optval == '""':
optval = ''
optname = self.optionxform(optname.rstrip())
if optname in cursect:
if not isinstance(cursect[optname], list):
cursect[optname] = [cursect[optname]]
cursect[optname].append(optval)
else:
cursect[optname] = optval
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
if not e:
e = ParsingError(fpname)
e.append(lineno, repr(line))
# if any parsing errors occurred, raise an exception
if e:
raise e
def split_remote_fileid(remote_file_id):
'''
Splite remote_file_id to (group_name, remote_file_name)
arguments:
@remote_file_id: string
@return tuple, (group_name, remote_file_name)
'''
index = remote_file_id.find(b'/')
if -1 == index:
return None
return (remote_file_id[0:index], remote_file_id[(index + 1):])
def fdfs_check_file(filename):
ret = True
errmsg = ''
if not os.path.isfile(filename):
ret = False
errmsg = '[-] Error: %s is not a file.' % filename
elif not stat.S_ISREG(os.stat(filename).st_mode):
ret = False
errmsg = '[-] Error: %s is not a regular file.' % filename
return (ret, errmsg)
if __name__ == '__main__':
print(get_file_ext_name('/bc.tar.gz'))
generate/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/12/2 17:21
generate/gen_user_report_auto_generated.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : gen_user_report_auto_generated
# @Author : LiuYan
# @Time : 2021/12/7 14:50
import re, os
import copy
import json
import chevron
import datetime
from docx import Document, shared
from docx.oxml.ns import qn
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
from generate.regular_extract import RegularExtract
from generate.pic_echarts import pic_echarts_pie, pic_echarts_bar, pic_echarts_line, pic_echarts_bar_line
# from generate.pic_plt import pic_plt_pie
from utils.log import logger
from copy_table import get_choose_table, generate_report
from copy_content import copy_content_main
class GeneralUserReportAutoGenerated(object):
"""
1. 扫描数据源
2. 拉取数据
3. 数据填充
"""
def __init__(self, project_name: str, input_template_path: str, output_report_path: str,
start_time: datetime, end_time: datetime, parameter='',
is_handle_success='ynHandleSuccess', result_data='resultData'):
super(GeneralUserReportAutoGenerated, self).__init__()
self._project_name = project_name
self._start_time = start_time
self._end_time = end_time
self._parameters = {
'start_time': self._start_time,
'end_time': self._end_time
}
self._parameter = parameter
self._is_handle_success = is_handle_success
self._result_data = result_data
self._input_template_path = input_template_path
self._output_report_path = output_report_path
self._document = Document(self._input_template_path)
self._paragraphs = self._document.paragraphs
self._tables = self._document.tables
self._regular_extract = RegularExtract()
self._table_dict_template_label = dict() # 模板标签索引 [表格]
# self._dict_template_label_del = dict() # 模板标签索引 [del]
self._dict_template_label = dict() # 模板标签索引——普通类型的标签索引
self._dict_padding_data = dict() # 填充数据
self._dict_dataset = dict() # 数据集信息
self._dict_list_render = dict() # 列表渲染
self._set_dataset_name = set() # 用于存放已填充数据集名称和未配置的数据集名称
self._list_multi_para_render = list() # 用于存放多段渲染
self._build()
def _build(self) -> None:
"""
创建 / 更新 段落 + 表格 [列表]
:return:
"""
self._paragraphs = self._document.paragraphs
self._tables = self._document.tables
def _build_parameter(self, parameter: str, json_config: dict) -> str:
"""
组织请求参数
:param parameter: 参数
:param json_config: 参数配置
:return: parameter [str]
"""
self._parameters = {
'start_time': self._start_time,
'end_time': self._end_time
}
for json_config_key in json_config:
list_para = json_config_key.split('.')
if len(list_para) == 2:
self._parameters[list_para[1]] = json_config[json_config_key]
parameter = chevron.render(parameter, self._parameters)
return parameter
def process(self, data_result: dict) -> None:
"""
1. 扫描文本数据集 + 图片生成引擎(Json + Data + Generate)
2. 拉取文本数据 + 删除未注册数据集
3. 填充对象型数据集 + 列表型数据集 [含多段渲染]
4. 表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)
5. 报告生成完毕 保存
[1. 图片 2. 文本(列表型) 3. 文本 4. 表格]
:return: None
"""
# 1. 扫描文本数据集 + 图片生成引擎(Json + Data + Generate)
logger.info('开始扫描文本数据集 + 启动图片生成引擎(Json + Data + Generate)')
self._scanning_data(set_dataset_name=self._set_dataset_name, data_result=data_result)
# 2. 获取数据信息
logger.info('获取数据信息')
self._getting_data(data_result)
# print("处理第一部分普通的info.year后的结果: {}".format(self._dict_template_label))
# todo：当前数据内容
logger.info("当前待填充数据内容{}".format(self._dict_padding_data))
# todo： 删除数据为空的内容
dict_finance = self._dict_padding_data["finance"]
dict_info = self._dict_padding_data["info"]
for key in list(dict_finance.keys()):
if not dict_finance.get(key):
del dict_finance[key]
for key in list(dict_info.keys()):
if not dict_info.get(key):
del dict_info[key]
self._dict_padding_data.update({"info": dict_info, "finance": dict_finance})
# 3. 文本数据填充
# todo： 基于self._getting_data 函数得到 self._dict_padding_data
for dataset_name in self._dict_template_label:
# print(dataset_name)
self._padding_dict(dataset_name=dataset_name)
# # 列表型填充
# self._process_text_list_render()
# 多段渲染 [统一渲染]
# self._process_multi_para_render()
# 4. 表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)
logger.info('启动表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)')
self._process_table()
# 5. 表格数据填充
# # todo： 基于self._getting_data 函数得到 self._dict_padding_data
# for dataset_name in self._table_dict_template_label:
# print(dataset_name)
# self._padding_table(dataset_name=dataset_name)
# 6. 报告生成完毕 保存
logger.info('报告生成完毕! 保存......')
self._save()
# todo: 先找到待填充数据的位置（相应的段落索引，索引下标默认从0开始计数）
def _scanning_data(self, set_dataset_name: set, data_result: dict) -> None:
"""
扫描数据
:param set_dataset_name: index已经重定位好[已填充]的数据 仅有data_type=1会插入其中
补充: 若数据集未注册，则也会在拉取数据集时存入，以后便不再扫描该数据源
:return:
"""
for index, para in enumerate(self._paragraphs):
set_result = self._regular_extract.match_pattern(para_text=para.text, patterns=['(?<={{).*?(?=}})'])
for result in set_result:
# 原始步骤1 扫描列表渲染 [start_index: end_index]
# 原始步骤2 特殊文本型渲染: {{ ^dataset.title }} 若title为None或''则删除该行(不进行渲染)。 数据对象为第一部分的info数据集 待处理样式 ^info.zlRisk
# 图片型渲染
try:
json_result = json.loads(result)
logger.info('Picture Json: {}'.format(json_result))
# todo: 基于传过来的数据生成相应的数据图
self._process_picture(index=index, json_pic_config=json_result, data_result=data_result)
continue
except Exception as e:
pass
# 文本渲染(对象型)
# todo： 处理第一部分info.year普通标签
list_template_label = result.split('.')
if len(list_template_label) == 2:
# dataset 为数据对象名称 初版有info 和 finance
template_dataset = list_template_label[0]
# label 为相应的属性名称
template_label = list_template_label[1].strip()
if template_dataset not in set_dataset_name:
if template_dataset in self._dict_template_label:
if template_label in self._dict_template_label[template_dataset]:
self._dict_template_label[template_dataset][template_label].add(index)
else:
self._dict_template_label[template_dataset][template_label] = {index}
else:
self._dict_template_label[template_dataset] = {template_label: {index}}
continue
def _getting_data(self, data_result: dict) -> None:
"""
本项目无需拉取数据，只需要存储数据结果
:return:
"""
list_dataset_name = list(
set(list(self._dict_template_label.keys()))
)
# 遍历扫描到的所有文本型数据集 [拉取 / 删除]
for dataset_name in list_dataset_name:
result = data_result[dataset_name]
# 4. 存储数据集的数据信息
self._dict_padding_data[dataset_name] = result
def _process_text_list_render(self) -> None:
"""
文本(列表型): 1. 渲染(cv)/删除(Delete) 2. 更新(update index) 3. 填充(padding data)
:return:
"""
list_render_dataset_name = list(self._dict_list_render.keys())
for dataset_name in list_render_dataset_name:
'''
在此判断:
1. 该数据集在已注册字典中
2. 该数据集类型是列表型
3. 该数据集数据不为None或[]
'''
if dataset_name in self._dict_dataset and self._dict_dataset[dataset_name]['dataset_type'] == 1:
'''
在此判断:
1. 该数据集数据不为None或[] -> 正常进行列表渲染 cv
2. 如果为None或[] -> 列表渲染部分删除[开头 -> 结尾] 重定位index
'''
# 1) 渲染(cv)/删除(Delete)
if self._dict_padding_data[dataset_name]:
render_sum_number = 0
# 列表渲染: 以正置负
for _label in self._dict_template_label[dataset_name]:
set_label_index = self._dict_template_label[dataset_name][_label]
set_label_index_update = set()
for label_index in set_label_index:
set_label_index_update.add(-label_index)
self._dict_template_label[dataset_name][_label] = set_label_index_update
# dict -> list
self._dict_template_label[dataset_name] = [
copy.deepcopy(self._dict_template_label[dataset_name]) for _ in range(
len(self._dict_padding_data[dataset_name])
)
]
# 遍历待渲染的列表 [同一列表型数据源在模板多处配置]
for render in self._dict_list_render[dataset_name]:
# 列表渲染: 0
for _label in self._dict_template_label[dataset_name][0]:
set_label_index = self._dict_template_label[dataset_name][0][_label]
set_label_index_update = set()
for label_index in set_label_index:
if render['start_index'] < -label_index < render['end_index']:
# 以负置正，并更新
set_label_index_update.add(-label_index + render_sum_number)
else:
set_label_index_update.add(label_index)
self._dict_template_label[dataset_name][0][_label] = set_label_index_update
para_index = render['end_index'] - 1 + render_sum_number
for padding_list_index in range(len(self._dict_padding_data[dataset_name]) - 1):
for index in range(
render['start_index'] + 1 + render_sum_number, render['end_index'] + render_sum_number
):
# 新型cv 完美复制
para = self._paragraphs[para_index]
_para = copy.deepcopy(self._paragraphs[index]._p)
para._p.addnext(_para)
para_index += 1
self._build()
# 老版cv 中文字体以及小标题编号格式丢失
# _para = self._paragraphs[index]
# self._add_para(para_before=para_before, para=para)
# 更新padding index
for _label in self._dict_template_label[dataset_name][padding_list_index + 1]:
set_label_index = self._dict_template_label[dataset_name][padding_list_index + 1][_label]
set_label_index_update = set()
for label_index in set_label_index:
if render['start_index'] < -label_index < render['end_index']:
# 以负为正，并更新
set_label_index_update.add(
-label_index + render_sum_number + (padding_list_index + 1) * (
render['end_index'] - render['start_index'] - 1
)
)
else:
set_label_index_update.add(label_index)
self._dict_template_label[dataset_name][padding_list_index + 1][_label] = set_label_index_update
# 删除列表渲染的开头与结尾
start_index = render['start_index'] + render_sum_number
end_index = para_index + 1
self._delete_paragraph(para=self._paragraphs[start_index])
self._delete_paragraph(para=self._paragraphs[end_index - 1])
# 更新列表渲染总计数器
render_sum_number += (
render['end_index'] - render['start_index'] - 1
) * (len(self._dict_padding_data[dataset_name]) - 1) - 2
# 因开头结尾渲染被删除，需重新更新该列表型数据集所有padding index
for dict_label_index in self._dict_template_label[dataset_name]:
for _label in dict_label_index:
set_output = set()
set_input = dict_label_index[_label]
for index in set_input:
if index > start_index:
index -= 1
if index > end_index:
index -= 1
set_output.add(index)
dict_label_index[_label] = set_output
# 2) 填充该列表数据
self._padding_list(dataset_name=dataset_name)
else:
'''
列表数据为None或[] 删除模板配置
'''
render_sum_number = 0
for render in self._dict_list_render[dataset_name]:
start_index = render['start_index'] - render_sum_number
end_index = render['end_index'] - render_sum_number
for _ in range(start_index, end_index + 1):
self._delete_paragraph(para=self._paragraphs[start_index])
render_sum_number += (end_index + 1 - start_index)
'''
*** 视情况加与不加 现版本采用 暂无副作用 ***
支持多加换行 如果列表不渲染[不存在] 一并删除待渲染块的末尾换行
'''
if self._paragraphs[start_index].text == '':
self._delete_paragraph(para=self._paragraphs[start_index])
render_sum_number += 1
# 对已渲染填充/删除的列表型数据集 -> 列入白名单
self._set_dataset_name.add(dataset_name)
# 3) update index 需重新扫描除该列表填充外的数据源
for dataset_ in self._dict_template_label:
# if dataset_ not in set_dataset:
self._dict_template_label[dataset_] = dict()
for dataset_ in list_render_dataset_name:
self._dict_list_render[dataset_] = []
self._scanning_data(set_dataset_name=self._set_dataset_name)
def _process_multi_para_render(self) -> None:
"""
1. 存储 [已存储] -> [self._list_multi_para_render]
2. 根据para_index升序 [在此实现]
3. 渲染 + style [在此实现]
:return:
"""
multi_para_render_num = 0
self._list_multi_para_render.sort(key=lambda k: k['para_index'])
for dict_multi_para_render in self._list_multi_para_render:
'''
{
'dataset_name': dataset_name,
'field_name': _label,
'para_index': index,
'padding_data': padding_data,
'para_num': len(padding_data)
}
1. 考虑首尾 [xxx, {{ key.value }}, yyy.] -> [xxx, key.value[0]] ... [key.value[-1], yyy.]
2. 考虑中间 [循环 + 删除首尾 + 渲染即可]
'''
dataset_name = dict_multi_para_render['dataset_name']
field_name = dict_multi_para_render['field_name']
para_index = dict_multi_para_render['para_index']
list_padding_data = dict_multi_para_render['list_padding_data']
para_num = dict_multi_para_render['para_num']
if len(dataset_name) > 0 and dataset_name[0] == '^':
pattern_template_label = '\^' + dataset_name[1:] + '\.' + field_name
else:
pattern_template_label = dataset_name + '\.' + field_name
pattern_label = re.compile(r'' + '{{{{\s*{}\s*}}}}'.format(pattern_template_label))
# 定位多段渲染run_index
list_run_index = self._position_run_index(
para_index=para_index + multi_para_render_num,
dataset_name=dataset_name,
field_name=field_name
)
list_run_index_new = []
list_para_style_index = []
if len(list_padding_data) > 0:
'''
1. cv run
2. cv para
'''
# 1. cv run
run_num = 0
for run_index in list_run_index:
# runs[run_index] = 'xx{{ dataset_name.field_name }}yy'切分为['xx', '{{ dataset_name.field_name }}', 'yy']
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num]
results = re.finditer(pattern_label, run.text)
run_text = copy.deepcopy(run.text)
# 改到这里了，总算是不乱了
# 1. cv run -> run_index_new
for _ in results:
result = re.search(pattern_label, run_text)
if result:
start_index, end_index = result.regs[0]
else:
continue
# text_head
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num]
run.text = run_text[: start_index]
# text_middle
_run = copy.deepcopy(run)
_run.text = run_text[start_index: end_index]
run._r.addnext(_run._r)
self._build()
# text_tail
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num + 1]
_run = copy.deepcopy(run)
_run.text = run_text[end_index:]
run._r.addnext(_run._r)
self._build()
list_run_index_new.append(run_index + run_num + 1)
run_text = run_text[end_index:]
run_num += 2
# 2. cv para
para_org = copy.deepcopy(self._paragraphs[para_index + multi_para_render_num])
for run_index in list_run_index_new:
# para_head
padding_data = str(list_padding_data[0]) if type(list_padding_data[0]) is int else list_padding_data[0]
runs = self._paragraphs[para_index + multi_para_render_num].runs
for i in range(run_index + 1, len(runs)):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
list_para_style_index.append(para_index + multi_para_render_num)
# para_middle
for index, padding_data in enumerate(list_padding_data[1: -1]):
padding_data = str(padding_data) if type(padding_data) is int else padding_data
para = self._paragraphs[para_index + multi_para_render_num + index]
_para = copy.deepcopy(para_org)
runs = _para.runs
for i in range(run_index):
runs[i].text = ''
for i in range(run_index + 1, len(runs)):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
para._p.addnext(_para._p)
self._build()
list_para_style_index.append(para_index + multi_para_render_num + index)
# para_tail
if len(list_padding_data) > 1:
padding_data = str(list_padding_data[-1]) if type(list_padding_data[-1]) is int else list_padding_data[-1]
para = self._paragraphs[para_index + multi_para_render_num + len(list_padding_data) - 2]
_para = copy.deepcopy(para_org)
runs = _para.runs
for i in range(run_index):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
para._p.addnext(_para._p)
self._build()
multi_para_render_num += len(list_padding_data) - 1
# 2) 定制style
if 'dict_style' in dict_multi_para_render:
for para_style_index in list_para_style_index:
self._process_style(
para_index=para_style_index,
list_padding_index=list_run_index_new,
dict_style=dict_multi_para_render['dict_style']
)
def _position_run_index(self, para_index, dataset_name: str, field_name: str) -> list:
begin, tmp = -100, ''
template_label = dataset_name + '.' + field_name
pattern_mustache = re.compile(r'{{.*?}}')
runs = self._paragraphs[para_index].runs
list_run_index = []
for index, run in enumerate(runs):
'''
runs[0].text = '{{ list'
runs[1].text = '_multi.info }}{{ list_multi.info }}
'''
if '{{' in run.text and begin == -100:
begin, tmp = index, run.text
elif '{' in run.text and begin == -100:
begin, tmp = index, run.text
else:
tmp += run.text
if pattern_mustache.findall(tmp):
if template_label in tmp:
# 如果存在匹配的字符串，那么将当前的run替换成合并后得字符串tmp
run.text = run.text.replace(run.text, tmp)
if begin != -100:
for i in range(begin, index):
runs[i].text = ''
list_run_index.append(index)
begin, tmp = -100, ''
elif '{{' in run.text:
begin, tmp = index, run.text
elif '{' in run.text:
begin, tmp = index, run.text
else:
begin, tmp = -100, ''
return list_run_index
def _process_table(self) -> None:
"""
Table
表格处理:
表格第一行Json -> [单一数据集模板配置]
表格内部 -> [已实现同一表格存在单一列表型数据集和多个对象型数据集配置]
处理过程: table: tables [一个一个扫描获取填充 获取可能为多次GET请求]
:return:
"""
for table in self._tables:
"""
1. 扫描列表型数据集 [只存在于table第一行, 且为JSON]
2. 扫描对象型数据集 + 定位padding index
3. 拉取对象型数据 + 数据填充 [对比dict_dataset, 以补全的策略拉取对象型数据集] 如数据集未注册, 同样删去
4. 拉取列表型数据 + 根据List长度cv表单 + 数据填充
"""
# 1. 扫描列表型数据集
json_table_config = None
row = table.rows[0]
for cell in row.cells:
set_result = self._regular_extract.match_pattern(
para_text=cell.text, patterns=['(?<={{).*?(?=}})']
)
for result in set_result:
try:
# Table Json
json_table_config = json.loads(result)
# print('Table Json: {}'.format(json_table_config))
row._element.getparent().remove(row._element)
except Exception as e:
pass
if json_table_config:
break
# 2. 扫描对象型数据集 + 定位padding index
self._table_dict_template_label = dict()
for index, cell in enumerate(table._cells):
set_result = self._regular_extract.match_pattern(
para_text=cell.text, patterns=['(?<={{).*?(?=}})']
)
for result in set_result:
list_template_label = result.split('.')
if len(list_template_label) == 2:
template_dataset = list_template_label[0]
template_label = list_template_label[1]
if template_dataset in self._table_dict_template_label:
if template_label in self._table_dict_template_label[template_dataset]:
self._table_dict_template_label[template_dataset][template_label].add(index)
else:
self._table_dict_template_label[template_dataset][template_label] = {index}
else:
self._table_dict_template_label[template_dataset] = {template_label: {index}}
# print("表格型数据对象{}".format(self._table_dict_template_label))
# 3. 拉取对象型数据 + 数据填充
list_dataset_name = list(self._table_dict_template_label.keys())
# print(list_dataset_name)
for dataset_name in self._table_dict_template_label:
table_dict_label_index = self._table_dict_template_label[dataset_name]
table_dict_padding_data = self._dict_padding_data[dataset_name]
for _label in table_dict_label_index:
if _label in table_dict_padding_data:
for index in table_dict_label_index[_label]:
paragraphs = table._cells[index].paragraphs
for para in paragraphs:
self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=table_dict_padding_data[_label]
)
# # 4. 拉取列表型数据 + cv表单 + 数据填充
# if json_table_config:
# # 1). 解析Json
# table_type = json_table_config['table.type'] if 'table.type' in json_table_config else None
# dataset_name = json_table_config['table.dataset'] if 'table.dataset' in json_table_config else None
# # 2). 查询数据集请求接口
# dataset = Dataset.query.filter(
# Dataset.project_name == self._project_name, Dataset.dataset_name == dataset_name
# ).first()
# # 3). 组织请求接口参数 + 拉取数据 + cv表单 + 数据填充
# if dataset:
# # 对于已注册的数据集: 信息存储 + 组织参数 + 请求数据
# self._request_data(
# dataset_name=dataset_name, dataset=dataset, json_config=json_table_config
# )
# # cv表单 + 数据填充
# if self._dict_dataset[dataset_name]['dataset_type'] == 0:
# """
# 字典型表格: 不需要cv 不需要更新padding index 直接padding
# """
# self._padding_table(
# table=table, dataset_name=dataset_name,
# table_dict_label_index=self._table_dict_template_label[dataset_name],
# table_dict_padding_data=self._dict_padding_data[dataset_name]
# )
# elif self._dict_dataset[dataset_name]['dataset_type'] == 1:
# """
# 列表型表格: 1. cv
# 2. 更新padding index
# 3. padding
# """
# # 完美cv
# self._table_dict_template_label[dataset_name] = [self._table_dict_template_label[dataset_name]]
# for _ in range(len(self._dict_padding_data[dataset_name]) - 1):
# row = table.rows[-1]
# _row = copy.deepcopy(table.rows[-1]._tr)
# row._tr.addnext(_row)
# # 更新 padding index
# dict_label_index = dict()
# for _label in self._table_dict_template_label[dataset_name][0]:
# set_label_index = self._table_dict_template_label[dataset_name][0][_label]
# dict_label_index[_label] = set()
# for label_index in set_label_index:
# dict_label_index[_label].add(
# label_index + (_ + 1) * len(
# list(self._table_dict_template_label[dataset_name][0].keys()))
# )
# self._table_dict_template_label[dataset_name].append(dict_label_index)
# # 填充Table数据
# for table_dict_label_index, table_dict_padding_data in zip(
# self._table_dict_template_label[dataset_name], self._dict_padding_data[dataset_name]
# ):
# self._padding_table(
# table=table, dataset_name=dataset_name,
# table_dict_label_index=table_dict_label_index,
# table_dict_padding_data=table_dict_padding_data
# )
# else:
# pass
def _process_picture(self, index: int, json_pic_config: dict, data_result: dict) -> None:
runs = self._paragraphs[index].runs
for run in runs:
run.text = ''
run = runs[0]
# 解析json
dataset_name = json_pic_config['chart.dataset'] if 'chart.dataset' in json_pic_config else None
# 获取数据对象
result_data = data_result[dataset_name]
# 绘图引擎
pic_name, pic_type = None, None
if 'chart.type' in json_pic_config and 'chart.type2' not in json_pic_config:
# 单图表 现版本支持: 饼图(pie) / 柱状图(bar) / 折线图(line)
if json_pic_config['chart.type'] == 'pie':
# echarts 饼图
keys_name = json_pic_config['chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
values_name = json_pic_config['chart.yaxis_column'] if 'chart.yaxis_column' in json_pic_config else None
keys, values = [], []
for data in result_data:
keys.append(data[keys_name])
values.append(data[values_name])
pic_name = pic_echarts_pie(
keys=keys,
values=values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None
)
logger.info("===={}--饼状图已生成====".format(dataset_name))
elif json_pic_config['chart.type'] == 'bar':
# echarts 柱状图
keys_name = json_pic_config[
'chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
list_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
keys, dict_values = [], {values_name: [] for values_name in list_values_name}
for data in result_data:
keys.append(data[keys_name])
for values_name in list_values_name:
dict_values[values_name].append(data[values_name])
pic_name = pic_echarts_bar(
keys=keys,
dict_values=dict_values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name=json_pic_config['chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
)
elif json_pic_config['chart.type'] == 'line':
# echarts 折线图
keys_name = json_pic_config[
'chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
list_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
keys, dict_values = [], {values_name: [] for values_name in list_values_name}
for data in result_data:
keys.append(data[keys_name])
for values_name in list_values_name:
dict_values[values_name].append(data[values_name])
pic_name = pic_echarts_line(
keys=keys,
dict_values=dict_values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name=json_pic_config['chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
)
elif 'chart.type' in json_pic_config and 'chart.type2' in json_pic_config:
# 组合图表 现版本支持: 双轴图: 柱状图(bar) + 折线图(line)
list_bar_values_name, list_line_values_name = [], []
y_name_left, y_name_right = None, None
if json_pic_config['chart.type'] == 'bar' and json_pic_config['chart.type2'] == 'line':
pic_type = 'bar_line'
list_bar_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
list_line_values_name = json_pic_config[
'chart.yaxis2_columns'] if 'chart.yaxis2_columns' in json_pic_config else []
y_name_left = json_pic_config[
'chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
y_name_right = json_pic_config[
'chart.yaxis2_name'] if 'chart.yaxis2_name' in json_pic_config else None
elif json_pic_config['chart.type'] == 'line' and json_pic_config['chart.type2'] == 'bar':
pic_type = 'line_bar'
list_bar_values_name = json_pic_config[
'chart.yaxis2_columns'] if 'chart.yaxis2_columns' in json_pic_config else []
list_line_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
y_name_left = json_pic_config[
'chart.yaxis2_name'] if 'chart.yaxis2_name' in json_pic_config else None,
y_name_right = json_pic_config[
'chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
if pic_type == 'bar_line' or pic_type == 'line_bar':
keys_name = json_pic_config[
'chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
keys = []
dict_bar_values = {bar_values_name: [] for bar_values_name in list_bar_values_name}
dict_line_values = {line_values_name: [] for line_values_name in list_line_values_name}
for data in result_data:
keys.append(data[keys_name])
for bar_values_name in list_bar_values_name:
dict_bar_values[bar_values_name].append(data[bar_values_name])
for line_values_name in list_line_values_name:
dict_line_values[line_values_name].append(data[line_values_name])
pic_name = pic_echarts_bar_line(
keys=keys,
dict_bar_values=dict_bar_values,
dict_line_values=dict_line_values,
colors=['#d14a61', '#5793f3', '#2f4554'],
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name_left=y_name_left,
y_name_right=y_name_right
)
if pic_name:
run.add_picture(pic_name, width=shared.Cm(15))
self._paragraphs[index].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
def _process_style(self, para_index: int, list_padding_index: list, dict_style: dict) -> None:
"""
支持定制加粗
list_padding_index:
list_bold:
1. 降序 去除重复子串 √
2. 遍历list_padding_index 待加粗字符串[max]
3. 两种思路 [均未实现]
1) 全程拆分 操作list
2) 记录start_index end_index列表[需控制start_index&end_index不在记录列表所有SE间] 最后切分
3) 先拆分为一个run存放一个char 并记录该label起始结束下标
选用3) 原因是方便后续支持颜色 下划线等等等
注意！！！若想支持其他 在1. 2.间加list即可 若太多可考虑dict 其余操作同2. 3.
:param para_index:
:param dict_style: 特定风格的字典{'bold': []..} [需加粗的文本列表]
:return:
"""
# 1. 降序 去除重复子串
list_bold = dict_style['bold'] if 'bold' in dict_style else []
list_bold_new = sorted(list_bold, key=lambda k: len(k), reverse=True)
'''
2022年04月01日 16:09:33修改
1. 降序问题 -> 按照字符串长度降序
2. 重复子串 -> 不可直接去除 因为公司全称不一定出现在文本中，也可能是简称
3. 策略为按照 长->短 对run进行切分 特定加粗
list_bold_new = []
for bold in list_bold:
bool_bold = False
for bold_new in list_bold_new:
if bold in bold_new:
bool_bold = True
if not bool_bold:
list_bold_new.append(bold)
'''
# 2. 拆分为一个run存放一个char 并记录该label起始结束下标
style_sum_number = 0
list_padding_index_new = []
for padding_index in list_padding_index:
start_index = padding_index + style_sum_number
run = self._paragraphs[para_index].runs[start_index]
run_text = copy.deepcopy(run.text)
len_run_rext = len(run_text)
run.text = run_text[: 1]
for i in range(len_run_rext - 1):
run = self._paragraphs[para_index].runs[padding_index + style_sum_number]
_run = copy.deepcopy(run._r)
_run.text = run_text[i + 1: i + 2]
run._r.addnext(_run)
style_sum_number += 1
self._build()
# 存放runs的[start_index, end_index]
list_padding_index_new.append(
{
'start_index': start_index,
'end_index': padding_index + style_sum_number
}
)
# 3. 设置加粗 run.font.bold = True
runs = self._paragraphs[para_index].runs
for dict_padding_index in list_padding_index_new:
padding_text = ''
for run_index in range(dict_padding_index['start_index'], dict_padding_index['end_index'] + 1):
padding_text += runs[run_index].text
for bold_new in list_bold_new:
for dict_index in self._regular_extract.match_index(para_text=padding_text, pattern_str=bold_new):
start_index = dict_index['start_index'] + dict_padding_index['start_index']
end_index = dict_index['end_index'] + dict_padding_index['start_index']
for i in range(start_index, end_index):
# todo: 加粗函数
runs[i].font.bold = True
'''
开放break: 只加粗首次出现的字体
注释break: 加粗所有匹配到的字体
'''
# break
def _padding_dict(self, dataset_name: str) -> None:
"""
填充数据 类型为: Dict(0)
:param dataset_name: 数据集名称 info | finance
:return:
"""
# todo: 此处由于对象有些区别，先获取待填充数据
dict_label_index = self._dict_template_label[dataset_name]
dict_padding_data = self._dict_padding_data[dataset_name]
# print(dict_padding_data)
# print(dict_label_index)
self._padding(
dataset_name=dataset_name,
dict_label_index=dict_label_index,
dict_padding_data=dict_padding_data
)
def _padding_list(self, dataset_name: str) -> None:
"""
填充数据 类型为: List(1)
:param dataset_name: 数据集名称
:return:
"""
list_label_index = self._dict_template_label[dataset_name]
list_padding_data = self._dict_padding_data[dataset_name]
for dict_label_index, dict_padding_data in zip(list_label_index, list_padding_data):
self._padding(
dataset_name=dataset_name, dict_label_index=dict_label_index, dict_padding_data=dict_padding_data
)
def _padding(self, dataset_name: str, dict_label_index: dict, dict_padding_data: dict) -> None:
"""
模板填充
填充文本为 str 或 list
str:
1. 填充
2. 定制style
list:
多段渲染[正文常用]
1. 存储 [在此实现]
2. 根据para_index升序
3. 渲染 + style
:param dataset_name: 数据集名称
:param dict_label_index: 待替换模板所在位置
:param dict_padding_data: 待填充数据集
:return:
"""
for _label in dict_label_index:
if _label in dict_padding_data:
for index in dict_label_index[_label]:
padding_data = dict_padding_data[_label]
# 防止返回的数据为None类型[且没配置删除'^'] (为None经测试也没任何问题, 填充替换结果会变成'')
if type(padding_data) is str or type(padding_data) is int or type(padding_data) is list:
padding_data = padding_data
else:
padding_data = ''
# print(padding_data, type(padding_data))
if type(padding_data) is str or type(padding_data) is int:
'''
1. 填充
2. 定制style
'''
# 1) 填充
para = self._paragraphs[index]
list_padding_index = self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=padding_data
)
# print(list_padding_index)
# 2) 定制style
if 'style' in dict_padding_data:
for dict_style in dict_padding_data['style']:
if _label == dict_style['field']:
self._process_style(
para_index=index,
list_padding_index=list_padding_index,
dict_style=dict_style
)
break
elif type(padding_data) is list:
'''
多段渲染[正文常用]
1. 存储 [在此实现]
2. 根据para_index升序
3. 渲染 + style
'''
dict_multi_para_render = {
'dataset_name': dataset_name,
'field_name': _label,
'para_index': index,
'list_padding_data': padding_data,
'para_num': len(padding_data)
}
if 'style' in dict_padding_data:
for dict_style in dict_padding_data['style']:
if _label == dict_style['field']:
dict_multi_para_render['dict_style'] = dict_style
self._list_multi_para_render.append(dict_multi_para_render)
def _padding_runs(self, runs, dataset_name: str, field_name: str, padding_data: str or int) -> list:
"""
核心替换 [重点关注] 后续可能问题多多！！！
功能描述:
文档某一段落 分runs进行替换
runs: list[]
padding_data_str: str
存在多处替换，但都是同一个dataset_name + dataset_label
:param runs: 代替换文本
:param dataset_name: 数据集名称 [模板配置前缀] (dataset).
:param field_name: 数据集字段 [模板配置后缀] .(company_name)
:param padding_data: 可用于填充的数据
:return: 返回runs中填充label的定位下标列表 [供给特定style使用]
"""
if type(padding_data) is int:
padding_data = str(padding_data)
begin, tmp = -100, ''
template_label = dataset_name + '.' + field_name # 模板标签全称[用于二次确认+替换]
pattern_mustache = re.compile(r'{{.*?}}')
if len(dataset_name) > 0 and dataset_name[0] == '^':
pattern_template_label = '\^' + dataset_name[1:] + '\.' + field_name
else:
pattern_template_label = dataset_name + '\.' + field_name
pattern_label = re.compile(r'' + '{{{{\s*{}\s*}}}}'.format(pattern_template_label))
list_padding_index = []
for index, run in enumerate(runs):
'''
重难点
有标题格式 开头出现的 {{ 会分为两个run
如果后续支持计算，需修改此处代码
这里为何用if 而不是 elif
run.text = '}}{{' 连一起这种情况
'''
if '{{' in run.text and begin == -100:
begin, tmp = index, run.text
elif '{' in run.text and begin == -100:
begin, tmp = index, run.text
else:
tmp += run.text
if pattern_mustache.findall(tmp):
if template_label in tmp:
# 如果存在匹配的字符串，那么将当前的run替换成合并后得字符串tmp
run.text = re.sub(pattern_label, padding_data, tmp)
'''
len(runs) = 1
runs[0].text = '{{ base_info.risk_title }}'
或者 runs[i].text = ' }}' runs[i + 1].text = '{{ dataset_name.filed_name }}'
此时会出现 begin = -100, end_index = 0
IndexError: list index out of range
'''
if begin != -100:
for i in range(begin, index):
runs[i].text = ''
list_padding_index.append(index)
begin, tmp = -100, ''
elif '{{' in run.text:
begin, tmp = index, run.text
elif '{' in run.text:
begin, tmp = index, run.text
else:
begin, tmp = -100, ''
return list_padding_index
def _padding_table(
self, table, dataset_name: str,
table_dict_label_index: dict,
table_dict_padding_data: dict
) -> None:
for _label in table_dict_label_index:
if _label in table_dict_padding_data:
for index in table_dict_label_index[_label]:
paragraphs = table._cells[index].paragraphs
for para in paragraphs:
self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=table_dict_padding_data[_label]
)
def _delete_annotation(self) -> None:
"""
删除模板中注释: {{ !xxx }}
:return: None
"""
for index, para in enumerate(self._paragraphs):
set_result = self._regular_extract.match_pattern(para_text=para.text, patterns=['(?<={{).*?(?=}})'])
for result in set_result:
if len(result) > 0 and result[0] == '!':
self._delete_paragraph(para=para)
pass
def _delete_paragraph(self, para) -> None:
"""
删除word文档某段落
:param para:
:return: None
"""
p = para._element
p.getparent().remove(p)
para._p = para._element = None
self._build()
def _save(self) -> None:
"""
将word文档保存至本地
:return:
"""
# self._document.styles['Normal']._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# self._document.styles['Normal'].font.name = 'Times New Roman'
self._delete_annotation()
self._document.save(self._output_report_path)
def _add_para(self, para_before, para) -> None:
"""
新插入段落 [已弃用]
:param para_before:
:param para:
:return:
"""
# 插入一个段落，代码换行，word中就是一个段落，不换行
new_para = para_before.insert_paragraph_before()
self._add_runs(new_para=new_para, para=para)
""" Paragraph 段落格式设置 """
# 设置段落文本右对齐
new_para.paragraph_format.alignment = para.paragraph_format.alignment
""" 段落缩进 """
# 左缩进
new_para.paragraph_format.left_indent = para.paragraph_format.left_indent
# 右缩进
new_para.paragraph_format.right_indent = para.paragraph_format.right_indent
# 首行缩进
new_para.paragraph_format.first_line_indent = para.paragraph_format.first_line_indent
""" 行间距 """
new_para.paragraph_format.line_spacing = para.paragraph_format.line_spacing
"当line_spacing设置为长度值时表示绝对距离，"
"设置为浮点数时表示行高的倍数"
# 段前间距
new_para.paragraph_format.space_before = para.paragraph_format.space_before
# 段后间距
new_para.paragraph_format.space_after = para.paragraph_format.space_after
""" 设置段落内部文字在遇到需分页情况时处理状态 """
new_para.paragraph_format.keep_together = para.paragraph_format.keep_together # 段中不分页
new_para.paragraph_format.keep_with_next = para.paragraph_format.keep_with_next # 与下段同页
new_para.paragraph_format.page_break_before = para.paragraph_format.page_break_before # 段前分页
new_para.paragraph_format.widow_control = para.paragraph_format.widow_control # 孤行控制
# 获取段落的左缩进，首行缩进，段前间距：
l_space = new_para.paragraph_format.left_indent
h_space = new_para.paragraph_format.first_line_indent
b_space = new_para.paragraph_format.space_before
# print(l_space, h_space, b_space) # 457200 914400 63500
self._build()
def _add_runs(self, new_para, para) -> None:
"""
新建段落[由runs组成] 新建run [已弃用]
:param new_para:
:param para:
:return:
"""
for run in para.runs:
# 新建一个段落，增加一段文字
new_run = new_para.add_run(run.text)
new_run.bold = run.bold
new_run.element = run.element
""" 设置字体格式 """
# new_run.font.name = run.font.name # 注：这个好像设置 run 中的西文字体
# new_run.font.element = run.font.element
# new_run._element.rPr.rFonts = run._element.rPr.rFonts
# new_run._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
new_run.font.name = 'Times New Roman' # 注：这个好像设置 run 中的西文字体
# new_run._element.rPr.rFonts.set(qn('w:eastAsia'), '微软雅黑')
new_run._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# new_run.style._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# new_run.style._element.rPr.rFonts.set(qn('w:ascii'), 'Times New Roman')
# print(new_run.font.name)
# 设置中文字体
# new_run.font.element.rPr.rFonts.set(qn('w:eastAsia'), '宋体')
# 设置字体大小
new_run.font.size = run.font.size
# 设置加粗
new_run.font.bold = run.font.bold
# 设置斜体
new_run.font.italic = run.font.italic
# 设置字体颜色
new_run.font.color.rgb = run.font.color.rgb
# 设置背景颜色
new_run.font.highlight_color = run.font.highlight_color
# 设置下划线
new_run.font.underline = run.font.underline
# 设置轮廓线
new_run.font.outline = run.font.outline
# 设置阴影
new_run.font.shadow = run.font.shadow
# 删除线
new_run.font.strike = run.font.strike
# 双删除线
new_run.font.double_strike = run.font.double_strike
# 设置下标
new_run.font.subscript = run.font.subscript
# 设置上标
new_run.font.superscript = run.font.superscript
new_run.italic = run.italic
new_run.style.name = run.style.name
new_run.underline = run.underline
def main_process(half_work_path, tables_dict, template_path, report_name, data_object, save_path):
document = Document(half_work_path)
# todo: 步骤1——从半成品中提取需要的表格（基于 tables_dict 定义的表格内容）
data_result = get_choose_table(document, list(tables_dict.values()))
if data_result["以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明"]:
data_result = data_result
else:
tables_dict.update({"table13": "本单位无以名义金额计量的资产"})
temp_table_result = get_choose_table(document, ["本单位无以名义金额计量的资产"])
del data_result["以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明"]
data_result.update(temp_table_result)
logger.info(data_result)
# todo： 步骤2——从半成品中复制需要的内容到模板中
copy_content_main(half_work_path, template_path)
# todo: 步骤3——将半成品中提取到的表格内容填充到模板中
generate_report(data_result, save_path, template_path, tables_dict)
# todo： 步骤4——填充基本数据到模板中
project_name = "四川报告"
template_dir, template_name = os.path.split(save_path)
gurag = GeneralUserReportAutoGenerated(
project_name=project_name,
input_template_path=save_path,
output_report_path=os.path.join(template_dir, report_name),
start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
)
gurag.process(data_object)
return None
if __name__ == "__main__":
import os
data_object = {
"finance": {
"publicInfrastructureRatio": "",
"affordabelHouseNewRatio": "",
"assetLiabilityRatio": "24.31%",
"assetLiabilityRatioRemark": "说明本单位财务风险低",
"beInDebt": "264.31",
"beInDebtChangeRatio": "24.88%",
"beInDebtChangeRatioRemark": "主要原因是本年的其他应付款的减少",
"cashRatio": "248.35%",
"cashRatioRemark": "说明本单位利用现金和现金等价物偿还短期债务的能力强",
"composition": "流动资产占100.00%",
"currentAssetsCompose": "货币资金",
"currentLiabilitiesCompose": "其他应付款",
"currentRatio": "248.35%",
"currentRatioRemark": "说明本单位流动资产偿还短期债务的能力强",
"debtComparisonRatio": "减少87.56万元，减少24.88%",
"debtComposition": "流动负债占100.00%",
"fixedAssetsDepreciationRatio": "77.75%",
"fixedAssetsDepreciationRatioRemark": "说明本单位固定资产持续服务能力较强",
"netAssets": "823.10",
"nonCurrentAssetsCompose": "固定资产原值、固定资产净值、无形资产原价和无形资产净值",
"otherRemark": "资产总额较上年增减变动幅度超过20%主要原因是货币资金减少265.35万元;负债总额较上年增减变动幅度超过20%主要原因是其他应付款减少87.56万元",
"revenueExpensesRatio": "100.00%",
"revenueExpensesRatioRemark": "小于",
"surplus": "",
"totalAssets": "1087.40",
"totalAssetsChangeRatio": "20.53%",
"totalAssetsChangeRatioRemark": "主要原因是本年的货币资金的减少",
"totalAssetsComparison": "减少280.84万元，减少20.53%",
"totalExpenses": "1890.01",
"totalExpensesChangeRatio": "0.20%",
"totalExpensesChangeRatioRemark": "主要原因是本年的业务活动费用的减少",
"totalExpensesComparison": "减少3.77万元，减少0.20%",
"totalExpensesCompose": "业务活动费用",
"totalRevenue": "1890.01",
"totalRevenueChangeRatio": "15.49%",
"totalRevenueChangeRatioRemark": "主要原因是本年的财政拨款收入的减少",
"totalRevenueComparison": "减少346.51万元，减少15.49%",
"totalRevenueCompose": "财政拨款收入和其他收入",
"totalRevenueComposeDetail": "财政拨款收入占比99.88%、其他收入占比0.12%",
"unitAssetComposition": "流动资产",
"unitDebtComposition": "流动负债"
},
"收入占比": [
{
"beforeDataValue": "1229.05",
"dataValue": "1887.83",
"indexName": "财政拨款收入",
"subtractValue": "658.78"
},
{
"beforeDataValue": "5.42",
"dataValue": "2.19",
"indexName": "其他收入",
"subtractValue": "3.23"
}
],
"流动负债占比": [
{
"beforeDataValue": "351.87",
"dataValue": "264.31",
"indexName": "其他应付款",
"subtractValue": "87.56"
}
],
"流动资产占比": [
{
"beforeDataValue": "921.76",
"dataValue": "656.41",
"indexName": "货币资金",
"subtractValue": "265.35"
}
],
"负债占比": [
{
"beforeDataValue": "351.87",
"dataValue": "264.31",
"indexName": "流动负债合计"
}
],
"费用占比": [
{
"beforeDataValue": "1893.72",
"dataValue": "1890.01",
"indexName": "业务活动费用",
"subtractValue": "3.71"
}
],
"资产占比": [
{
"beforeDataValue": "921.76",
"dataValue": "656.41",
"indexName": "流动资产合计"
},
{
"beforeDataValue": "446.48",
"dataValue": "431.00",
"indexName": "非流动资产合计"
}
],
"非流动资产占比": [
{
"beforeDataValue": "554.31",
"dataValue": "554.31",
"indexName": "固定资产原值",
"subtractValue": "0.00"
},
{
"beforeDataValue": "446.48",
"dataValue": "431.00",
"indexName": "固定资产净值",
"subtractValue": "15.48"
},
{
"beforeDataValue": "0.00",
"dataValue": "0.00",
"indexName": "无形资产原价",
"subtractValue": "0.00"
},
{
"beforeDataValue": "0.00",
"dataValue": "0.00",
"indexName": "无形资产净值",
"subtractValue": "0.00"
}
],
"info": {
"internalControl": "2021年，本单位加强学习国家和省关于内部控制的文件。建立健全了单位层面的内部控制体系和制度，健全了预算、收支、采购、建设、资产和合同的内控流程和制度，把内部控制落实在业务流程中，实现了不相容岗位相互分离、形成相互制约、相互监督的工作机制；实现了内部授权审批控制。",
"unitName": "安岳县元坝镇人民政府",
"unitCall": "本部门",
"mainFunctions": "无资料数据",
"year": "2021",
"amountDescription": "本单位无以名义金额计量的资产。",
"unitBudgetLevel": "二级预算单位",
"institutionalSituation": "无资料数据",
"performanceManagement": "2021年，本单位按照绩效管理要求对照设定预算绩效目标、绩效指标的成本指标、产出指标、效益指标、满意度指标等具体内容，开展项目绩效目标申报、运行监控和自评工作。通过预算绩效管理对工作中存在的薄弱环节作出针对性查漏补缺和持续完善。",
"LastYear": "2020",
"personnelSituation": "无资料数据",
"unitType": "行政单位",
"budgetManagement": "2021年，本单位严格按照《预算法》、《会计法》、《政府会计制度》和上级的文件建立健全财务制度；严格执行财经纪律和各项财务制度；强化预算管理，加强对银行存款和现金的管理；单位对年终决算高度重视，组织专人负责编制决算报告，对决算数据进行了严格审核，认真分析并应用到下年的预算工作。",
"assetManagement": "2021年，本单位资产实行分类管理，建立健全了资产内部管理制度；单位加强对实物资产和无形资产的管理，明确相关部门和岗位的职责权限，强化对配置、使用和处置等关键环节的管控；明确资产使用和保管责任人，落实资产使用人在资产管理中的责任。",
"pppProject": "本单位无PPP项目。",
"careerAchievements": "无资料数据"
}
}
half_work_path = "../data/3月23测试半成品.docx"
template_path = "../data/new_财务报告模板.docx"
tables_dict = {
"table13": "以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明",
"table5": "收入费用表（2）",
"table4": "收入费用表（1）",
"table3": "资产负债表续表2",
"table2": "资产负债表续表1",
"table1": "资产负债表",
}
save_path = "../data/temp_module.docx"
report_name = "4月11号财务报告测试.docx"
main_process(half_work_path, tables_dict, template_path, report_name, data_object, save_path)
generate/pic_echarts.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : pic_echarts
# @Author : LiuYan
# @Time : 2021/12/16 19:29
import os
from pyecharts import options as opts
from pyecharts.charts import Pie, Bar, Line, Grid
from pyecharts.faker import Faker
from pyecharts.render import make_snapshot # 导入输出图片工具
from snapshot_selenium import snapshot # 使用snapshot-selenium 渲染图片
from pyecharts.globals import CurrentConfig, ThemeType
from pathlib import Path
# import time
from unittest import mock
from base.config.base_config import root_dir
from utils.tools import timeit
# root_dir = '..'
# 解决linux 下图片生成失败问题
def get_chrome_driver():
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument("headless")
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
return webdriver.Chrome(options=options)
# def timeit(f):
# def timed(*args, **kw):
# ts = time.time()
# print('......begin {0:8s}......'.format(f.__name__))
# result = f(*args, **kw)
# te = time.time()
# print('......finish {0:8s}, took:{1:.4f} sec......'.format(f.__name__, te - ts))
# return result
#
# return timed
"""
关于: [图片生成的中文字体样式渲染问题]
中文字体全局配置默认为None, 即为读取服务器端已有中文字体样式, 按照优先顺序[源码与定义好的]进行中文渲染。
例如39服务器上有simhei和simsun, 中文字体样式会选用simsun
目前39服务器上仅放置simhei, 后续可根据具体需要放置其他中文字体样式, 并实现在代码中全局或局部控制生成的中文样式
"""
# CurrentConfig.ONLINE_HOST = 'https://cdn.jsdelivr.net/gh/pyecharts/pyecharts-assets@master/assets/'
# define "echarts.min.js"文件存放地址
CurrentConfig.ONLINE_HOST = 'http://39.105.62.235:8000/assets/'
# CurrentConfig.ONLINE_HOST = 'http://192.168.1.149:8000/assets/'
pic_echarts_dir = os.path.join(root_dir, 'generate/echarts')
Path(pic_echarts_dir).mkdir(parents=True, exist_ok=True)
@timeit
def pic_echarts_pie(keys: list, values: list, title: str or None) -> str:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_pie.png')
pie = (
Pie().add(
series_name='',
data_pair=[list(z) for z in zip(keys, values)],
center=['45%', '50%'],
is_clockwise=False
).set_global_opts(
title_opts=opts.TitleOpts(title=title),
legend_opts=opts.LegendOpts(type_='scroll', pos_left='80%', orient='vertical', textstyle_opts=opts.TextStyleOpts(font_size=20))
).set_series_opts(
# label_opts=opts.LabelOpts(formatter='{b}: {c}({d}%)')
label_opts=opts.LabelOpts(formatter="{b}: {d}%", font_size=20)
)
# # 设置标签字体大小
# .set_series_opts(label_opts=opts.LabelOpts(font_size=22))
)
with mock.patch('snapshot_selenium.snapshot.get_chrome_driver', get_chrome_driver):
make_snapshot(snapshot, pie.render(), pic_echarts_path)
# make_snapshot(snapshot, pie.render(), pic_echarts_path)
return pic_echarts_path
@timeit
def pic_echarts_bar(
keys: list, dict_values: dict, title=None,
x_name=None, y_name=None
) -> str:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_bar.png')
bar = (
Bar().add_xaxis(
xaxis_data=keys
).set_global_opts(
title_opts=opts.TitleOpts(title=title),
xaxis_opts=opts.AxisOpts(name=x_name, axislabel_opts=opts.LabelOpts(rotate=-15)),
yaxis_opts=opts.AxisOpts(name=y_name),
legend_opts=opts.LegendOpts(type_='scroll', pos_left='80%', orient='vertical', textstyle_opts=opts.TextStyleOpts(font_size=18))
)
# 设置标签字体大小
.set_series_opts(label_opts=opts.LabelOpts(font_size=20))
)
for key in dict_values:
bar.add_yaxis(series_name=key, y_axis=dict_values[key])
make_snapshot(snapshot, bar.render(), pic_echarts_path)
return pic_echarts_path
@timeit
def pic_echarts_line(
keys: list, dict_values: dict, title=None,
x_name=None, y_name=None
) -> str:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_line.png')
line = (
Line().add_xaxis(
xaxis_data=keys
).set_global_opts(
title_opts=opts.TitleOpts(title=title),
xaxis_opts=opts.AxisOpts(name=x_name, axislabel_opts=opts.LabelOpts(rotate=-15)),
yaxis_opts=opts.AxisOpts(name=y_name),
legend_opts=opts.LegendOpts(textstyle_opts=opts.TextStyleOpts(font_size=20))
)
# 设置标签字体大小
.set_series_opts(label_opts=opts.LabelOpts(font_size=20))
)
for key in dict_values:
line.add_yaxis(series_name=key, y_axis=dict_values[key])
make_snapshot(snapshot, line.render(), pic_echarts_path)
return pic_echarts_path
@timeit
def pic_echarts_line_test() -> None:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_line_test.png')
x_data = ['周一', '周二', '周三', '周四', '周五', '周六', '周日']
y_data = [820, 932, 901, 934, 1290, 1330, 1320]
line = (
Line()
.add_xaxis(xaxis_data=x_data)
.add_yaxis(
series_name='邮件营销',
stack='总量',
y_axis=[120, 132, 101, 134, 90, 230, 210],
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
series_name='联盟广告',
stack='总量',
y_axis=[220, 182, 191, 234, 290, 330, 310],
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
series_name='视频广告',
stack='总量',
y_axis=[150, 232, 201, 154, 190, 330, 410],
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
series_name='直接访问',
stack='总量',
y_axis=[320, 332, 301, 334, 390, 330, 320],
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
series_name='搜索引擎',
stack='总量',
y_axis=[820, 932, 901, 934, 1290, 1330, 1320],
label_opts=opts.LabelOpts(is_show=False),
)
.set_global_opts(
title_opts=opts.TitleOpts(title='折线图堆叠'),
tooltip_opts=opts.TooltipOpts(trigger='axis'),
yaxis_opts=opts.AxisOpts(
type_='value',
axistick_opts=opts.AxisTickOpts(is_show=True),
splitline_opts=opts.SplitLineOpts(is_show=True),
),
xaxis_opts=opts.AxisOpts(type_='category', boundary_gap=False),
)
)
make_snapshot(snapshot, line.render(), pic_echarts_path)
@timeit
def pic_echarts_bar_line(
keys=['2016年报', '2017年报', '2018年报', '2019年报', '2020年报', '2021年报'],
dict_bar_values={
'总资产': [1905.11, 1998.17, 2009.65, 2031.37, 1950.35, 1988.65],
'总负债': [1367.25, 1340.74, 1332.95, 1323.80, 1238.31, 1230.78]
},
dict_line_values={
'负债率': [70.72, 67.10, 66.33, 65.17, 63.49, 61.89]
},
colors=['#d14a61', '#5793f3', '#2f4554'],
title='资产负债表(CNY)',
x_name='年度', y_name_left='金额/(亿元)', y_name_right='负债率/(%)'
) -> str:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_bar_line.png')
bar = (
Bar().add_xaxis(
xaxis_data=keys
).extend_axis(
yaxis=opts.AxisOpts(
name=y_name_right,
position='right'
)
).set_global_opts(
xaxis_opts=opts.AxisOpts(name=x_name),
yaxis_opts=opts.AxisOpts(
name=y_name_left,
position='left'
),
title_opts=opts.TitleOpts(title=title),
tooltip_opts=opts.TooltipOpts(trigger='axis', axis_pointer_type='cross'),
legend_opts=opts.LegendOpts(textstyle_opts=opts.TextStyleOpts(font_size=20))
)
# 设置标签字体大小
.set_series_opts(label_opts=opts.LabelOpts(font_size=20))
)
index = 0
for bar_key in dict_bar_values:
bar.add_yaxis(
series_name=bar_key,
y_axis=dict_bar_values[bar_key],
yaxis_index=0, z=0,
color=colors[index]
)
index += 1
line = (
Line().add_xaxis(
xaxis_data=keys
)
)
for line_key in dict_line_values:
line.add_yaxis(
series_name=line_key,
y_axis=dict_line_values[line_key],
# color=colors[index],
yaxis_index=1,
label_opts=opts.LabelOpts(is_show=False)
)
index += 1
bar.overlap(line)
grid = Grid()
grid.add(bar, opts.GridOpts(pos_left='5%', pos_right='5%'), is_control_axis_index=True)
make_snapshot(snapshot, grid.render(), pic_echarts_path)
return pic_echarts_path
@timeit
def pic_echarts_bar_line_test() -> str:
pic_echarts_path = os.path.join(pic_echarts_dir, 'echarts_bar_line_test.png')
x_data = ['{}月'.format(i) for i in range(1, 13)]
bar = (
Bar().add_xaxis(
xaxis_data=x_data
).add_yaxis(
series_name='蒸发量',
y_axis=[2.0, 4.9, 7.0, 23.2, 25.6, 76.7, 135.6, 162.2, 32.6, 20.0, 6.4, 3.3],
yaxis_index=0, color='#d14a61',
).add_yaxis(
series_name='降水量',
y_axis=[2.6, 5.9, 9.0, 26.4, 28.7, 70.7, 175.6, 182.2, 48.7, 18.8, 6.0, 2.3],
yaxis_index=1, color='#5793f3',
).extend_axis(
yaxis=opts.AxisOpts(
name='蒸发量',
type_='value',
min_=0,
max_=250,
position='right',
axisline_opts=opts.AxisLineOpts(
linestyle_opts=opts.LineStyleOpts(color='#d14a61')
),
axislabel_opts=opts.LabelOpts(formatter='{value} ml'),
)
).extend_axis(
yaxis=opts.AxisOpts(
type_='value',
name='温度',
min_=0,
max_=25,
position="left",
axisline_opts=opts.AxisLineOpts(
linestyle_opts=opts.LineStyleOpts(color='#675bba')
),
axislabel_opts=opts.LabelOpts(formatter='{value} °C'),
splitline_opts=opts.SplitLineOpts(
is_show=True, linestyle_opts=opts.LineStyleOpts(opacity=1)
),
)
).set_global_opts(
yaxis_opts=opts.AxisOpts(
name='降水量',
min_=0,
max_=250,
position='right',
offset=80,
axisline_opts=opts.AxisLineOpts(
linestyle_opts=opts.LineStyleOpts(color='#5793f3')
),
axislabel_opts=opts.LabelOpts(formatter='{value} ml'),
),
title_opts=opts.TitleOpts(title='多 Y 轴图示例'),
tooltip_opts=opts.TooltipOpts(trigger='axis', axis_pointer_type='cross'),
)
)
line = (
Line().add_xaxis(
xaxis_data=x_data
).add_yaxis(
'平均温度',
[2.0, 2.2, 3.3, 4.5, 6.3, 10.2, 20.3, 23.4, 23.0, 16.5, 12.0, 6.2],
yaxis_index=2,
color='#675bba',
label_opts=opts.LabelOpts(is_show=False),
)
)
bar.overlap(line)
grid = Grid()
grid.add(bar, opts.GridOpts(pos_left='5%', pos_right='20%'), is_control_axis_index=True)
make_snapshot(snapshot, grid.render(), pic_echarts_path)
return pic_echarts_path
if __name__ == '__main__':
pic_echarts_pie(keys=Faker.choose(), values=Faker.values(), title='Echarts Pie 标题1')
# pic_echarts_bar(
# keys=Faker.choose(),
# dict_values={
# '2019': Faker.values(),
# '2020': Faker.values(),
# '2021': Faker.values()
# },
# title='Echarts Bar 标题'
# )
# pic_echarts_line(
# keys=Faker.choose(),
# dict_values={
# '折线1': Faker.values(),
# '折线2': Faker.values()
# },
# title='Echarts Line 标题',
# x_name='X轴名称',
# y_name='Y轴名称'
# )
# pic_echarts_line_test()
# pic_echarts_bar_line()
generate/pic_plt.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : pic_plt
# @Author : LiuYan
# @Time : 2021/12/17 16:21
import os
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
from matplotlib import rcParams
from pathlib import Path
from base.config.base_config import root_dir
pic_plt_dir = os.path.join(root_dir, 'generate/plt')
Path(pic_plt_dir).mkdir(parents=True, exist_ok=True)
# for chinese show
# 1. 中英文统一: 中文格式
plt.rcParams['font.family'] = 'sans-serif'
plt.rcParams['font.sans-serif'] = ['SimSun'] # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False
# 2. 中文√ 英文: Latex
# config = {
# "font.family": 'serif',
# "mathtext.fontset": 'stix',
# "font.serif": ['SimSun'],
# }
# rcParams.update(config)
# 3. error
# mpl.use('pgf')
# pgf_config = {
# "font.family": 'serif',
# "mathtext.fontset": 'stix',
# "pgf.rcfonts": False,
# "text.usetex": True,
# "pgf.preamble": [
# r"\usepackage{unicode-math}",
# r"\setmainfont{Times New Roman}",
# r"\usepackage{xeCJK}",
# r"\xeCJKsetup{CJKmath=true}",
# r"\setCJKmainfont{SimSun}"
# ]
# }
# rcParams.update(pgf_config)
def plt_bar(names, values, picture):
plt.figure(figsize=(9, 3))
plt.bar(names, values) # bar
# plt.suptitle('柱状图', fontproperties=myfont)
plt.savefig(picture) # eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff 都可以
plt.show()
def plt_plot(names, values, picture):
plt.figure(figsize=(9, 3))
plt.plot(names, values) # line
# plt.suptitle('折线图', fontproperties=myfont)
plt.savefig(picture) # eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff 都可以
plt.show()
def pic_plt_pie(keys: list, values: list, title: str or None) -> str:
# plt画饼图（数据，数据对应的标签，百分数保留两位小数点）
pic_plt_path = os.path.join(pic_plt_dir, 'plt_pie.png')
plt.pie(
x=values,
labels=keys,
autopct='%1.1f%%',
startangle=90
)
plt.title(title)
plt.savefig(pic_plt_path) # eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff 都可以
# plt.show()
plt.close()
return pic_plt_path
def plt_pie_2(labels, sizes, picture):
# Pie chart, where the slices will be ordered and plotted counter-clockwise:
explode = (0.1, 0.1, 0.1, 0.1, 0.1, 0.1) # only "explode" the 2nd slice (i.e. 'Hogs')
fig1, ax1 = plt.subplots()
labels = ['China', 'Swiss', 'USA', 'UK', 'Laos在', 'Spain']
X = [222, 42, 455, 664, 454, 334]
ax1.pie(x=X, explode=explode, labels=labels, autopct='%1.1f%%',
shadow=True, startangle=90)
ax1.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.
# plt.suptitle('饼状图', fontproperties=myfont)
plt.savefig(picture) # eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff 都可以
plt.show()
def plt_scatter(names, values, picture):
plt.figure(figsize=(9, 3))
plt.scatter(names, values) # scatter
# plt.suptitle('散点图', fontproperties=myfont)
plt.savefig(picture) # eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff 都可以
plt.show()
if __name__ == '__main__':
# print(mpl.matplotlib_fname())
# Test -> plt
# plt.title(r'宋体 Times New Roman')
# plt.axis('off')
# plt.savefig('usestix.png')
# plt.show()
keys = ['Frogs', 'Hogs', 'Dogs', 'Logs']
values = [15, 30, 45, 10]
pic_plt_pie(keys, values, "国资研究")
# plt_bar(names, values, '柱状图.jpg')
# plt_plot(names, values, '折线图.jpg')
# plt_pie_2(labels, sizes, '饼状图.jpg')
# plt_scatter(names, values, '散点图.jpg')
generate/platform_generated.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : gen_user_report_auto_generated
# @Author : LiuYan
# @Time : 2021/12/7 14:50
import re, os
import copy
import json
import chevron
import datetime
import requests
from docx import Document, shared
from docx.oxml.ns import qn
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
from generate.regular_extract import RegularExtract
from generate.pic_echarts import pic_echarts_pie, pic_echarts_bar, pic_echarts_line, pic_echarts_bar_line
# from generate.pic_plt import pic_plt_pie
from utils.log import logger
from utils.database_mysql import DatabaseMySQL
from pathlib import Path
# from app_run import doc2docx
# from transform_doc_to_docx import doc2docx
UPLOAD_FOLDER = r'../data' # 上传路径
Path(UPLOAD_FOLDER).mkdir(parents=True, exist_ok=True)
# 定义数据库链接基础信息
# database_config = {
# 'host': '114.115.159.144',
# 'port': 3306,
# 'user': 'root',
# 'password': 'zzsn9988',
# 'database': 'clb_project'
# }
database_config = {
'host': '114.115.205.50',
# 'host': '114.116.44.11',
'port': 3306,
'user': 'root',
# 'password': 'f7s0&7qqtK',
'password': 'yotop@123456',
'database': 'clb_project'
}
dbm = DatabaseMySQL(database_config=database_config)
temp_url = "http://114.115.215.96/"
# temp_url = "http://114.115.205.50/"
# task_id = '1641261934625521666'
# dataset_id = '1641045122365317122'
class GeneralUserReportAutoGenerated(object):
"""
1. 扫描数据源
2. 拉取数据
3. 数据填充
"""
def __init__(self, project_name: str, input_template_path: str, output_report_path: str,
start_time: datetime, end_time: datetime, parameter='',
is_handle_success='ynHandleSuccess', result_data='resultData'):
super(GeneralUserReportAutoGenerated, self).__init__()
self._project_name = project_name
self._start_time = start_time
self._end_time = end_time
self._parameters = {
'start_time': self._start_time,
'end_time': self._end_time
}
self._parameter = parameter
self._is_handle_success = is_handle_success
self._result_data = result_data
self._input_template_path = input_template_path
self._output_report_path = output_report_path
self._document = Document(self._input_template_path)
self._paragraphs = self._document.paragraphs
self._tables = self._document.tables
self._regular_extract = RegularExtract()
self._table_dict_template_label = dict() # 模板标签索引 [表格]
self._dict_template_label_del = dict() # 模板标签索引 [del]
self._dict_template_label = dict() # 模板标签索引——普通类型的标签索引
self._dict_padding_data = dict() # 填充数据
self._dict_dataset = dict() # 数据集信息
self._dict_list_render = dict() # 列表渲染
self._set_dataset_name = set() # 用于存放已填充数据集名称和未配置的数据集名称
self._list_multi_para_render = list() # 用于存放多段渲染
self._build()
def _build(self) -> None:
"""
创建 / 更新 段落 + 表格 [列表]
:return:
"""
self._paragraphs = self._document.paragraphs
self._tables = self._document.tables
def _build_parameter(self, parameter: str, json_config: dict) -> str:
"""
组织请求参数
:param parameter: 参数
:param json_config: 参数配置
:return: parameter [str]
"""
self._parameters = {
'start_time': self._start_time,
'end_time': self._end_time
}
for json_config_key in json_config:
list_para = json_config_key.split('.')
if len(list_para) == 2:
self._parameters[list_para[1]] = json_config[json_config_key]
parameter = chevron.render(parameter, self._parameters)
return parameter
def process(self, data_result: dict) -> None:
"""
1. 扫描文本数据集 + 图片生成引擎(Json + Data + Generate)
2. 拉取文本数据 + 删除未注册数据集
3. 填充对象型数据集 + 列表型数据集 [含多段渲染]
4. 表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)
5. 报告生成完毕 保存
[1. 图片 2. 文本(列表型) 3. 文本 4. 表格]
:return: None
"""
# 1. 扫描文本数据集 + 图片生成引擎(Json + Data + Generate)
logger.info('开始扫描文本数据集 + 启动图片生成引擎(Json + Data + Generate)')
self._scanning_data(set_dataset_name=self._set_dataset_name, data_result=data_result)
# 2. 获取数据信息
logger.info('获取数据信息')
# self._getting_data(data_result)
self._dict_padding_data = data_result
logger.info("处理第一部分普通的info.year后的结果: {}".format(self._dict_template_label))
# todo：当前数据内容
logger.info("当前待填充数据内容{}".format(self._dict_padding_data))
# 3. 文本数据填充
# todo： 基于self._getting_data 函数得到 self._dict_padding_data
for dataset_name in self._dict_template_label:
# print(dataset_name)
self._padding_dict(dataset_name=dataset_name)
# 列表型填充
self._process_text_list_render(data_result)
# 多段渲染 [统一渲染]
self._process_multi_para_render()
# 4. 表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)
logger.info('启动表格生成引擎(表格第一行Json + 表格内对象型数据集 + 填充)')
# 5. 表格数据填充
# todo： 基于self._getting_data 函数得到 self._dict_padding_data
self._process_table()
# 6. 报告生成完毕 保存
logger.info('报告生成完毕! 保存......')
self._save()
# todo: 先找到待填充数据的位置（相应的段落索引，索引下标默认从0开始计数）
def _scanning_data(self, set_dataset_name: set, data_result: dict) -> None:
"""
扫描数据
:param set_dataset_name: index已经重定位好[已填充]的数据 仅有data_type=1会插入其中
补充: 若数据集未注册，则也会在拉取数据集时存入，以后便不再扫描该数据源
:return:
"""
start_index, end_index, dataset_name = None, None, None
for index, para in enumerate(self._paragraphs):
set_result = self._regular_extract.match_pattern(para_text=para.text, patterns=['(?<={{).*?(?=}})'])
for result in set_result:
# 原始步骤1 扫描列表渲染 [start_index: end_index]
# todo: 先处理“#zhanlue”样式 找到所有待渲染的数据集名称并将其相应的段落索引（起始）加入待渲染列表self._dict_list_render中，数据对象为第二部分的数据集列表
# {'zhanlue': [{'start_index': 12, 'end_index': 14}], 'caiwu': [{'start_index': 17, 'end_index': 19}], 'shichang': [{'start_index': 22, 'end_index': 24}], 'yunying': [{'start_index': 27, 'end_index': 29}], 'falv': [{'start_index': 32, 'end_index': 34}], 'centerHonest': [{'start_index': 37, 'end_index': 39}], 'localHonest': [{'start_index': 40, 'end_index': 42}], 'private': [{'start_index': 46, 'end_index': 48}]}
if len(result) > 0 and result.strip()[0] == '#':
start_index = index
dataset_name = result.strip()[1:]
continue
elif len(result) > 0 and result.strip()[0] == '/':
end_index = index
if result.strip()[1:] == dataset_name and dataset_name not in set_dataset_name:
if dataset_name not in self._dict_list_render:
self._dict_list_render[dataset_name] = []
self._dict_list_render[dataset_name].append(
{
'start_index': start_index,
'end_index': end_index
}
)
start_index, end_index, dataset_name = None, None, None
continue
# 原始步骤2 特殊文本型渲染: {{ ^dataset.title }} 若title为None或''则删除该行(不进行渲染)。 数据对象为第一部分的info数据集 待处理样式 ^info.zlRisk
if len(result) > 0 and result[0] == '^':
list_template_label = result[1:].split('.')
if len(list_template_label) == 2:
template_dataset_name = list_template_label[0]
template_dataset_label = list_template_label[1]
if template_dataset_name not in set_dataset_name:
# print("当前模板标签索引对象为: {}".format(self._dict_template_label_del))
if template_dataset_name in self._dict_template_label_del:
# print(self._dict_template_label_del[template_dataset_name])
# 如果info数据集的label 对象已存在，则追加段落索引位置，反之则插入新的label对象到模板标签集合中，当前该情况不存在。
if template_dataset_label in self._dict_template_label_del[template_dataset_name]:
self._dict_template_label_del[template_dataset_name][template_dataset_label].add(index)
else:
# 当前info数据集的label 对象不存在，将新的标签对象插入到模板标签集合中
self._dict_template_label_del[template_dataset_name][template_dataset_label] = {index}
else:
# 由于数据集名称都是第一部分的对象“info”，则只有初始的时候进入该部分，处理后的结果为{"info": {'zlRisk': {11}}}
self._dict_template_label_del[template_dataset_name] = {template_dataset_label: {index}}
continue
# 图片型渲染
try:
json_result = json.loads(result)
logger.info('Picture Json: {}'.format(json_result))
# todo: 基于传过来的数据生成相应的数据图
self._process_picture(index=index, json_pic_config=json_result, data_result=data_result)
continue
except Exception as e:
pass
# 文本渲染(对象型)
# todo： 处理第一部分info.year普通标签
list_template_label = result.split('.')
if len(list_template_label) == 2:
# dataset 为数据对象名称 初版有info 和 finance
template_dataset = list_template_label[0]
# label 为相应的属性名称
template_label = list_template_label[1].strip()
if template_dataset not in set_dataset_name:
if template_dataset in self._dict_template_label:
if template_label in self._dict_template_label[template_dataset]:
self._dict_template_label[template_dataset][template_label].add(index)
else:
self._dict_template_label[template_dataset][template_label] = {index}
else:
self._dict_template_label[template_dataset] = {template_label: {index}}
continue
def _getting_data(self, data_result: dict) -> None:
"""
本项目无需拉取数据，只需要存储数据结果
:return:
"""
list_dataset_name = list(
set(list(self._dict_template_label.keys()))
)
# 遍历扫描到的所有文本型数据集 [拉取 / 删除]
for dataset_name in list_dataset_name:
result = data_result[dataset_name]
# 4. 存储数据集的数据信息
self._dict_padding_data[dataset_name] = result
def _process_text_list_render(self, data_result) -> None:
"""
文本(列表型): 1. 渲染(cv)/删除(Delete) 2. 更新(update index) 3. 填充(padding data)
:return:
"""
list_render_dataset_name = list(self._dict_list_render.keys())
for dataset_name in list_render_dataset_name:
'''
在此判断:
1. 该数据集在已注册字典中
2. 该数据集类型是列表型
3. 该数据集数据不为None或[]
'''
'''
在此判断:
1. 该数据集数据不为None或[] -> 正常进行列表渲染 cv
2. 如果为None或[] -> 列表渲染部分删除[开头 -> 结尾] 重定位index
'''
# 1) 渲染(cv)/删除(Delete)
if self._dict_padding_data[dataset_name]:
render_sum_number = 0
# 列表渲染: 以正置负
for _label in self._dict_template_label[dataset_name]:
# print(self._dict_template_label)
set_label_index = self._dict_template_label[dataset_name][_label]
set_label_index_update = set()
for label_index in set_label_index:
set_label_index_update.add(-label_index)
self._dict_template_label[dataset_name][_label] = set_label_index_update
# dict -> list
self._dict_template_label[dataset_name] = [
copy.deepcopy(self._dict_template_label[dataset_name]) for _ in range(
len(self._dict_padding_data[dataset_name])
)
]
# 遍历待渲染的列表 [同一列表型数据源在模板多处配置]
for render in self._dict_list_render[dataset_name]:
# 列表渲染: 0
for _label in self._dict_template_label[dataset_name][0]:
set_label_index = self._dict_template_label[dataset_name][0][_label]
set_label_index_update = set()
for label_index in set_label_index:
if render['start_index'] < -label_index < render['end_index']:
# 以负置正，并更新
set_label_index_update.add(-label_index + render_sum_number)
else:
set_label_index_update.add(label_index)
self._dict_template_label[dataset_name][0][_label] = set_label_index_update
para_index = render['end_index'] - 1 + render_sum_number
for padding_list_index in range(len(self._dict_padding_data[dataset_name]) - 1):
for index in range(
render['start_index'] + 1 + render_sum_number, render['end_index'] + render_sum_number
):
# 新型cv 完美复制
para = self._paragraphs[para_index]
_para = copy.deepcopy(self._paragraphs[index]._p)
para._p.addnext(_para)
para_index += 1
self._build()
# 老版cv 中文字体以及小标题编号格式丢失
# _para = self._paragraphs[index]
# self._add_para(para_before=para_before, para=para)
# 更新padding index
for _label in self._dict_template_label[dataset_name][padding_list_index + 1]:
set_label_index = self._dict_template_label[dataset_name][padding_list_index + 1][_label]
set_label_index_update = set()
for label_index in set_label_index:
if render['start_index'] < -label_index < render['end_index']:
# 以负为正，并更新
set_label_index_update.add(
-label_index + render_sum_number + (padding_list_index + 1) * (
render['end_index'] - render['start_index'] - 1
)
)
else:
set_label_index_update.add(label_index)
self._dict_template_label[dataset_name][padding_list_index + 1][_label] = set_label_index_update
# 删除列表渲染的开头与结尾
start_index = render['start_index'] + render_sum_number
end_index = para_index + 1
self._delete_paragraph(para=self._paragraphs[start_index])
self._delete_paragraph(para=self._paragraphs[end_index - 1])
# 更新列表渲染总计数器
render_sum_number += (
render['end_index'] - render['start_index'] - 1
) * (len(self._dict_padding_data[dataset_name]) - 1) - 2
# 因开头结尾渲染被删除，需重新更新该列表型数据集所有padding index
for dict_label_index in self._dict_template_label[dataset_name]:
for _label in dict_label_index:
set_output = set()
set_input = dict_label_index[_label]
for index in set_input:
if index > start_index:
index -= 1
if index > end_index:
index -= 1
set_output.add(index)
dict_label_index[_label] = set_output
# 2) 填充该列表数据
self._padding_list(dataset_name=dataset_name)
else:
'''
列表数据为None或[] 删除模板配置
'''
render_sum_number = 0
for render in self._dict_list_render[dataset_name]:
start_index = render['start_index'] - render_sum_number
end_index = render['end_index'] - render_sum_number
for _ in range(start_index, end_index + 1):
self._delete_paragraph(para=self._paragraphs[start_index])
render_sum_number += (end_index + 1 - start_index)
'''
*** 视情况加与不加 现版本采用 暂无副作用 ***
支持多加换行 如果列表不渲染[不存在] 一并删除待渲染块的末尾换行
'''
if self._paragraphs[start_index].text == '':
self._delete_paragraph(para=self._paragraphs[start_index])
render_sum_number += 1
# 对已渲染填充/删除的列表型数据集 -> 列入白名单
self._set_dataset_name.add(dataset_name)
# 3) update index 需重新扫描除该列表填充外的数据源
for dataset_ in self._dict_template_label:
# if dataset_ not in set_dataset:
self._dict_template_label[dataset_] = dict()
for dataset_ in list_render_dataset_name:
self._dict_list_render[dataset_] = []
self._scanning_data(set_dataset_name=self._set_dataset_name, data_result=data_result)
def _process_multi_para_render(self) -> None:
"""
1. 存储 [已存储] -> [self._list_multi_para_render]
2. 根据para_index升序 [在此实现]
3. 渲染 + style [在此实现]
:return:
"""
multi_para_render_num = 0
self._list_multi_para_render.sort(key=lambda k: k['para_index'])
for dict_multi_para_render in self._list_multi_para_render:
'''
{
'dataset_name': dataset_name,
'field_name': _label,
'para_index': index,
'padding_data': padding_data,
'para_num': len(padding_data)
}
1. 考虑首尾 [xxx, {{ key.value }}, yyy.] -> [xxx, key.value[0]] ... [key.value[-1], yyy.]
2. 考虑中间 [循环 + 删除首尾 + 渲染即可]
'''
dataset_name = dict_multi_para_render['dataset_name']
field_name = dict_multi_para_render['field_name']
para_index = dict_multi_para_render['para_index']
list_padding_data = dict_multi_para_render['list_padding_data']
para_num = dict_multi_para_render['para_num']
if len(dataset_name) > 0 and dataset_name[0] == '^':
pattern_template_label = '\^' + dataset_name[1:] + '\.' + field_name
else:
pattern_template_label = dataset_name + '\.' + field_name
pattern_label = re.compile(r'' + '{{{{\s*{}\s*}}}}'.format(pattern_template_label))
# 定位多段渲染run_index
list_run_index = self._position_run_index(
para_index=para_index + multi_para_render_num,
dataset_name=dataset_name,
field_name=field_name
)
list_run_index_new = []
list_para_style_index = []
if len(list_padding_data) > 0:
'''
1. cv run
2. cv para
'''
# 1. cv run
run_num = 0
for run_index in list_run_index:
# runs[run_index] = 'xx{{ dataset_name.field_name }}yy'切分为['xx', '{{ dataset_name.field_name }}', 'yy']
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num]
results = re.finditer(pattern_label, run.text)
run_text = copy.deepcopy(run.text)
# 改到这里了，总算是不乱了
# 1. cv run -> run_index_new
for _ in results:
result = re.search(pattern_label, run_text)
if result:
start_index, end_index = result.regs[0]
else:
continue
# text_head
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num]
run.text = run_text[: start_index]
# text_middle
_run = copy.deepcopy(run)
_run.text = run_text[start_index: end_index]
run._r.addnext(_run._r)
self._build()
# text_tail
run = self._paragraphs[para_index + multi_para_render_num].runs[run_index + run_num + 1]
_run = copy.deepcopy(run)
_run.text = run_text[end_index:]
run._r.addnext(_run._r)
self._build()
list_run_index_new.append(run_index + run_num + 1)
run_text = run_text[end_index:]
run_num += 2
# 2. cv para
para_org = copy.deepcopy(self._paragraphs[para_index + multi_para_render_num])
for run_index in list_run_index_new:
# para_head
padding_data = str(list_padding_data[0]) if type(list_padding_data[0]) is int else list_padding_data[0]
runs = self._paragraphs[para_index + multi_para_render_num].runs
for i in range(run_index + 1, len(runs)):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
list_para_style_index.append(para_index + multi_para_render_num)
# para_middle
for index, padding_data in enumerate(list_padding_data[1: -1]):
padding_data = str(padding_data) if type(padding_data) is int else padding_data
para = self._paragraphs[para_index + multi_para_render_num + index]
_para = copy.deepcopy(para_org)
runs = _para.runs
for i in range(run_index):
runs[i].text = ''
for i in range(run_index + 1, len(runs)):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
para._p.addnext(_para._p)
self._build()
list_para_style_index.append(para_index + multi_para_render_num + index)
# para_tail
if len(list_padding_data) > 1:
padding_data = str(list_padding_data[-1]) if type(list_padding_data[-1]) is int else list_padding_data[-1]
para = self._paragraphs[para_index + multi_para_render_num + len(list_padding_data) - 2]
_para = copy.deepcopy(para_org)
runs = _para.runs
for i in range(run_index):
runs[i].text = ''
runs[run_index].text = re.sub(pattern_label, padding_data, runs[run_index].text)
para._p.addnext(_para._p)
self._build()
multi_para_render_num += len(list_padding_data) - 1
# 2) 定制style
if 'dict_style' in dict_multi_para_render:
for para_style_index in list_para_style_index:
self._process_style(
para_index=para_style_index,
list_padding_index=list_run_index_new,
dict_style=dict_multi_para_render['dict_style']
)
def _position_run_index(self, para_index, dataset_name: str, field_name: str) -> list:
begin, tmp = -100, ''
template_label = dataset_name + '.' + field_name
pattern_mustache = re.compile(r'{{.*?}}')
runs = self._paragraphs[para_index].runs
list_run_index = []
for index, run in enumerate(runs):
'''
runs[0].text = '{{ list'
runs[1].text = '_multi.info }}{{ list_multi.info }}
'''
if '{{' in run.text and begin == -100:
begin, tmp = index, run.text
elif '{' in run.text and begin == -100:
begin, tmp = index, run.text
else:
tmp += run.text
if pattern_mustache.findall(tmp):
if template_label in tmp:
# 如果存在匹配的字符串，那么将当前的run替换成合并后得字符串tmp
run.text = run.text.replace(run.text, tmp)
if begin != -100:
for i in range(begin, index):
runs[i].text = ''
list_run_index.append(index)
begin, tmp = -100, ''
elif '{{' in run.text:
begin, tmp = index, run.text
elif '{' in run.text:
begin, tmp = index, run.text
else:
begin, tmp = -100, ''
return list_run_index
def _process_table(self) -> None:
"""
Table
表格处理:
表格第一行Json -> [单一数据集模板配置]
表格内部 -> [已实现同一表格存在单一列表型数据集和多个对象型数据集配置]
处理过程: table: tables [一个一个扫描获取填充 获取可能为多次GET请求]
:return:
"""
for table in self._tables:
"""
1. 扫描列表型数据集 [只存在于table第一行, 且为JSON]
2. 扫描对象型数据集 + 定位padding index
3. 拉取对象型数据 + 数据填充 [对比dict_dataset, 以补全的策略拉取对象型数据集] 如数据集未注册, 同样删去
4. 拉取列表型数据 + 根据List长度cv表单 + 数据填充
"""
# 1. 扫描列表型数据集
json_table_config = None
row = table.rows[0]
for cell in row.cells:
set_result = self._regular_extract.match_pattern(
para_text=cell.text, patterns=['(?<={{).*?(?=}})']
)
for result in set_result:
try:
# Table Json
json_table_config = json.loads(result)
logger.info('Table Json: {}'.format(json_table_config))
row._element.getparent().remove(row._element)
except Exception as e:
pass
if json_table_config:
break
# 2. 扫描对象型数据集 + 定位padding index
self._table_dict_template_label = dict()
for index, cell in enumerate(table._cells):
set_result = self._regular_extract.match_pattern(
para_text=cell.text, patterns=['(?<={{).*?(?=}})']
)
for result in set_result:
list_template_label = result.split('.')
if len(list_template_label) == 2:
template_dataset = list_template_label[0].strip()
template_label = list_template_label[1].strip()
if template_dataset in self._table_dict_template_label:
if template_label in self._table_dict_template_label[template_dataset]:
self._table_dict_template_label[template_dataset][template_label].add(index)
else:
self._table_dict_template_label[template_dataset][template_label] = {index}
else:
self._table_dict_template_label[template_dataset] = {template_label: {index}}
logger.info("表格型数据对象{}".format(self._table_dict_template_label))
# 3. 拉取对象型数据 + 数据填充
list_dataset_name = list(self._table_dict_template_label.keys())
# print(list_dataset_name)
for dataset_name in self._table_dict_template_label:
table_dict_label_index = self._table_dict_template_label[dataset_name]
table_dict_padding_data = self._dict_padding_data[dataset_name]
for _label in table_dict_label_index:
if _label in table_dict_padding_data:
for index in table_dict_label_index[_label]:
paragraphs = table._cells[index].paragraphs
for para in paragraphs:
self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=table_dict_padding_data[_label]
)
# 4. 拉取列表型数据 + cv表单 + 数据填充
if json_table_config:
# 1). 解析Json
table_type = json_table_config['table.type'] if 'table.type' in json_table_config else None
dataset_name = json_table_config['table.dataset'].strip() if 'table.dataset' in json_table_config else None
# 3). 组织请求接口参数 + 拉取数据 + cv表单 + 数据填充
"""
先处理字典型表格: 不需要cv 不需要更新padding index 直接padding
"""
# cv表单 + 数据填充
if type(self._dict_padding_data[dataset_name]) is dict:
self._padding_table(
table=table, dataset_name=dataset_name,
table_dict_label_index=self._table_dict_template_label[dataset_name],
table_dict_padding_data=self._dict_padding_data[dataset_name]
)
elif type(self._dict_padding_data[dataset_name]) is list:
"""
列表型表格: 1. cv
2. 更新padding index
3. padding
"""
# 完美cv
self._table_dict_template_label[dataset_name] = [self._table_dict_template_label[dataset_name]]
for _ in range(len(self._dict_padding_data[dataset_name]) - 1):
row = table.rows[-1]
_row = copy.deepcopy(table.rows[-1]._tr)
row._tr.addnext(_row)
# 更新 padding index
dict_label_index = dict()
for _label in self._table_dict_template_label[dataset_name][0]:
set_label_index = self._table_dict_template_label[dataset_name][0][_label]
dict_label_index[_label] = set()
for label_index in set_label_index:
dict_label_index[_label].add(
label_index + (_ + 1) * len(
list(self._table_dict_template_label[dataset_name][0].keys()))
)
self._table_dict_template_label[dataset_name].append(dict_label_index)
# 填充Table数据
for table_dict_label_index, table_dict_padding_data in zip(
self._table_dict_template_label[dataset_name], self._dict_padding_data[dataset_name]
):
self._padding_table(
table=table, dataset_name=dataset_name,
table_dict_label_index=table_dict_label_index,
table_dict_padding_data=table_dict_padding_data
)
else:
pass
def _process_picture(self, index: int, json_pic_config: dict, data_result: dict) -> None:
runs = self._paragraphs[index].runs
for run in runs:
run.text = ''
run = runs[0]
# 解析json
dataset_name = json_pic_config['chart.dataset'] if 'chart.dataset' in json_pic_config else None
# 获取数据对象
result_data = data_result[dataset_name]
# 绘图引擎
pic_name, pic_type = None, None
if 'chart.type' in json_pic_config and 'chart.type2' not in json_pic_config:
# 单图表 现版本支持: 饼图(pie) / 柱状图(bar) / 折线图(line)
if json_pic_config['chart.type'] == 'pie':
# echarts 饼图
keys_name = json_pic_config['chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
values_name = json_pic_config['chart.yaxis_column'] if 'chart.yaxis_column' in json_pic_config else None
keys, values = [], []
# todo: 不能放在模板里来获取， 将key 和 vlaue 作为参数来进行赋值
for data in result_data:
keys.append(data[keys_name])
values.append(data[values_name])
pic_name = pic_echarts_pie(
keys=keys,
values=values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None
)
logger.info("===={}--饼状图已生成====".format(dataset_name))
elif json_pic_config['chart.type'] == 'bar':
# echarts 柱状图
keys_name = json_pic_config['chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
list_values_name = json_pic_config['chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
keys, dict_values = [], {values_name: [] for values_name in list_values_name}
for data in result_data:
keys.append(data[keys_name])
for values_name in list_values_name:
dict_values[values_name].append(data[values_name])
pic_name = pic_echarts_bar(
keys=keys,
dict_values=dict_values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name=json_pic_config['chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
)
elif json_pic_config['chart.type'] == 'line':
# echarts 折线图
keys_name = json_pic_config[
'chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
list_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
keys, dict_values = [], {values_name: [] for values_name in list_values_name}
for data in result_data:
keys.append(data[keys_name])
for values_name in list_values_name:
dict_values[values_name].append(data[values_name])
pic_name = pic_echarts_line(
keys=keys,
dict_values=dict_values,
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name=json_pic_config['chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
)
elif 'chart.type' in json_pic_config and 'chart.type2' in json_pic_config:
# 组合图表 现版本支持: 双轴图: 柱状图(bar) + 折线图(line)
list_bar_values_name, list_line_values_name = [], []
y_name_left, y_name_right = None, None
if json_pic_config['chart.type'] == 'bar' and json_pic_config['chart.type2'] == 'line':
pic_type = 'bar_line'
list_bar_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
list_line_values_name = json_pic_config[
'chart.yaxis2_columns'] if 'chart.yaxis2_columns' in json_pic_config else []
y_name_left = json_pic_config[
'chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
y_name_right = json_pic_config[
'chart.yaxis2_name'] if 'chart.yaxis2_name' in json_pic_config else None
elif json_pic_config['chart.type'] == 'line' and json_pic_config['chart.type2'] == 'bar':
pic_type = 'line_bar'
list_bar_values_name = json_pic_config[
'chart.yaxis2_columns'] if 'chart.yaxis2_columns' in json_pic_config else []
list_line_values_name = json_pic_config[
'chart.yaxis_columns'] if 'chart.yaxis_columns' in json_pic_config else []
y_name_left = json_pic_config[
'chart.yaxis2_name'] if 'chart.yaxis2_name' in json_pic_config else None,
y_name_right = json_pic_config[
'chart.yaxis_name'] if 'chart.yaxis_name' in json_pic_config else None
if pic_type == 'bar_line' or pic_type == 'line_bar':
keys_name = json_pic_config[
'chart.xaxis_column'] if 'chart.xaxis_column' in json_pic_config else None
keys = []
dict_bar_values = {bar_values_name: [] for bar_values_name in list_bar_values_name}
dict_line_values = {line_values_name: [] for line_values_name in list_line_values_name}
for data in result_data:
keys.append(data[keys_name])
for bar_values_name in list_bar_values_name:
dict_bar_values[bar_values_name].append(data[bar_values_name])
for line_values_name in list_line_values_name:
dict_line_values[line_values_name].append(data[line_values_name])
pic_name = pic_echarts_bar_line(
keys=keys,
dict_bar_values=dict_bar_values,
dict_line_values=dict_line_values,
colors=['#d14a61', '#5793f3', '#2f4554'],
title=json_pic_config['chart.title'] if 'chart.title' in json_pic_config else None,
x_name=json_pic_config['chart.xaxis_name'] if 'chart.xaxis_name' in json_pic_config else None,
y_name_left=y_name_left,
y_name_right=y_name_right
)
if pic_name:
run.add_picture(pic_name, width=shared.Cm(15))
self._paragraphs[index].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
def _process_style(self, para_index: int, list_padding_index: list, dict_style: dict) -> None:
"""
支持定制加粗
list_padding_index:
list_bold:
1. 降序 去除重复子串 √
2. 遍历list_padding_index 待加粗字符串[max]
3. 两种思路 [均未实现]
1) 全程拆分 操作list
2) 记录start_index end_index列表[需控制start_index&end_index不在记录列表所有SE间] 最后切分
3) 先拆分为一个run存放一个char 并记录该label起始结束下标
选用3) 原因是方便后续支持颜色 下划线等等等
注意！！！若想支持其他 在1. 2.间加list即可 若太多可考虑dict 其余操作同2. 3.
:param para_index:
:param dict_style: 特定风格的字典{'bold': []..} [需加粗的文本列表]
:return:
"""
# 1. 降序 去除重复子串
list_bold = dict_style['bold'] if 'bold' in dict_style else []
list_bold_new = sorted(list_bold, key=lambda k: len(k), reverse=True)
'''
2022年04月01日 16:09:33修改
1. 降序问题 -> 按照字符串长度降序
2. 重复子串 -> 不可直接去除 因为公司全称不一定出现在文本中，也可能是简称
3. 策略为按照 长->短 对run进行切分 特定加粗
list_bold_new = []
for bold in list_bold:
bool_bold = False
for bold_new in list_bold_new:
if bold in bold_new:
bool_bold = True
if not bool_bold:
list_bold_new.append(bold)
'''
# 2. 拆分为一个run存放一个char 并记录该label起始结束下标
style_sum_number = 0
list_padding_index_new = []
for padding_index in list_padding_index:
start_index = padding_index + style_sum_number
run = self._paragraphs[para_index].runs[start_index]
run_text = copy.deepcopy(run.text)
len_run_rext = len(run_text)
run.text = run_text[: 1]
for i in range(len_run_rext - 1):
run = self._paragraphs[para_index].runs[padding_index + style_sum_number]
_run = copy.deepcopy(run._r)
_run.text = run_text[i + 1: i + 2]
run._r.addnext(_run)
style_sum_number += 1
self._build()
# 存放runs的[start_index, end_index]
list_padding_index_new.append(
{
'start_index': start_index,
'end_index': padding_index + style_sum_number
}
)
# 3. 设置加粗 run.font.bold = True
runs = self._paragraphs[para_index].runs
for dict_padding_index in list_padding_index_new:
padding_text = ''
for run_index in range(dict_padding_index['start_index'], dict_padding_index['end_index'] + 1):
padding_text += runs[run_index].text
for bold_new in list_bold_new:
for dict_index in self._regular_extract.match_index(para_text=padding_text, pattern_str=bold_new):
start_index = dict_index['start_index'] + dict_padding_index['start_index']
end_index = dict_index['end_index'] + dict_padding_index['start_index']
for i in range(start_index, end_index):
# todo: 加粗函数
runs[i].font.bold = True
'''
开放break: 只加粗首次出现的字体
注释break: 加粗所有匹配到的字体
'''
# break
def _padding_dict(self, dataset_name: str) -> None:
"""
填充数据 类型为: Dict(0)
:param dataset_name: 数据集名称 info | finance
:return:
"""
# todo: 此处由于对象有些区别，先获取待填充数据
dict_label_index = self._dict_template_label[dataset_name]
dict_padding_data = self._dict_padding_data[dataset_name]
# print(dict_padding_data)
# print(dict_label_index)
self._padding(
dataset_name=dataset_name,
dict_label_index=dict_label_index,
dict_padding_data=dict_padding_data
)
def _padding_list(self, dataset_name: str) -> None:
"""
填充数据 类型为: List(1)
:param dataset_name: 数据集名称
:return:
"""
list_label_index = self._dict_template_label[dataset_name]
list_padding_data = self._dict_padding_data[dataset_name]
for dict_label_index, dict_padding_data in zip(list_label_index, list_padding_data):
self._padding(
dataset_name=dataset_name, dict_label_index=dict_label_index, dict_padding_data=dict_padding_data
)
def _padding(self, dataset_name: str, dict_label_index: dict, dict_padding_data: dict) -> None:
"""
模板填充
填充文本为 str 或 list
str:
1. 填充
2. 定制style
list:
多段渲染[正文常用]
1. 存储 [在此实现]
2. 根据para_index升序
3. 渲染 + style
:param dataset_name: 数据集名称
:param dict_label_index: 待替换模板所在位置
:param dict_padding_data: 待填充数据集
:return:
"""
for _label in dict_label_index:
if _label in dict_padding_data:
for index in dict_label_index[_label]:
padding_data = dict_padding_data[_label]
# 防止返回的数据为None类型[且没配置删除'^'] (为None经测试也没任何问题, 填充替换结果会变成'')
if type(padding_data) is str or type(padding_data) is int or type(padding_data) is list:
padding_data = padding_data
else:
padding_data = ''
# print(padding_data, type(padding_data))
if type(padding_data) is str or type(padding_data) is int:
'''
1. 填充
2. 定制style
'''
# 1) 填充
para = self._paragraphs[index]
list_padding_index = self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=padding_data
)
# print(list_padding_index)
# 2) 定制style
if 'style' in dict_padding_data:
for dict_style in dict_padding_data['style']:
if _label == dict_style['field']:
self._process_style(
para_index=index,
list_padding_index=list_padding_index,
dict_style=dict_style
)
break
elif type(padding_data) is list:
'''
多段渲染[正文常用]
1. 存储 [在此实现]
2. 根据para_index升序
3. 渲染 + style
'''
dict_multi_para_render = {
'dataset_name': dataset_name,
'field_name': _label,
'para_index': index,
'list_padding_data': padding_data,
'para_num': len(padding_data)
}
if 'style' in dict_padding_data:
for dict_style in dict_padding_data['style']:
if _label == dict_style['field']:
dict_multi_para_render['dict_style'] = dict_style
self._list_multi_para_render.append(dict_multi_para_render)
def _padding_runs(self, runs, dataset_name: str, field_name: str, padding_data: str or int) -> list:
"""
核心替换 [重点关注] 后续可能问题多多！！！
功能描述:
文档某一段落 分runs进行替换
runs: list[]
padding_data_str: str
存在多处替换，但都是同一个dataset_name + dataset_label
:param runs: 代替换文本
:param dataset_name: 数据集名称 [模板配置前缀] (dataset).
:param field_name: 数据集字段 [模板配置后缀] .(company_name)
:param padding_data: 可用于填充的数据
:return: 返回runs中填充label的定位下标列表 [供给特定style使用]
"""
if type(padding_data) is int:
padding_data = str(padding_data)
begin, tmp = -100, ''
template_label = dataset_name + '.' + field_name # 模板标签全称[用于二次确认+替换]
pattern_mustache = re.compile(r'{{.*?}}')
if len(dataset_name) > 0 and dataset_name[0] == '^':
pattern_template_label = '\^' + dataset_name[1:] + '\.' + field_name
else:
pattern_template_label = dataset_name + '\.' + field_name
pattern_label = re.compile(r'' + '{{{{\s*{}\s*}}}}'.format(pattern_template_label))
list_padding_index = []
for index, run in enumerate(runs):
'''
重难点
有标题格式 开头出现的 {{ 会分为两个run
如果后续支持计算，需修改此处代码
这里为何用if 而不是 elif
run.text = '}}{{' 连一起这种情况
'''
if '{{' in run.text and begin == -100:
begin, tmp = index, run.text
elif '{' in run.text and begin == -100:
begin, tmp = index, run.text
else:
tmp += run.text
if pattern_mustache.findall(tmp):
if template_label in tmp:
# 如果存在匹配的字符串，那么将当前的run替换成合并后得字符串tmp
run.text = re.sub(pattern_label, padding_data, tmp)
'''
len(runs) = 1
runs[0].text = '{{ base_info.risk_title }}'
或者 runs[i].text = ' }}' runs[i + 1].text = '{{ dataset_name.filed_name }}'
此时会出现 begin = -100, end_index = 0
IndexError: list index out of range
'''
if begin != -100:
for i in range(begin, index):
runs[i].text = ''
list_padding_index.append(index)
begin, tmp = -100, ''
elif '{{' in run.text:
begin, tmp = index, run.text
elif '{' in run.text:
begin, tmp = index, run.text
else:
begin, tmp = -100, ''
return list_padding_index
def _padding_table(
self, table, dataset_name: str,
table_dict_label_index: dict,
table_dict_padding_data: dict
) -> None:
for _label in table_dict_label_index:
if _label in table_dict_padding_data:
for index in table_dict_label_index[_label]:
paragraphs = table._cells[index].paragraphs
for para in paragraphs:
self._padding_runs(
runs=para.runs, dataset_name=dataset_name, field_name=_label,
padding_data=table_dict_padding_data[_label]
)
def _delete_annotation(self) -> None:
"""
删除模板中注释: {{ !xxx }}
:return: None
"""
for index, para in enumerate(self._paragraphs):
set_result = self._regular_extract.match_pattern(para_text=para.text, patterns=['(?<={{).*?(?=}})'])
for result in set_result:
if len(result) > 0 and result[0] == '!':
self._delete_paragraph(para=para)
pass
def _delete_paragraph(self, para) -> None:
"""
删除word文档某段落
:param para:
:return: None
"""
p = para._element
p.getparent().remove(p)
para._p = para._element = None
self._build()
def _save(self) -> None:
"""
将word文档保存至本地
:return:
"""
# self._document.styles['Normal']._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# self._document.styles['Normal'].font.name = 'Times New Roman'
self._delete_annotation()
self._document.save(self._output_report_path)
def _add_para(self, para_before, para) -> None:
"""
新插入段落 [已弃用]
:param para_before:
:param para:
:return:
"""
# 插入一个段落，代码换行，word中就是一个段落，不换行
new_para = para_before.insert_paragraph_before()
self._add_runs(new_para=new_para, para=para)
""" Paragraph 段落格式设置 """
# 设置段落文本右对齐
new_para.paragraph_format.alignment = para.paragraph_format.alignment
""" 段落缩进 """
# 左缩进
new_para.paragraph_format.left_indent = para.paragraph_format.left_indent
# 右缩进
new_para.paragraph_format.right_indent = para.paragraph_format.right_indent
# 首行缩进
new_para.paragraph_format.first_line_indent = para.paragraph_format.first_line_indent
""" 行间距 """
new_para.paragraph_format.line_spacing = para.paragraph_format.line_spacing
"当line_spacing设置为长度值时表示绝对距离，"
"设置为浮点数时表示行高的倍数"
# 段前间距
new_para.paragraph_format.space_before = para.paragraph_format.space_before
# 段后间距
new_para.paragraph_format.space_after = para.paragraph_format.space_after
""" 设置段落内部文字在遇到需分页情况时处理状态 """
new_para.paragraph_format.keep_together = para.paragraph_format.keep_together # 段中不分页
new_para.paragraph_format.keep_with_next = para.paragraph_format.keep_with_next # 与下段同页
new_para.paragraph_format.page_break_before = para.paragraph_format.page_break_before # 段前分页
new_para.paragraph_format.widow_control = para.paragraph_format.widow_control # 孤行控制
# 获取段落的左缩进，首行缩进，段前间距：
l_space = new_para.paragraph_format.left_indent
h_space = new_para.paragraph_format.first_line_indent
b_space = new_para.paragraph_format.space_before
# print(l_space, h_space, b_space) # 457200 914400 63500
self._build()
def _add_runs(self, new_para, para) -> None:
"""
新建段落[由runs组成] 新建run [已弃用]
:param new_para:
:param para:
:return:
"""
for run in para.runs:
# 新建一个段落，增加一段文字
new_run = new_para.add_run(run.text)
new_run.bold = run.bold
new_run.element = run.element
""" 设置字体格式 """
# new_run.font.name = run.font.name # 注：这个好像设置 run 中的西文字体
# new_run.font.element = run.font.element
# new_run._element.rPr.rFonts = run._element.rPr.rFonts
# new_run._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
new_run.font.name = 'Times New Roman' # 注：这个好像设置 run 中的西文字体
# new_run._element.rPr.rFonts.set(qn('w:eastAsia'), '微软雅黑')
new_run._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# new_run.style._element.rPr.rFonts.set(qn('w:eastAsia'), u'宋体')
# new_run.style._element.rPr.rFonts.set(qn('w:ascii'), 'Times New Roman')
# print(new_run.font.name)
# 设置中文字体
# new_run.font.element.rPr.rFonts.set(qn('w:eastAsia'), '宋体')
# 设置字体大小
new_run.font.size = run.font.size
# 设置加粗
new_run.font.bold = run.font.bold
# 设置斜体
new_run.font.italic = run.font.italic
# 设置字体颜色
new_run.font.color.rgb = run.font.color.rgb
# 设置背景颜色
new_run.font.highlight_color = run.font.highlight_color
# 设置下划线
new_run.font.underline = run.font.underline
# 设置轮廓线
new_run.font.outline = run.font.outline
# 设置阴影
new_run.font.shadow = run.font.shadow
# 删除线
new_run.font.strike = run.font.strike
# 双删除线
new_run.font.double_strike = run.font.double_strike
# 设置下标
new_run.font.subscript = run.font.subscript
# 设置上标
new_run.font.superscript = run.font.superscript
new_run.italic = run.italic
new_run.style.name = run.style.name
new_run.underline = run.underline
# def pl_process_pro(task_id):
# project_name = "平台报告"
# report_name = "平台模板样例报告.docx"
# # template_dir, template_name = os.path.split(template_path)
# # todo: 先基于任务id获取数据集信息 包括数据集id， 参数值， 模板地址
# # dataset_sql = '''SELECT ds.id,ds.param_value,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
# # inner join clb_report_data_set ds on te.data_set_id = ds.id
# # where t.id = {};'''.format(task_id)
# dataset_sql = """SELECT ds.id,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
# inner join clb_report_data_set ds on te.data_set_id = ds.id
# where t.id = {};""".format(task_id)
#
# dataset_result = dbm.query(sql=dataset_sql)[0]
# dataset_id, temp_path = dataset_result["id"], dataset_result["file_path"]
# # dataset_id, param_value, temp_path = dataset_result["id"], dataset_result["param_value"], dataset_result["file_path"]
# # print(type(param_value))
# # param_value = json.loads(param_value)
# # todo: 再基于数据集id 获取数据集地址，参数，返回数据类型，数据对象
# # data_source_sql = """select ds.url,ds.params,ds.type,ds.data_name from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
# # where m.data_set_id = {};""".format(dataset_id)
#
# data_source_sql = """select ds.url,m.param_value,ds.type,m.return_object from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
# where m.data_set_id = {};""".format(dataset_id)
#
# datasource_result_list = dbm.query(sql=data_source_sql)
# # 关闭数据库连接
# dbm.close()
# # todo: 第一步：基于模板路径和模板url 获取下载模板链接
# template_request = temp_url + "/" + temp_path
# # 先判断是否是docx 格式
# template_filename = template_request.split("/")[-1]
# # 文件先下载再判断是否转换
# if ".doc" in template_request:
# r = requests.get(template_request, stream=True)
# with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
# for chunk in r.iter_content(chunk_size=512):
# f.write(chunk)
#
# temp_template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# # 获取文件路径前缀
# template_path = os.path.splitext(temp_template_path)[0] + '.docx'
# # 将doc转换为docx
# # doc2docx(temp_template_path, template_path)
#
# elif ".docx" in template_request:
# r = requests.get(template_request, stream=True)
# with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
# for chunk in r.iter_content(chunk_size=512):
# f.write(chunk)
# template_path = os.path.join(UPLOAD_FOLDER, template_filename)
#
# # todo: 第二步：基于数据源信息获取数据对象
# dict_data = dict()
# for datasource_result in datasource_result_list:
# dataset_url = datasource_result["url"]
# params = datasource_result["param_value"]
# # 4月23号调整
# dict_param = json.loads(params)
# # {"id": 2, "name": "nihao"}
# connect_list = []
# for key, value in dict_param.items():
# connect_param = key + "=" + value
# connect_list.append(connect_param)
#
# # connect_list = []
# # for temp_param in params_list:
# # connect_param = temp_param + "=" + param_value[temp_param]
# # connect_list.append(connect_param)
# request_str = "&".join(connect_list)
# dataset_request = dataset_url + "?" + request_str
# str_dataset_info = requests.get(dataset_request, stream=True)
# # logger.info(str_dataset_info.content)
# dataset_info = json.loads(str_dataset_info.content)
# data_name = datasource_result["return_object"]
# dict_data[data_name] = dataset_info["result"]
# logger.info(dict_data)
# template_dir, template_name = os.path.split(template_path)
# gurag = GeneralUserReportAutoGenerated(
# project_name=project_name,
# input_template_path=template_path,
# output_report_path=os.path.join(template_dir, report_name),
# start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
# )
# gurag.process(dict_data)
# return None
def pl_process(template_path, dict_data, report_name):
project_name = "平台报告"
template_dir, template_name = os.path.split(template_path)
gurag = GeneralUserReportAutoGenerated(
project_name=project_name,
input_template_path=template_path,
output_report_path=os.path.join(template_dir, report_name),
start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
)
gurag.process(dict_data)
return None
def new_pl_process_pro(task_id):
project_name = "平台报告"
report_name = "平台模板样例报告.docx"
# template_dir, template_name = os.path.split(template_path)
# todo: 先基于任务id获取数据集信息 包括数据集id， 参数值， 模板地址
# dataset_sql = '''SELECT ds.id,ds.param_value,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
# inner join clb_report_data_set ds on te.data_set_id = ds.id
# where t.id = {};'''.format(task_id)
dataset_sql = """SELECT ds.id,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
inner join clb_report_data_set ds on te.data_set_id = ds.id
where t.id = {};""".format(task_id)
dataset_result = dbm.query(sql=dataset_sql)[0]
dataset_id, temp_path = dataset_result["id"], dataset_result["file_path"]
# dataset_id, param_value, temp_path = dataset_result["id"], dataset_result["param_value"], dataset_result["file_path"]
# print(type(param_value))
# param_value = json.loads(param_value)
# todo: 再基于数据集id 获取数据集地址，参数，返回数据类型，数据对象
# data_source_sql = """select ds.url,ds.params,ds.type,ds.data_name from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
# where m.data_set_id = {};""".format(dataset_id)
data_source_sql = """select ds.url,m.param_value,ds.type,m.return_object from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
where m.data_set_id = {};""".format(dataset_id)
datasource_result_list = dbm.query(sql=data_source_sql)
# 关闭数据库连接
dbm.close()
# todo: 第一步：基于模板路径和模板url 获取下载模板链接
template_request = temp_url + "/" + temp_path
# 先判断是否是docx 格式
template_filename = template_request.split("/")[-1]
# 文件先下载再判断是否转换
if ".doc" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
temp_template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# 获取文件路径前缀
template_path = os.path.splitext(temp_template_path)[0] + '.docx'
# 将doc转换为docx
# doc2docx(temp_template_path, template_path)
elif ".docx" in template_request:
r = requests.get(template_request, stream=True)
with open(os.path.join(UPLOAD_FOLDER, template_filename), "wb") as f:
for chunk in r.iter_content(chunk_size=512):
f.write(chunk)
template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# todo: 第二步：基于数据源信息获取数据对象
dict_data = dict()
for datasource_result in datasource_result_list:
dataset_url = datasource_result["url"]
params = datasource_result["param_value"]
print(params)
# 4月23号调整
dict_param = json.loads(params)
# {"id": 2, "name": "nihao"}
connect_list = []
for key, value in dict_param.items():
connect_param = key + "=" + value
connect_list.append(connect_param)
# connect_list = []
# for temp_param in params_list:
# connect_param = temp_param + "=" + param_value[temp_param]
# connect_list.append(connect_param)
request_str = "&".join(connect_list)
dataset_request = dataset_url + "?" + request_str
str_dataset_info = requests.get(dataset_request, stream=True)
# logger.info(str_dataset_info.content)
dataset_info = json.loads(str_dataset_info.content)
data_name = datasource_result["return_object"]
dict_data[data_name] = dataset_info["result"]
logger.info(dict_data)
template_dir, template_name = os.path.split(template_path)
gurag = GeneralUserReportAutoGenerated(
project_name=project_name,
input_template_path=template_path,
output_report_path=os.path.join(template_dir, report_name),
start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
)
gurag.process(dict_data)
return None
# print(pl_process(task_id="1641261934625521666"))
if __name__ == "__main__":
new_pl_process_pro("1650786947993526274")
# import os
# project_name = "平台报告"
# # save_path = "../data/new_平台模板定义样例.docx"
# save_path = "../data/4月25日周报模板(2)(2).docx"
# template_dir, template_name = os.path.split(save_path)
# report_name = "4月25平台模板样例报告.docx"
# gurag = GeneralUserReportAutoGenerated(
# project_name=project_name,
# input_template_path=save_path,
# output_report_path=os.path.join(template_dir, report_name),
# start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
# )
# data_result = {'info': {'year': 2023, 'num': '3', 'beginDate': '2023年04月17日', 'endDate': '2023年04月23日'},
# 'zhongyaojianghua': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# 'zhongdahuiyi': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# 'diyiyiti': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# 'qqjjfzqs': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# 'cyzcjzx': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# 'zhongdashijian': [{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'},{'title': "11", 'summary': '22', 'origin': '33', 'publishDate': '2023年04月23日'}],
# }
# # data_result = {'originDateRatio': [{'date': '2023-02', '网易新闻': 551, '新浪财经': 145, '占比': '26.32%'}, {'date': '2023-03', '网易新闻': 68, '新浪财经': 83, '占比': '122.06%'}], 'originDate': [{'name': '澎湃新闻', '2023-02': 100, '2023-03': 39}, {'name': '光明网', '2023-02': 78, '2023-03': 14}, {'name': '网易新闻', '2023-02': 551, '2023-03': 68}, {'name': '新浪财经', '2023-02': 145, '2023-03': 83}, {'name': '新浪新闻', '2023-02': 112, '2023-03': 72}, {'name': '国际在线', '2023-02': 50, '2023-03': 25}, {'name': '手机光明网', '2023-02': 73, '2023-03': 25}, {'name': '腾讯新闻', '2023-02': 54, '2023-03': 16}, {'name': '环球网', '2023-02': 85, '2023-03': 35}, {'name': '网易', '2023-02': 326, '2023-03': 159}], 'origin': [{'name': '网易新闻', 'value': 619}, {'name': '网易', 'value': 485}, {'name': '新浪财经', 'value': 228}, {'name': '新浪新闻', 'value': 184}, {'name': '澎湃新闻', 'value': 139}, {'name': '环球网', 'value': 120}, {'name': '手机光明网', 'value': 98}, {'name': '光明网', 'value': 92}, {'name': '国际在线', 'value': 75}, {'name': '腾讯新闻', 'value': 70}], 'newList': [{'title': '联合国安理会就叙利亚局势举行公开会', 'summary': '当地时间3月23日，联合国安理会就叙利亚局势举行定期公开会。中方代表表示对阿勒颇机场再度遭袭感到关切，并呼吁有关国家取消所有非法单边制裁。', 'origin': '网易', 'publishDate': '2023-03-24 10:50:11'}, {'title': '总台记者探访丨强震后土耳其货币贬值 加剧药物短缺困境', 'summary': '当地时间2月6日土耳其南部发生强震。由于投资者担心强震对经济的影响，近期，土耳其里拉持续贬值，里拉对美元汇率跌破1美元兑换19里拉关口，创下历史新低。近日，中央广播电视总台记者探访后发现，里拉持续贬值，给多行业带来负面连锁反应，其中，本已非常严重的药物短缺问题也因为里拉贬值进一步加剧。', 'origin': '荆楚网', 'publishDate': '2023-03-24 08:43:00'}, {'title': '外媒:沙特与叙利亚就重开两国大使馆达成一致', 'summary': '央视新闻客户端消息，据路透社3月23日报道，沙特阿拉伯与叙利亚近期已就重新开放两国大使馆达成一致。报道称，两国使馆预计将于4月下旬恢复工作，而这一行动也将为叙利亚重返阿拉伯国家联盟“铺平道路”。', 'origin': '上游新闻', 'publishDate': '2023-03-24 08:23:18'}, {'title': '强震后土耳其农业生产遭重创 农户无力恢复生产', 'summary': '东北新闻网微博\n 北斗融媒\n *本网站有关内容转载自合法授权网站，如果您认为转载内容侵犯了您的权益，\n 请您来信来电(024-23187042)声明，本网站将在收到信息核实后24小时内删除相关内容。', 'origin': '东北新闻网', 'publishDate': '2023-03-24 07:52:01'}, {'title': '中方敦促消除对叙利亚人道救援的障碍', 'summary': '新华社联合国3月23日电 中国常驻联合国副代表耿爽23日在安理会叙利亚政治人道问题公开会上发言时指出，进一步改善对叙人道救援工作，需要克服三方面的障碍。', 'origin': '新京报网', 'publishDate': '2023-03-24 07:02:00'}, {'title': '中方敦促消除对叙利亚人道救援的障碍', 'summary': '新华社联合国3月23日电 中国常驻联合国副代表耿爽23日在安理会叙利亚政治人道问题公开会上发言时指出，进一步改善对叙人道救援工作，需要克服三方面的障碍。', 'origin': '手机光明网', 'publishDate': '2023-03-24 06:59:00'}, {'title': '外媒:沙特与叙利亚就重开两国大使馆达成一致', 'summary': '每经AI快讯，据路透社3月23日报道，沙特阿拉伯与叙利亚近期已就重新开放两国大使馆达成一致。报道称，两国使馆预计将于4月下旬恢复工作，而这一行动也将为叙利亚重返阿拉伯国家联盟“铺平道路”。', 'origin': '每日经济新闻', 'publishDate': '2023-03-24 05:12:35'}, {'title': '中方敦促消除对叙利亚人道救援的障碍-中新网', 'summary': '新华社联合国3月23日电 中国常驻联合国副代表耿爽23日在安理会叙利亚政治人道问题公开会上发言时指出，进一步改善对叙人道救援工作，需要克服三方面的障碍。', 'origin': '中国新闻网', 'publishDate': '2023-03-24 03:07:57'}, {'title': '强震后土耳其制造业损失惨重,恢复生产困难重重', 'summary': '近日，记者探访了土耳其强震灾区的一个制鞋业中心，发现强震后当地大量厂房坍塌、生产设备受损严重，大批劳动力流失，目前恢复生产面临重重困难。', 'origin': '澎湃新闻', 'publishDate': '2023-03-23 22:35:00'}, {'title': '程昊:我是记者,也是名救援队员_中安在线', 'summary': '合肥—武汉—土耳其—合肥，7天7夜，安徽新媒体集团记者、合肥市蓝天救援队队员程昊在土耳其完成了自己的很多的第一次。第一次跨出国门，第一次在异国他乡升起无人机。', 'origin': '中安在线网站', 'publishDate': '2023-03-23 21:07:00'}], 'info': {'total': 4295, 'createTime': '2023年02月25日', 'name': '叙利亚地震'}}
#
# gurag.process(data_result)
generate/process.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : process
# @Author : LiuYan
# @Time : 2021/11/9 16:06
import chevron
from generate.gen_user_report_auto_generated import GeneralUserReportAutoGenerated
if __name__ == '__main__':
# from docxtpl import DocxTemplate
#
# doc = DocxTemplate("template.docx") # 模板文档
# context = {'app_name': "模块测试", 'dataset_1_company_name': "2020-07-01", 'vac_time': "8:30", "reason": "上班未打卡"} # 待替换对象
# doc.render(context) # 执行替换
# doc.save("generated_doc.docx") # 保存新的文档
# from jinja2 import Template
#
# person = {'name': 'Person', 'age': 34}
#
# tm = Template("My name is {{ per.name }} and I am {{ per.age }}, {{ per.test }} ")
# # tm = Template("My name is {{ per['name'] }} and I am {{ per['age'] }}")
# msg = tm.render(per=person)
#
# print(msg)
# s = chevron.render('Hello, {{ dataset_1_Company_Name }}!', {'dataset_1.Company_Name': 'World'})
# s = chevron.render(
# 'start_date={{start_time}}&end_date={{end_time}}&top_n={{top_n}}',
# {'start_time': 'st', 'end_time': 'et'}
# )
# project_name = '研究中心'
# input_template_path = '../data/datasource/input/template.docx'
# output_report_path = '../data/datasource/output/企业分析报告_20211223.docx'
project_name = '评价中心'
input_template_path = '../data/datasource/input/template_kaige.docx'
output_report_path = '../data/datasource/output/template_kaige_2048.docx'
gurag = GeneralUserReportAutoGenerated(
project_name=project_name,
input_template_path=input_template_path,
output_report_path=output_report_path,
start_time='2022-01-10', end_time='2022-01-17',
parameter='reportId=2048'
)
gurag.process()
generate/regular_extract.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : regular_extract
# @Author : LiuYan
# @Time : 2021/12/7 18:01
import re
class RegularExtract:
def match_pattern(self, para_text: str, patterns: list) -> set:
"""
提取符合规则的字符串
:param para_text: 待匹配字符串
:param patterns: 正则列表
:return:
"""
set_match = set()
for pattern_str in patterns:
pattern = re.compile(r'' + pattern_str)
# 找到与之匹配的所有子串，并以迭代器形式返回，与findall 类似
results = re.finditer(pattern, para_text)
for result in results:
# 带索引位置
# print(result)
result = result.group().strip()
# 不带索引位置信息
# print(result)
set_match.add(result)
return set_match
def match_index(self, para_text: str, pattern_str: str) -> list:
"""
提取符合规则的字符串开始结束位置列表
:param para_text: 待匹配字符串
:param pattern_str: 正则字符串
:return:
"""
list_index = []
pattern = re.compile(r'' + pattern_str)
results = re.finditer(pattern, para_text)
for result in results:
start_index, end_index = result.span()
list_index.append(
{
'start_index': start_index,
'end_index': end_index
}
)
return list_index
generate/snapshot.py 0 → 100644
import os
import time
from typing import Any
from selenium import webdriver
SNAPSHOT_JS = """
var ele = document.querySelector('div[_echarts_instance_]');
var mychart = echarts.getInstanceByDom(ele);
return mychart.getDataURL({
type: '%s',
pixelRatio: %s,
excludeComponents: ['toolbox']
});
"""
SNAPSHOT_SVG_JS = """
var element = document.querySelector('div[_echarts_instance_] div');
return element.innerHTML;
"""
def make_snapshot(
html_path: str,
file_type: str,
pixel_ratio: int = 2,
delay: int = 2,
browser="Chrome",
driver: Any = None,
):
if delay < 0:
raise Exception("Time travel is not possible")
if not driver:
if browser == "Chrome":
driver = get_chrome_driver()
elif browser == "Safari":
driver = get_safari_driver()
else:
raise Exception("Unknown browser!")
if file_type == "svg":
snapshot_js = SNAPSHOT_SVG_JS
else:
snapshot_js = SNAPSHOT_JS % (file_type, pixel_ratio)
if not html_path.startswith("http"):
html_path = "file://" + os.path.abspath(html_path)
driver.get(html_path)
time.sleep(delay)
return driver.execute_script(snapshot_js)
# def get_chrome_driver():
# options = webdriver.ChromeOptions()
# options.add_argument("headless")
# return webdriver.Chrome(options=options)
def get_chrome_driver():
options = webdriver.ChromeOptions()
options.add_argument("headless")
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
return webdriver.Chrome(options=options)
def get_safari_driver():
return webdriver.Safari()
generate/tools.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : utils
# @Author : LiuYan
# @Time : 2021/11/9 16:50
from docx.shared import Pt
from docx.shared import RGBColor
from docx.oxml.ns import qn
def paragraph_attribute(pa, size, family, r=0x00, g=0x00, b=0x00, bold=False):
pa.font.size = Pt(size)
pa.font.name = family
pa.font.bold = bold
pa.font.color.rgb = RGBColor(r, g, b)
p = pa._element.rPr.rFonts.set(qn('w:eastAsia'), family)
return p
def judge_str_pre_suf(s1: str, s2: str) -> bool:
"""
比较字符串s1后缀(suffix)与字符串s2前缀(prefix)是否有重叠
有重叠 -> True
无重叠 -> False
:param s1: str
:param s2: str
:return: bool
"""
m = min(len(s1), len(s2))
for i in range(1, m + 1):
print(i)
if s1[-i:] == s2[: i]:
return True
return False
if __name__ == '__main__':
s1 = 'company&'
b = judge_str_pre_suf(s1='company&', s2='&company_title&')
print(b)
print(s1[-1:])
generate_report.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/3/23 09:43
# @Author : bruxellse_li
# @File : generate_report.py
# @Project : 从word中提取指定表格
from docx import Document
import os, time
from pathlib import Path
import requests
from flask import request
from flask import Flask, send_file
# from transform_doc_to_docx import doc2docx, closesoft
import subprocess
from generate.gen_user_report_auto_generated import main_process
UPLOAD_FOLDER = r'data' # 上传路径
Path(UPLOAD_FOLDER).mkdir(parents=True, exist_ok=True)
abs_path = os.path.split(os.path.realpath(__file__))[0]
def doc2docx(doc_path, docx_path):
# 使用LibreOffice将doc文件转换为docx文件
subprocess.call(['libreoffice', '--headless', '--convert-to', 'docx', doc_path, '--outdir', os.path.dirname(docx_path)])
# 将转换后的docx文件重命名为目标文件名
os.rename(os.path.splitext(doc_path)[0] + '.docx', docx_path)
def generate_report(template_path, document_path, report_name, object):
"""
template_path : 模板文件下载地址
document_path: 半成品文件下载地址
report_name: 报告名称
data_object: 待填充数据
:return:
"""
template_request = template_path
doc_request = document_path
report_name = report_name + ".docx"
data_object = object["data_object"]
tables_dict = object["tables_dict"]
current_filename = time.strftime('%Y_%m_%d-%H_%M_%S') + ".docx"
save_path = UPLOAD_FOLDER + "/" + current_filename
# 先判断是否是docx 格式
template_filename = template_request.split("/")[-1]
if ".doc" in template_request:
temp_template_path = os.path.join(UPLOAD_FOLDER, template_filename)
# 获取文件路径前缀
template_path = os.path.splitext(temp_template_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(temp_template_path, template_path)
elif ".docx" in template_request:
template_path = os.path.join(UPLOAD_FOLDER, template_filename)
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
doc_filename = doc_request.split("/")[-1]
if ".doc" in doc_request:
temp_doc_path = os.path.join(UPLOAD_FOLDER, doc_filename)
# 获取文件路径前缀
doc_path = os.path.splitext(temp_doc_path)[0] + '.docx'
# 将doc转换为docx
doc2docx(temp_doc_path, doc_path)
half_work_path = doc_path
elif ".docx" in doc_request:
half_work_path = os.path.join(UPLOAD_FOLDER, template_filename)
else:
return "上传文件格式有误，当前仅支持doc 和 docx 格式，请选择正确文件重新上传！"
main_process(half_work_path, tables_dict, template_path, report_name, data_object, save_path)
# send_path = os.path.join(UPLOAD_FOLDER, report_name)
# return send_file(send_path, as_attachment=True)
if __name__ == "__main__":
template_path = "data/wKjIbGQb3gaARMRCAACAACgxnK856.docx"
document_path = "data/wKjIbGQb66OAH-8eAAQAAJHFYeM24.docx"
report_name = "财务报告"
object = {
"tables_dict": {
"table13": "（2）以名义金额计量的资产名称、数量等情况，以及以名义金额计量理由的说明。",
"table5": "收入费用表（2）",
"table4": "收入费用表（1）",
"table3": "资产负债表续表2",
"table2": "资产负债表续表1",
"table1": "资产负债表",
"table9": "（17）其他应付款明细信息如下：",
"table8": "（9）无形资产明细信息如下：",
"table10": "（24）其他收入明细信息如下：",
"table7": "（7）固定资产明细信息如下：",
"table11": "（25）业务活动费用明细信息如下：",
"table6": "（1）货币资金明细信息如下：",
"table12": "（28）商品和服务费用明细信息如下："
},
"data_object": {
"负债占比": [],
"费用占比": [],
"流动资产占比": [],
"流动负债占比": [],
"收入占比": [],
"finance": {
"currentLiabilitiesCompose": "",
"cashRatio": "",
"revenueExpensesRatio": "",
"currentRatioRemark": "说明本单位流动资产偿还短期债务的能力弱",
"beInDebtChangeRatioRemark": "",
"assetLiabilityRatioRemark": "说明本单位财务风险低",
"totalExpensesCompose": "",
"debtComposition": "流动负债占""，非流动负债占""",
"totalRevenueCompose": "",
"totalExpensesChangeRatioRemark": "",
"composition": "流动资产占比，非流动资产占比",
"unitDebtComposition": "流动负债和非流动负债",
"cashRatioRemark": "说明本单位利用现金和现金等价物偿还短期债务的能力弱",
"affordableHouseNewRation": "",
"totalAssetsChangeRatioRemark": "",
"currentRatio": "",
"currentAssetsCompose": "",
"publicInfrastructureNewRatio": "",
"totalRevenueComposeDetail": "",
"assetLiabilityRatio": "",
"unitAssetComposition": "流动资产和非流动资产",
"fixedAssetsDepreciationRatio": "",
"revenueExpensesRatioRemark": "大于",
"totalRevenueChangeRatioRemark": "",
"otherRemark": "",
"nonCurrentAssetsCompose": ""
},
"资产占比": [],
"info": {
"internalControl": "2021年，本单位加强学习国家和省关于内部控制的文件。建立健全了单位层面的内部控制体系和制度，健全了预算、收支、采购、建设、资产和合同的内控流程和制度，把内部控制落实在业务流程中，实现了不相容岗位相互分离、形成相互制约、相互监督的工作机制；实现了内部授权审批控制。",
"unitName": "安丘速度单位",
"unitCall": "本部门",
"mainFunctions": "无资料数据",
"year": "2023",
"unitBudgetLevel": "",
"institutionalSituation": "无资料数据",
"performanceManagement": "2021年，本单位按照绩效管理要求对照设定预算绩效目标、绩效指标的成本指标、产出指标、效益指标、满意度指标等具体内容，开展项目绩效目标申报、运行监控和自评工作。通过预算绩效管理对工作中存在的薄弱环节作出针对性查漏补缺和持续完善。",
"LastYear": "2022",
"personnelSituation": "无资料数据",
"unitType": "",
"budgetManagement": "2021年，本单位严格按照《预算法》、《会计法》、《政府会计制度》和上级的文件建立健全财务制度；严格执行财经纪律和各项财务制度；强化预算管理，加强对银行存款和现金的管理；单位对年终决算高度重视，组织专人负责编制决算报告，对决算数据进行了严格审核，认真分析并应用到下年的预算工作。",
"assetManagement": "2021年，本单位资产实行分类管理，建立健全了资产内部管理制度；单位加强对实物资产和无形资产的管理，明确相关部门和岗位的职责权限，强化对配置、使用和处置等关键环节的管控；明确资产使用和保管责任人，落实资产使用人在资产管理中的责任。",
"pppProject": "本单位无PPP项目。",
"careerAchievements": "无资料数据"
},
"非流动资产占比": []
}
}
generate_report(template_path, document_path, report_name, object)
lac_method.py 0 → 100644
#!/user/bin/env python
# coding=utf-8
"""
@project : 500_资讯
@author : bruxelles_li
@file : lac_ner_text.py
@ide : PyCharm
@time : 2022-07-04 09:19:43
"""
from LAC import LAC
import pandas as pd
import tqdm
import re
lac = LAC(mode="lac")
# text_path = "领导讲话_1370.xlsx"
# 句子提取人名
def lac_username(sentences):
# 装载LAC模型
user_name_list = []
lac = LAC(mode="lac")
lac_result = lac.run(sentences)
# print(lac_result)
for index, lac_label in enumerate(lac_result[1]):
if lac_label == "PER":
user_name_list.append(lac_result[0][index])
# print(user_name_list)
# print(user_name_list)
return user_name_list
# 句子提取机构名
def lac_organize_name(sentences):
# 装载LAC模型
user_name_list = []
lac = LAC(mode="lac")
lac_result = lac.run(sentences)
# print(lac_result)
for index, lac_label in enumerate(lac_result[1]):
if lac_label == "ORG":
user_name_list.append(lac_result[0][index])
return user_name_list
# 句子提取地名
def lac_location_name(sentences):
# 装载LAC模型
user_name_list = []
lac = LAC(mode="lac")
lac_result = lac.run(sentences)
# print(lac_result)
for index, lac_label in enumerate(lac_result[1]):
if lac_label == "LOC":
user_name_list.append(lac_result[0][index])
return user_name_list
def match_text_one(rule, text):
# rule = ";".join(new_one)
# print(rule)
# text_one = match_text_one(rule, title)
# print(text_one)
rules = '|'.join(rule.split(';')).strip('\n')
replaced_rules = rules.replace('.', '\.')\
.replace('*', '\*')\
.replace('(', '\(')\
.replace(')', '\)')\
.replace('+', '.+')
pattern = re.compile(r'' + replaced_rules)
print(pattern)
match_result = re.sub(pattern, "A", text)
print(match_result)
return match_result
if __name__ == '__main__':
print(lac_organize_name("广元市朝天区曾家镇人民政府辖11个村1社区78个村民小组2个居民小组，全镇4784户16355人。财政供养人口67人，其中行政人员25人，事业人员39人，机关工勤3人。相比上年增加6人，增加的原因是工作变动人员正常调出。"))
# print(lac_username(sentences="习近平指出阿里分析师研报，权威，专业，及时，全面，助您挖掘潜力主题机会！ 【概述】泡财经获悉，4月29日晚间，上汽集团（600104.SH）公告称，一季度净利润55.16亿元，同比下降19.44%。"))
# data_df = pd.read_excel(text_path, nrows=1).astype(str)
# result_list = []
# for idx, row in tqdm.tqdm(data_df.iterrows()):
# title = row['title']
# a_user = lac_username(title)
# a_organize = lac_organize_name(title)
# a_location = lac_location_name(title)
# if a_user:
# user_rule = '|'.join(a_user).strip()
# pattern0 = re.compile(r'' + user_rule)
# result_one = re.sub(pattern0, 'A', title)
# title = result_one
# if a_organize:
# a_organize_rule = '|'.join(a_organize).strip()
# pattern1 = re.compile(r'' + a_organize_rule)
# result_two = re.sub(pattern1, 'B', result_one)
# title = result_two
# if a_location:
# a_location_rule = '|'.join(a_location).strip()
# pattern2 = re.compile(r'' + a_location_rule)
# print(pattern2)
# result_three = re.sub(pattern2, 'C', result_two)
# print(result_three)
# title = result_three
#
# row['title'] = title
# result_list.append(row)
# print(result_list)
#
# # new_one = a_user + a_organize + a_location
# # rule = "|".join(new_one)
# # pattern = re.compile(r'' + rule)
# # result_one = re.sub(pattern, "A", title)
# # title = result_one
# # print(title)
requirements.txt 0 → 100644
# python 3.8.5
beautifulsoup4==4.12.2
chevron==0.14.0
cx_Oracle==8.3.0
docx==0.2.4
fdfs_client==4.0.7
flask_cors==3.0.10
flask_sqlalchemy==3.0.3
LAC==2.1.2
matplotlib==3.5.1
numpy==1.21.5
pandas==1.4.2
pyecharts==2.0.2
pymysql==1.0.3
requests==2.27.1
selenium==4.8.3
snapshot_selenium==0.0.2
tqdm==4.64.0
gunicorn
Flask
python-docx
xlsxwriter
scbg_app_config.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : app_config
# @Author : LiuYan&bruxelles_li
# @Time : 2021/4/22 10:51
import os
import multiprocessing
from pathlib import Path
bind = '0.0.0.0:4000' # 绑定ip和端口号
backlog = 512 # 监听队列
# chdir = '/home/zzsn/liuyan/bin' # gunicorn要切换到的目的工作目录
timeout = 300 # 超时 -> 目前为迎合ZZSN_NLP平台 一带一路要素抽取(文件)需求 暂时关闭超时
# worker_class = 'gevent' # 使用gevent模式，还可以使用sync 模式，默认的是sync模式
# workers = multiprocessing.cpu_count() # 进程数 12
workers = 3 # 低资源 13G 服务器负载过大可调整此处为 1
threads = 50 # 指定每个进程开启的线程数
loglevel = 'error' # 日志级别，这个日志级别指的是错误日志的级别，而访问日志的级别无法设置
access_log_format = '%(t)s %(p)s %(h)s "%(r)s" %(s)s %(L)s %(b)s %(f)s" "%(a)s"' # 设置gunicorn访问日志格式，错误日志无法设置
"""
其每个选项的含义如下：
h remote address
l '-'
u currently '-', may be user name in future releases
t date of the request
r status line (e.g. ``GET / HTTP/1.1``)
s status
b response length or '-'
f referer
a user agent
T request time in seconds
D request time in microseconds
L request time in decimal seconds
p process ID
"""
_tmp_path = os.path.dirname(os.path.abspath(__file__))
_tmp_path = os.path.join(_tmp_path, 'log')
Path(_tmp_path).mkdir(parents=True, exist_ok=True)
accesslog = os.path.join(_tmp_path, 'gunicorn_access.log') # 访问日志文件
errorlog = os.path.join(_tmp_path, 'gunicorn_error.log') # 错误日志文件
# gunicorn -c app_config.py app_run:app -D --daemon
sentence_split.py 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
start.sh 0 → 100644
#!/bin/sh
# description: auto_run
# 四川报告生成监控
# 检测脚本是否在运行，若已经在运行，则等待一段时间后再次检查，若未启动则进行启动
function start_interface() {
INTERFACE_IS_STRAT=`ps -ef | grep scbg_app_config.py | grep -v grep | wc -l`
if [ $INTERFACE_IS_STRAT -eq 4 ] ; then
usleep
else
echo "=========Service Will Start=========="
# cd /data/lzc/scbg-python/SCBG-PYTHON && nohup gunicorn -c scbg_app_config.py app_run:app 2>&1 &
cd /opt/SCBG-PYTHON && exec nohup gunicorn -c scbg_app_config.py app_run:app 2>&1 &
echo "=========Service Start Completed!========"
fi
}
# 方法一：使用 echo 命令保持服务运行状态（容器启动时）
# 方法二： 使用自定义接口检测函数来保持服务运行状态
while true
do
echo "PYTHON SERVICE is running..."
start_interface
sleep 30m
done
\ No newline at end of file
static/assets/Forget-legacy.b0807a5d.js 0 → 100644
!function(){function n(n,e,t,r,u,o,a){try{var i=n[o](a),c=i.value}catch(s){return void t(s)}i.done?e(c):Promise.resolve(c).then(r,u)}System.register(["./index-legacy.90883ee8.js","./vendor-legacy.bbc7855f.js","./index-legacy.f25f109a.js"],(function(e){"use strict";var t,r,u,o,a,i,c,s,l,f,d;return{setters:[function(n){t=n.l},function(n){r=n.d,u=n.U,o=n.L,a=n.u,i=n.r,c=n.c,s=n.F,l=n.I,f=n.B,d=n.b},function(){}],execute:function(){e("default",r({name:"app",components:{UserOutlined:u,LockOutlined:o},setup:function(){var e=a(),r=i({username:"",password:"",password2:""}),p=function(n){console.log(n,r)},m=function(n){console.log(n)},v=function(){var u,o=(u=regeneratorRuntime.mark((function n(){return regeneratorRuntime.wrap((function(n){for(;;)switch(n.prev=n.next){case 0:return n.next=2,t(r);case 2:n.sent.isHandleSuccess&&e.replace("/home");case 4:case"end":return n.stop()}}),n)})),function(){var e=this,t=arguments;return new Promise((function(r,o){var a=u.apply(e,t);function i(e){n(a,r,o,i,c,"next",e)}function c(e){n(a,r,o,i,c,"throw",e)}i(void 0)}))});return function(){return o.apply(this,arguments)}}();return function(){return c("div",{class:"login"},[c(s,{layout:"vertical",model:r,onFinish:p,onFinishFailed:m},{default:function(){return[c(s.Item,null,{default:function(){return[c(l,{value:r.username,"onUpdate:value":function(n){return r.username=n},placeholder:"Username"},{prefix:function(){return c(u,{style:"color: rgba(0, 0, 0, 0.25)"},null)}})]}}),c(s.Item,null,{default:function(){return[c(l,{value:r.password,"onUpdate:value":function(n){return r.password=n},type:"password",placeholder:"Password"},{prefix:function(){return c(o,{style:"color: rgba(0, 0, 0, 0.25)"},null)}})]}}),c(s.Item,null,{default:function(){return[c(f,{type:"primary","html-type":"submit",disabled:""===r.username||""===r.password,onClick:v},{default:function(){return[d("Log in")]}})]}})]}})])}}}))}}}))}();
static/assets/Forget.2626476b.js 0 → 100644
import{l as e}from"./index.f608f11e.js";import{d as a,U as s,L as l,u as o,r,c as n,F as t,I as d,B as u,b as p}from"./vendor.98dba853.js";/* empty css */const i=a({name:"app",components:{UserOutlined:s,LockOutlined:l},setup(){const a=o(),i=r({username:"",password:"",password2:""}),m=e=>{console.log(e,i)},c=e=>{console.log(e)},f=async()=>{(await e(i)).isHandleSuccess&&a.replace("/home")};return()=>n("div",{class:"login"},[n(t,{layout:"vertical",model:i,onFinish:m,onFinishFailed:c},{default:()=>[n(t.Item,null,{default:()=>[n(d,{value:i.username,"onUpdate:value":e=>i.username=e,placeholder:"Username"},{prefix:()=>n(s,{style:"color: rgba(0, 0, 0, 0.25)"},null)})]}),n(t.Item,null,{default:()=>[n(d,{value:i.password,"onUpdate:value":e=>i.password=e,type:"password",placeholder:"Password"},{prefix:()=>n(l,{style:"color: rgba(0, 0, 0, 0.25)"},null)})]}),n(t.Item,null,{default:()=>[n(u,{type:"primary","html-type":"submit",disabled:""===i.username||""===i.password,onClick:f},{default:()=>[p("Log in")]})]})]})])}});export{i as default};
static/assets/Login-legacy.972ebfb1.js 0 → 100644
!function(){function n(n,e,t,r,u,o,a){try{var i=n[o](a),c=i.value}catch(l){return void t(l)}i.done?e(c):Promise.resolve(c).then(r,u)}System.register(["./vendor-legacy.bbc7855f.js","./index-legacy.f25f109a.js","./index-legacy.90883ee8.js"],(function(e){"use strict";var t,r,u,o,a,i,c,l,s,f,d,p;return{setters:[function(n){t=n.d,r=n.U,u=n.L,o=n.u,a=n.r,i=n.c,c=n.b,l=n.j,s=n.F,f=n.I,d=n.B},function(){},function(n){p=n.s}],execute:function(){e("default",t({name:"app",components:{UserOutlined:r,LockOutlined:u},setup:function(){o();var e=a({username:"",password:""}),t=function(){var t,r=(t=regeneratorRuntime.mark((function n(){var t;return regeneratorRuntime.wrap((function(n){for(;;)switch(n.prev=n.next){case 0:t={username:e.username,password:e.password},p.dispatch("app/login",t);case 2:case"end":return n.stop()}}),n)})),function(){var e=this,r=arguments;return new Promise((function(u,o){var a=t.apply(e,r);function i(e){n(a,u,o,i,c,"next",e)}function c(e){n(a,u,o,i,c,"throw",e)}i(void 0)}))});return function(){return r.apply(this,arguments)}}();return function(){return i("div",{class:"login"},[i("h1",null,[c("用户登陆"),i(l,{to:"/forget",class:"login-forget"},{default:function(){return[c("找回密码")]}})]),i(s,{layout:"vertical",model:e},{default:function(){return[i(s.Item,null,{default:function(){return[i(f,{value:e.username,"onUpdate:value":function(n){return e.username=n},placeholder:"Username"},{prefix:function(){return i(r,{style:"color: rgba(0, 0, 0, 0.25)"},null)}})]}}),i(s.Item,null,{default:function(){return[i(f,{value:e.password,"onUpdate:value":function(n){return e.password=n},type:"password",placeholder:"Password"},{prefix:function(){return i(u,{style:"color: rgba(0, 0, 0, 0.25)"},null)}})]}}),i(s.Item,null,{default:function(){return[i(d,{block:!0,type:"primary","html-type":"submit",onClick:t},{default:function(){return[c("登 录")]}})]}})]}})])}}}))}}}))}();
static/assets/Login.10fa1365.js 0 → 100644
import{d as e,U as a,L as s,u as l,r as o,c as r,b as t,j as n,F as u,I as d,B as p}from"./vendor.98dba853.js";/* empty css */import{s as m}from"./index.f608f11e.js";const i=e({name:"app",components:{UserOutlined:a,LockOutlined:s},setup(){l();const e=o({username:"",password:""}),i=async()=>{const a={username:e.username,password:e.password};m.dispatch("app/login",a)};return()=>r("div",{class:"login"},[r("h1",null,[t("用户登陆"),r(n,{to:"/forget",class:"login-forget"},{default:()=>[t("找回密码")]})]),r(u,{layout:"vertical",model:e},{default:()=>[r(u.Item,null,{default:()=>[r(d,{value:e.username,"onUpdate:value":a=>e.username=a,placeholder:"Username"},{prefix:()=>r(a,{style:"color: rgba(0, 0, 0, 0.25)"},null)})]}),r(u.Item,null,{default:()=>[r(d,{value:e.password,"onUpdate:value":a=>e.password=a,type:"password",placeholder:"Password"},{prefix:()=>r(s,{style:"color: rgba(0, 0, 0, 0.25)"},null)})]}),r(u.Item,null,{default:()=>[r(p,{block:!0,type:"primary","html-type":"submit",onClick:i},{default:()=>[t("登 录")]})]})]})])}});export{i as default};
static/assets/ProductDataSet-legacy.4637cbae.js 0 → 100644
!function(){function e(e){return function(e){if(Array.isArray(e))return t(e)}(e)||function(e){if("undefined"!=typeof Symbol&&null!=e[Symbol.iterator]||null!=e["@@iterator"])return Array.from(e)}(e)||function(e,a){if(!e)return;if("string"==typeof e)return t(e,a);var n=Object.prototype.toString.call(e).slice(8,-1);"Object"===n&&e.constructor&&(n=e.constructor.name);if("Map"===n||"Set"===n)return Array.from(e);if("Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n))return t(e,a)}(e)||function(){throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}()}function t(e,t){(null==t||t>e.length)&&(t=e.length);for(var a=0,n=new Array(t);a<t;a++)n[a]=e[a];return n}function a(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function n(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}var u=document.createElement("style");u.innerHTML=".dataset .database-config{padding:10px 10px 0 0;border:1px solid #eee;border-radius:4px;margin-left:0;margin-bottom:20px}.dataset .database-config h3{width:100px;text-align:right}.dataset .database-config .ant-form-item{margin-bottom:10px}.dataset.sql{display:flex;flex-direction:row}.dataset.sql .ant-form-item-explain,.dataset.sql .ant-form-item-extra{min-height:0!important}.dataset.sql .ant-form-item{margin-bottom:14px}.dataset.sql .sql-form{width:100%}.dataset.sql .sql-test{width:0}.dataset.sql .sql-test h3{display:none}.dataset.sql.test .sql-form{width:50%}.dataset.sql.test .sql-test{border:1px solid green;background-color:#f5f5f5;border-radius:2px;height:100%;width:48%;margin:0 1% 24px;padding:12px;align-self:flex-end;box-sizing:border-box}.dataset.sql.test .sql-test h3{display:block;color:green}.sql-table .ant-table.ant-table-bordered>.ant-table-container>.ant-table-content>table>thead>tr>th{background-color:#3896de;color:#fff;text-align:center}.label-info{line-height:24px;height:24px;font-size:14px;color:#717171}.label-info-label{display:inline-block;font-weight:600;min-width:80px}.label-info-list{padding:0 40px}.label-info-list-row{border-bottom:1px dashed #ccc}\n",document.head.appendChild(u),System.register(["./index-legacy.90883ee8.js","./vendor-legacy.bbc7855f.js"],(function(t){"use strict";var a,r,u,l,o,i,s,d,c,f,v,p,b,m,g,_,h,y,q,w,x,O,S,j,U,I,k,C,D;return{setters:[function(e){a=e.t,r=e.s,u=e.r},function(e){l=e.d,o=e.J,i=e.t,s=e.r,d=e.F,c=e.c,f=e.K,v=e.x,p=e.I,b=e.O,m=e.P,g=e.b,_=e.T,h=e.H,y=e.Q,q=e.m,w=e.B,x=e.V,O=e.n,S=e.o,j=e.W,U=e.X,I=e.Y,k=e.Z,C=e.k,D=e._}],execute:function(){var L=l({name:"URLDataset",props:{data:{type:Object,required:!0},setVisible:{type:Function,required:!0},onSubmit:{type:Function,required:!0},action:{type:Object,required:!0}},setup:function(e,t){t.emit;var a=o(n({},e.data));i((function(){e.data.id!=a.value.id&&(a.value=n({},e.data))}));var r=s({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{required:!0,message:"请输入数据源名称"}],dataset_name:[{required:!0,message:"请输入数据集名称"}],dataset_url:[{required:!0,message:"请输入数据集URL"}],parameter:[{required:!0,message:"请输入数据集类型"}],dataset_type:[{required:!0,message:"请输入数据集类型"}],dataset_describe:[{required:!0,message:"请输入数据集说明"}]}),u=d.useForm(a,r),l=(u.resetFields,u.validate),w=u.validateInfos;u.mergeValidateInfo;return function(){return c(y,{title:function(){return c(h,null,[c("strong",null,[e.action.name]),g(" API数据集配置")])},width:700,visible:e.action.visible,closable:!0,onOk:function(){console.log("itemData:: ",a),l().then((function(){e.setVisible(!1),e.onSubmit(a.value)})).catch((function(e){q.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:function(){return e.setVisible(!1)},destroyOnClose:!1},{default:function(){return[c(d,{name:"dataset",class:"dataset"},{default:function(){return[c(f,v({label:"所属项目"},w.project_name),{default:function(){return[c(p,{value:a.value.project_name,"onUpdate:value":function(e){return a.value.project_name=e}},null)]}}),c(f,v({label:"数据源"},w.datasource_name),{default:function(){return[c(p,{value:a.value.datasource_name,"onUpdate:value":function(e){return a.value.datasource_name=e}},null)]}}),c(f,v({label:"数据集"},w.dataset_name),{default:function(){return[c(p,{value:a.value.dataset_name,"onUpdate:value":function(e){return a.value.dataset_name=e}},null)]}}),c(f,v({label:"URL"},w.dataset_url),{default:function(){return[c(p,{value:a.value.dataset_url,"onUpdate:value":function(e){return a.value.dataset_url=e}},null)]}}),c(f,v({label:"请求参数"},w.parameter),{default:function(){return[c(p,{value:a.value.parameter,"onUpdate:value":function(e){return a.value.parameter=e}},null)]}}),c(f,v({label:"数据类型"},w.dataset_type),{default:function(){return[c(b,{value:a.value.dataset_type,"onUpdate:value":function(e){return a.value.dataset_type=e}},{default:function(){return[c(m,{value:0},{default:function(){return[g("对象")]}}),c(m,{value:1},{default:function(){return[g("列表")]}})]}})]}}),c(f,v({label:"数据集信息"},w.dataset_describe),{default:function(){return[c(_,{value:a.value.dataset_describe,"onUpdate:value":function(e){return a.value.dataset_describe=e}},null)]}})]}})]}})}}}),P=l({name:"SQLDataset",props:{data:{type:Object,required:!0},setVisible:{type:Function,required:!0},onSubmit:{type:Function,required:!0},action:{type:Object,required:!0}},setup:function(t,r){r.emit;var u=o(n(n({},t.data),t.data.database_config)),l=o(700),O=o();i((function(){t.data.id!=u.value.id&&(u.value=n(n({},t.data),t.data.database_config))}));var S=s({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{message:"请输入数据源名称"}],dataset_name:[{required:!0,message:"请输入数据集名称"}],dataset_type:[{required:!0,message:"请输入数据集类型"}],dataset_describe:[{required:!0,message:"请输入数据集说明"}],database_type:[{required:!0,message:"请输入数据库类型"}],host:[{required:!0,message:"请输入数据库IP"}],port:[{required:!0,message:"请输入数据库监听端口"}],user:[{required:!0,message:"请输入数据库用户名"}],password:[{required:!0,message:"请输入数据库密码"}],database:[{required:!0,message:"请输入数据库名称"}],database_sql:[{required:!0,message:"请输入数据集SQL"}]}),j=d.useForm(u,S),U=(j.resetFields,j.validate),I=j.validateInfos,k=(j.mergeValidateInfo,function(){U().then((function(){var e={host:u.value.host,port:u.value.port,user:u.value.user,password:u.value.password,database_sql:u.value.database_sql,database_type:u.value.database_type,database:u.value.database};console.log("config :: ",e),a(e).then((function(e){e.isHandleSuccess&&(O.value=e.resultData,l.value="85%")}))}))}),C=function t(a){if(a instanceof Array){if(0==a.length)return c("div",null,[g("[]")]);a.forEach((function(e,t){return e._no=t+1}));var n=[{title:"序号",dataIndex:"_no",width:80}];return n=[].concat(e(n),e(Object.keys(a[0]).filter((function(e){return"_no"!=e})).map((function(e){return{title:e,dataIndex:e,width:200}})))),c(x,{class:"sql-table",size:"small",rowKey:"_no",bordered:!0,columns:n,dataSource:a,pagination:!1},null)}return a instanceof Object?t([a]):c("div",null,[a.toString()])};return function(){return c(y,{title:function(){return c(h,null,[c("strong",null,[t.action.name]),g(" SQL数据集配置")])},width:l.value,visible:t.action.visible,closable:!0,onOk:function(){console.log("itemData:: ",u),U().then((function(){var e=n(n({},u.value),{},{database_config:{host:u.value.host,port:u.value.port,user:u.value.user,password:u.value.password,database:u.value.database}});delete e.host,delete e.port,delete e.user,delete e.password,delete e.database,t.setVisible(!1),t.onSubmit(e)})).catch((function(e){q.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:function(){O.value=void 0,l.value=700,t.setVisible(!1)},destroyOnClose:!1},{default:function(){return[c("div",{class:"dataset sql"+(O.value?" test":"")},[c(d,{name:"dataset",class:"sql-form"},{default:function(){return[c(f,v({label:"所属项目"},I.project_name),{default:function(){return[c(p,{value:u.value.project_name,"onUpdate:value":function(e){return u.value.project_name=e}},null)]}}),c(f,v({label:"数据源"},I.datasource_name),{default:function(){return[c(p,{value:u.value.datasource_name,"onUpdate:value":function(e){return u.value.datasource_name=e}},null)]}}),c(f,v({label:"数据集"},I.dataset_name),{default:function(){return[c(p,{value:u.value.dataset_name,"onUpdate:value":function(e){return u.value.dataset_name=e}},null)]}}),c(f,v({label:"数据类型"},I.dataset_type),{default:function(){return[c(b,{value:u.value.dataset_type,"onUpdate:value":function(e){return u.value.dataset_type=e}},{default:function(){return[c(m,{value:0},{default:function(){return[g("列表")]}}),c(m,{value:1},{default:function(){return[g("对象")]}})]}})]}}),c(f,v({label:"数据集信息"},I.dataset_describe),{default:function(){return[c(_,{value:u.value.dataset_describe,"onUpdate:value":function(e){return u.value.dataset_describe=e}},null)]}}),c("div",{class:"database-config"},[c("h3",null,[g("数据库配置")]),c(f,v({label:"HOST"},I.host),{default:function(){return[c(p,{value:u.value.host,"onUpdate:value":function(e){return u.value.host=e}},null)]}}),c(f,v({label:"端口"},I.port),{default:function(){return[c(p,{value:u.value.port,"onUpdate:value":function(e){return u.value.port=e}},null)]}}),c(f,v({label:"数据库"},I.database),{default:function(){return[c(p,{value:u.value.database,"onUpdate:value":function(e){return u.value.database=e}},null)]}}),c(f,v({label:"用户名"},I.user),{default:function(){return[c(p,{value:u.value.user,"onUpdate:value":function(e){return u.value.user=e}},null)]}}),c(f,v({label:"密码"},I.password),{default:function(){return[c(p,{value:u.value.password,"onUpdate:value":function(e){return u.value.password=e}},null)]}}),c(f,v({label:"SQL语句"},I.database_sql),{default:function(){return[c(p,{value:u.value.database_sql,"onUpdate:value":function(e){return u.value.database_sql=e}},null)]}}),c(f,v({label:"数据库类型"},I.database_type),{default:function(){return[c(b,{value:u.value.database_type,"onUpdate:value":function(e){return u.value.database_type=e}},{default:function(){return[c(m,{value:0},{default:function(){return[g("MYSQL")]}}),c(m,{value:1},{default:function(){return[g("ORACLE")]}})]}}),c(w,{onClick:k,style:"float: right",type:"primary",ghost:!0},{default:function(){return[g("测试SQL")]}})]}})])]}}),c("div",{class:"sql-test"},[c("h3",null,[g("测试成功，返回结果")]),O.value&&C(O.value)])])]}})}}}),R=function(e,t){t.emit;return c("div",{class:"label-info"},[c("span",{class:"label-info-label"},[e.label,g("：")]),c("span",{class:"label-info-value"},[e.value])])},A={add:"新增",modify:"修改",delete:"删除",none:""},T={id:0,project_name:"",datasource_name:"",dataset_name:"",dataset_type:0,dataset_url:"",dataset_describe:"",topic_name:"",parameter:"",dataset_field:[]},V=n(n({},T),{},{dataset_source_type:0}),z=n(n({},T),{},{dataset_source_type:1,database_type:0,database_config:{host:"",port:0,user:"",password:"",database:""},database_sql:""});t("default",l({name:"ProductDataSet",setup:function(){var t=O((function(){return"/product/urldataset"==C().path?0:1})),a=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1,a=arguments.length>2&&void 0!==arguments[2]?arguments[2]:10;r.dispatch("dataset/getDatasets",{page_index:t,page_size:a,dataset_source_type:e})};S((function(){a(t.value)})),i((function(){t.value!=r.state.dataset.queryDataset.dataset_source_type&&a(t.value)}));var n=o([]),l=o("none"),s=0==t.value?o(V):o(z),d=function(e){0==e.id?r.dispatch("dataset/postDataset",e):r.dispatch("dataset/putDataset",e)},f=function(){s.value=0==t.value?V:z,l.value="add"},v=function(e){console.log("delete item:: ",e);var t=e instanceof Array?e:[e.id];y.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:function(){r.dispatch("dataset/deleteDatasets",{list_id:t})}})},p=[{title:"NO",dataIndex:"id",resizable:!0,width:50,customRender:function(e){e.record;return e.index+1}},{title:"所属项目",dataIndex:"project_name",width:160},{title:"数据源名称",dataIndex:"datasource_name",width:200},{title:"数据集名称",dataIndex:"dataset_name",width:160,customRender:function(e){var t=e.record;return c(D,{title:"点击查看数据集包含的字段"},{default:function(){return[c(w,{type:"link",onClick:function(){return u.push("/product/datasetField?dataset_id="+t.id+"&dataset_name="+t.dataset_name)}},{default:function(){return[t.dataset_name]}})]}})}}],b=O((function(){var a=[{title:"数据库类型",dataIndex:"database_type",width:200,customRender:function(e){var t=e.record;return c(h,null,[1==t.database_type?"ORACLE":"MYSQL"])}},{title:"数据库配置",dataIndex:"database_config",width:300,customRender:function(e){var t=e.record.database_config||{};return c(h,null,[c(R,{label:"HOST",value:t.host},null),c(R,{label:"PORT",value:(t.port||"").toString()},null),c(R,{label:"用户名",value:t.user},null),c(R,{label:"数据库",value:t.database},null)])}}];return[].concat(p,e(0==t.value?[{title:"URL",dataIndex:"dataset_url",width:160},{title:"URL参数",dataIndex:"parameter",width:160}]:a),[{title:"数据类型",dataIndex:"dataset_type",width:160,customRender:function(e){var t=e.record;return c(h,null,[0==t.dataset_type?"对象":"列表"])}},{title:"说明",dataIndex:"dataset_describe",width:200},{title:"创建时间",dataIndex:"create_time",width:140},{key:"action",title:"操作",width:80,customRender:function(e){var t=e.record;return c(h,null,[c(w,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:function(){return e=t,s.value=e,void(l.value="modify");var e}},{default:function(){return[g("修改")]}}),c(w,{size:"small",type:"ghost",danger:!0,onClick:function(){return v(t)}},{default:function(){return[g("删除")]}})])}}])}));return function(){return c(h,null,[c("h2",{class:"title"},[g("数据集配置管理"),c(j,{onSearch:function(){},class:"title-search",placeholder:"请输入数据集名称"},null)]),c(U,null,null),c("div",{class:"table"},[c(w,{type:"ghost",danger:!0,icon:c(I,null,null),class:"table-delete ".concat(0==n.value.length?"hidden":""),onClick:function(){return v(n.value)}},{default:function(){return[g("删除")]}}),c(w,{type:"primary",icon:c(k,null,null),class:"table-add",onClick:f},{default:function(){return[g("新增")]}}),c(x,{rowKey:"id",bordered:!0,loading:r.state.dataset.loading,columns:b.value,dataSource:r.state.dataset.datasets.rows,rowSelection:{onChange:function(e){n.value=e}},pagination:{showQuickJumper:!0,showSizeChanger:!0,pageSize:r.state.dataset.datasets.pageSize,total:r.state.dataset.datasets.total,current:r.state.dataset.datasets.pageNo,onChange:function(e,n){return a(t.value,e,n)}}},null),0==t.value?c(L,{data:s.value,onSubmit:d,action:{name:A[l.value],visible:"add"==l.value||"modify"==l.value},setVisible:function(){return l.value="none"}},null):c(P,{data:s.value,onSubmit:d,action:{name:A[l.value],visible:"add"==l.value||"modify"==l.value},setVisible:function(){return l.value="none"}},null)])])}}}))}}}))}();
static/assets/ProductDataSet.2eb44551.js 0 → 100644
var e=Object.defineProperty,a=Object.defineProperties,t=Object.getOwnPropertyDescriptors,l=Object.getOwnPropertySymbols,s=Object.prototype.hasOwnProperty,d=Object.prototype.propertyIsEnumerable,u=(a,t,l)=>t in a?e(a,t,{enumerable:!0,configurable:!0,writable:!0,value:l}):a[t]=l,n=(e,a)=>{for(var t in a||(a={}))s.call(a,t)&&u(e,t,a[t]);if(l)for(var t of l(a))d.call(a,t)&&u(e,t,a[t]);return e},r=(e,l)=>a(e,t(l));import{t as o,s as i,r as v}from"./index.f608f11e.js";import{d as c,J as p,t as b,r as m,F as _,c as f,K as y,x as g,I as h,O as q,P as w,b as O,T as S,H as x,Q as j,m as U,B as I,V as k,n as C,o as L,W as R,X as D,Y as V,Z as F,k as P,_ as T}from"./vendor.98dba853.js";const Q=c({name:"URLDataset",props:{data:{type:Object,required:!0},setVisible:{type:Function,required:!0},onSubmit:{type:Function,required:!0},action:{type:Object,required:!0}},setup:(e,{emit:a})=>{const t=p(n({},e.data));b((()=>{e.data.id!=t.value.id&&(t.value=n({},e.data))}));const l=m({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{required:!0,message:"请输入数据源名称"}],dataset_name:[{required:!0,message:"请输入数据集名称"}],dataset_url:[{required:!0,message:"请输入数据集URL"}],parameter:[{required:!0,message:"请输入数据集类型"}],dataset_type:[{required:!0,message:"请输入数据集类型"}],dataset_describe:[{required:!0,message:"请输入数据集说明"}]}),{resetFields:s,validate:d,validateInfos:u,mergeValidateInfo:r}=_.useForm(t,l);return()=>f(j,{title:()=>f(x,null,[f("strong",null,[e.action.name]),O(" API数据集配置")]),width:700,visible:e.action.visible,closable:!0,onOk:()=>{console.log("itemData:: ",t),d().then((()=>{e.setVisible(!1),e.onSubmit(t.value)})).catch((e=>{U.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:()=>e.setVisible(!1),destroyOnClose:!1},{default:()=>[f(_,{name:"dataset",class:"dataset"},{default:()=>[f(y,g({label:"所属项目"},u.project_name),{default:()=>[f(h,{value:t.value.project_name,"onUpdate:value":e=>t.value.project_name=e},null)]}),f(y,g({label:"数据源"},u.datasource_name),{default:()=>[f(h,{value:t.value.datasource_name,"onUpdate:value":e=>t.value.datasource_name=e},null)]}),f(y,g({label:"数据集"},u.dataset_name),{default:()=>[f(h,{value:t.value.dataset_name,"onUpdate:value":e=>t.value.dataset_name=e},null)]}),f(y,g({label:"URL"},u.dataset_url),{default:()=>[f(h,{value:t.value.dataset_url,"onUpdate:value":e=>t.value.dataset_url=e},null)]}),f(y,g({label:"请求参数"},u.parameter),{default:()=>[f(h,{value:t.value.parameter,"onUpdate:value":e=>t.value.parameter=e},null)]}),f(y,g({label:"数据类型"},u.dataset_type),{default:()=>[f(q,{value:t.value.dataset_type,"onUpdate:value":e=>t.value.dataset_type=e},{default:()=>[f(w,{value:0},{default:()=>[O("对象")]}),f(w,{value:1},{default:()=>[O("列表")]})]})]}),f(y,g({label:"数据集信息"},u.dataset_describe),{default:()=>[f(S,{value:t.value.dataset_describe,"onUpdate:value":e=>t.value.dataset_describe=e},null)]})]})]})}}),z=c({name:"SQLDataset",props:{data:{type:Object,required:!0},setVisible:{type:Function,required:!0},onSubmit:{type:Function,required:!0},action:{type:Object,required:!0}},setup:(e,{emit:a})=>{const t=p(n(n({},e.data),e.data.database_config)),l=p(700),s=p();b((()=>{e.data.id!=t.value.id&&(t.value=n(n({},e.data),e.data.database_config))}));const d=m({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{message:"请输入数据源名称"}],dataset_name:[{required:!0,message:"请输入数据集名称"}],dataset_type:[{required:!0,message:"请输入数据集类型"}],dataset_describe:[{required:!0,message:"请输入数据集说明"}],database_type:[{required:!0,message:"请输入数据库类型"}],host:[{required:!0,message:"请输入数据库IP"}],port:[{required:!0,message:"请输入数据库监听端口"}],user:[{required:!0,message:"请输入数据库用户名"}],password:[{required:!0,message:"请输入数据库密码"}],database:[{required:!0,message:"请输入数据库名称"}],database_sql:[{required:!0,message:"请输入数据集SQL"}]}),{resetFields:u,validate:i,validateInfos:v,mergeValidateInfo:c}=_.useForm(t,d),C=()=>{i().then((()=>{const e={host:t.value.host,port:t.value.port,user:t.value.user,password:t.value.password,database_sql:t.value.database_sql,database_type:t.value.database_type,database:t.value.database};console.log("config :: ",e),o(e).then((e=>{e.isHandleSuccess&&(s.value=e.resultData,l.value="85%")}))}))},L=e=>{if(e instanceof Array){if(0==e.length)return f("div",null,[O("[]")]);e.forEach(((e,a)=>e._no=a+1));let a=[{title:"序号",dataIndex:"_no",width:80}];return a=[...a,...Object.keys(e[0]).filter((e=>"_no"!=e)).map((e=>({title:e,dataIndex:e,width:200})))],f(k,{class:"sql-table",size:"small",rowKey:"_no",bordered:!0,columns:a,dataSource:e,pagination:!1},null)}return e instanceof Object?L([e]):f("div",null,[e.toString()])};return()=>f(j,{title:()=>f(x,null,[f("strong",null,[e.action.name]),O(" SQL数据集配置")]),width:l.value,visible:e.action.visible,closable:!0,onOk:()=>{console.log("itemData:: ",t),i().then((()=>{const a=r(n({},t.value),{database_config:{host:t.value.host,port:t.value.port,user:t.value.user,password:t.value.password,database:t.value.database}});delete a.host,delete a.port,delete a.user,delete a.password,delete a.database,e.setVisible(!1),e.onSubmit(a)})).catch((e=>{U.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:()=>{s.value=void 0,l.value=700,e.setVisible(!1)},destroyOnClose:!1},{default:()=>[f("div",{class:"dataset sql"+(s.value?" test":"")},[f(_,{name:"dataset",class:"sql-form"},{default:()=>[f(y,g({label:"所属项目"},v.project_name),{default:()=>[f(h,{value:t.value.project_name,"onUpdate:value":e=>t.value.project_name=e},null)]}),f(y,g({label:"数据源"},v.datasource_name),{default:()=>[f(h,{value:t.value.datasource_name,"onUpdate:value":e=>t.value.datasource_name=e},null)]}),f(y,g({label:"数据集"},v.dataset_name),{default:()=>[f(h,{value:t.value.dataset_name,"onUpdate:value":e=>t.value.dataset_name=e},null)]}),f(y,g({label:"数据类型"},v.dataset_type),{default:()=>[f(q,{value:t.value.dataset_type,"onUpdate:value":e=>t.value.dataset_type=e},{default:()=>[f(w,{value:0},{default:()=>[O("列表")]}),f(w,{value:1},{default:()=>[O("对象")]})]})]}),f(y,g({label:"数据集信息"},v.dataset_describe),{default:()=>[f(S,{value:t.value.dataset_describe,"onUpdate:value":e=>t.value.dataset_describe=e},null)]}),f("div",{class:"database-config"},[f("h3",null,[O("数据库配置")]),f(y,g({label:"HOST"},v.host),{default:()=>[f(h,{value:t.value.host,"onUpdate:value":e=>t.value.host=e},null)]}),f(y,g({label:"端口"},v.port),{default:()=>[f(h,{value:t.value.port,"onUpdate:value":e=>t.value.port=e},null)]}),f(y,g({label:"数据库"},v.database),{default:()=>[f(h,{value:t.value.database,"onUpdate:value":e=>t.value.database=e},null)]}),f(y,g({label:"用户名"},v.user),{default:()=>[f(h,{value:t.value.user,"onUpdate:value":e=>t.value.user=e},null)]}),f(y,g({label:"密码"},v.password),{default:()=>[f(h,{value:t.value.password,"onUpdate:value":e=>t.value.password=e},null)]}),f(y,g({label:"SQL语句"},v.database_sql),{default:()=>[f(h,{value:t.value.database_sql,"onUpdate:value":e=>t.value.database_sql=e},null)]}),f(y,g({label:"数据库类型"},v.database_type),{default:()=>[f(q,{value:t.value.database_type,"onUpdate:value":e=>t.value.database_type=e},{default:()=>[f(w,{value:0},{default:()=>[O("MYSQL")]}),f(w,{value:1},{default:()=>[O("ORACLE")]})]}),f(I,{onClick:C,style:"float: right",type:"primary",ghost:!0},{default:()=>[O("测试SQL")]})]})])]}),f("div",{class:"sql-test"},[f("h3",null,[O("测试成功，返回结果")]),s.value&&L(s.value)])])]})}});const A=(e,{emit:a})=>f("div",{class:"label-info"},[f("span",{class:"label-info-label"},[e.label,O("：")]),f("span",{class:"label-info-value"},[e.value])]),E={add:"新增",modify:"修改",delete:"删除",none:""},H={id:0,project_name:"",datasource_name:"",dataset_name:"",dataset_type:0,dataset_url:"",dataset_describe:"",topic_name:"",parameter:"",dataset_field:[]},K=r(n({},H),{dataset_source_type:0}),Y=r(n({},H),{dataset_source_type:1,database_type:0,database_config:{host:"",port:0,user:"",password:"",database:""},database_sql:""}),J=c({name:"ProductDataSet",setup(){const e=C((()=>"/product/urldataset"==P().path?0:1)),a=(e,a=1,t=10)=>{i.dispatch("dataset/getDatasets",{page_index:a,page_size:t,dataset_source_type:e})};L((()=>{a(e.value)})),b((()=>{e.value!=i.state.dataset.queryDataset.dataset_source_type&&a(e.value)}));const t=p([]),l=p("none"),s=0==e.value?p(K):p(Y),d=e=>{0==e.id?i.dispatch("dataset/postDataset",e):i.dispatch("dataset/putDataset",e)},u=()=>{s.value=0==e.value?K:Y,l.value="add"},n=e=>{console.log("delete item:: ",e);const a=e instanceof Array?e:[e.id];j.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:()=>{i.dispatch("dataset/deleteDatasets",{list_id:a})}})},r=[{title:"NO",dataIndex:"id",resizable:!0,width:50,customRender:({record:e,index:a})=>a+1},{title:"所属项目",dataIndex:"project_name",width:160},{title:"数据源名称",dataIndex:"datasource_name",width:200},{title:"数据集名称",dataIndex:"dataset_name",width:160,customRender:({record:e})=>f(T,{title:"点击查看数据集包含的字段"},{default:()=>[f(I,{type:"link",onClick:()=>v.push("/product/datasetField?dataset_id="+e.id+"&dataset_name="+e.dataset_name)},{default:()=>[e.dataset_name]})]})}],o=C((()=>{const a=[{title:"数据库类型",dataIndex:"database_type",width:200,customRender:({record:e})=>f(x,null,[1==e.database_type?"ORACLE":"MYSQL"])},{title:"数据库配置",dataIndex:"database_config",width:300,customRender:({record:e})=>{const a=e.database_config||{};return f(x,null,[f(A,{label:"HOST",value:a.host},null),f(A,{label:"PORT",value:(a.port||"").toString()},null),f(A,{label:"用户名",value:a.user},null),f(A,{label:"数据库",value:a.database},null)])}}];return[...r,...0==e.value?[{title:"URL",dataIndex:"dataset_url",width:160},{title:"URL参数",dataIndex:"parameter",width:160}]:a,{title:"数据类型",dataIndex:"dataset_type",width:160,customRender:({record:e})=>f(x,null,[0==e.dataset_type?"对象":"列表"])},{title:"说明",dataIndex:"dataset_describe",width:200},{title:"创建时间",dataIndex:"create_time",width:140},{key:"action",title:"操作",width:80,customRender:({record:e})=>f(x,null,[f(I,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:()=>{return a=e,s.value=a,void(l.value="modify");var a}},{default:()=>[O("修改")]}),f(I,{size:"small",type:"ghost",danger:!0,onClick:()=>n(e)},{default:()=>[O("删除")]})])}]}));return()=>f(x,null,[f("h2",{class:"title"},[O("数据集配置管理"),f(R,{onSearch:()=>{},class:"title-search",placeholder:"请输入数据集名称"},null)]),f(D,null,null),f("div",{class:"table"},[f(I,{type:"ghost",danger:!0,icon:f(V,null,null),class:"table-delete "+(0==t.value.length?"hidden":""),onClick:()=>n(t.value)},{default:()=>[O("删除")]}),f(I,{type:"primary",icon:f(F,null,null),class:"table-add",onClick:u},{default:()=>[O("新增")]}),f(k,{rowKey:"id",bordered:!0,loading:i.state.dataset.loading,columns:o.value,dataSource:i.state.dataset.datasets.rows,rowSelection:{onChange:e=>{t.value=e}},pagination:{showQuickJumper:!0,showSizeChanger:!0,pageSize:i.state.dataset.datasets.pageSize,total:i.state.dataset.datasets.total,current:i.state.dataset.datasets.pageNo,onChange:(t,l)=>a(e.value,t,l)}},null),0==e.value?f(Q,{data:s.value,onSubmit:d,action:{name:E[l.value],visible:"add"==l.value||"modify"==l.value},setVisible:()=>l.value="none"},null):f(z,{data:s.value,onSubmit:d,action:{name:E[l.value],visible:"add"==l.value||"modify"==l.value},setVisible:()=>l.value="none"},null)])])}});export{J as default};
static/assets/ProductDataSet.daa66c6c.css 0 → 100644
.dataset .database-config{padding:10px 10px 0 0;border:1px solid #eee;border-radius:4px;margin-left:0;margin-bottom:20px}.dataset .database-config h3{width:100px;text-align:right}.dataset .database-config .ant-form-item{margin-bottom:10px}.dataset.sql{display:flex;flex-direction:row}.dataset.sql .ant-form-item-explain,.dataset.sql .ant-form-item-extra{min-height:0!important}.dataset.sql .ant-form-item{margin-bottom:14px}.dataset.sql .sql-form{width:100%}.dataset.sql .sql-test{width:0}.dataset.sql .sql-test h3{display:none}.dataset.sql.test .sql-form{width:50%}.dataset.sql.test .sql-test{border:1px solid green;background-color:#f5f5f5;border-radius:2px;height:100%;width:48%;margin:0 1% 24px;padding:12px;align-self:flex-end;box-sizing:border-box}.dataset.sql.test .sql-test h3{display:block;color:green}.sql-table .ant-table.ant-table-bordered>.ant-table-container>.ant-table-content>table>thead>tr>th{background-color:#3896de;color:#fff;text-align:center}.label-info{line-height:24px;height:24px;font-size:14px;color:#717171}.label-info-label{display:inline-block;font-weight:600;min-width:80px}.label-info-list{padding:0 40px}.label-info-list-row{border-bottom:1px dashed #ccc}
static/assets/ProductDataSetField-legacy.81eaee2a.js 0 → 100644
System.register(["./index-legacy.90883ee8.js","./vendor-legacy.bbc7855f.js"],(function(e){"use strict";var t,n,a,l,i,d,u,r,o,c,s,f,_,v,m,p,y,b,g,h,x,k,I,w,C,F;return{setters:[function(e){t=e.s,n=e.r},function(e){a=e.d,l=e.k,i=e.o,d=e.J,u=e.r,r=e.F,o=e.c,c=e.b,s=e.W,f=e.X,_=e.B,v=e.Y,m=e.Z,p=e.$,y=e.V,b=e.Q,g=e.H,h=e.K,x=e.x,k=e.I,I=e.O,w=e.P,C=e.T,F=e.m}],execute:function(){var D={add:"新增",modify:"修改",delete:"删除",none:""};e("default",a({name:"ProductDataSet",setup:function(){var e=l();console.log("route:: ",e),i((function(){t.dispatch("dataset/getDatasetFields",{dataset_id:e.query.dataset_id,page_size:10,page_index:1})}));var a=d([]),q=d("none"),z=d(""),O=u({id:0,field_name:"",field_type:0,field_describe:""}),S=u({field_name:[{required:!0,message:"请输入字段名称"}],field_type:[{required:!0,message:"请输入字段类型"}],field_describe:[{required:!0,message:"请输入字段说明"}]}),T=r.useForm(O,S),R=T.resetFields,U=T.validate,j=T.validateInfos,K=(T.mergeValidateInfo,function(){R(),q.value="add"}),P=function(e){console.log("delete item:: ",e);var n=e instanceof Array?e:[e.id];b.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:function(){t.dispatch("dataset/deleteDatasetFields",{list_id:n})}})},V=[{title:"ID",dataIndex:"id",resizable:!0,width:50},{title:"所属数据集",dataIndex:"dataset_id",resizable:!0,width:200,customRender:function(t){return t.record,o(g,null,[e.query.dataset_name])}},{title:"字段名称",dataIndex:"field_name"},{title:"字段类型",dataIndex:"field_type",customRender:function(e){var t=e.record;return o(g,null,[0==t.field_type?"数字":"字符串"])}},{title:"说明",dataIndex:"field_describe"},{title:"创建时间",dataIndex:"create_time",resizable:!0,width:200},{key:"action",title:"操作",width:135,customRender:function(e){var t=e.record;return o(g,null,[o(_,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:function(){return e=t,O.field_name=e.field_name,O.field_type=e.field_type,O.field_describe=e.field_describe,O.id=e.id,void(q.value="modify");var e}},{default:function(){return[c("修改")]}}),o(_,{size:"small",type:"ghost",danger:!0,onClick:function(){return P(t)}},{default:function(){return[c("删除")]}})])}}];return function(){return o(g,null,[o("h2",{class:"title"},[c("字段配置管理"),o(s,{onSearch:function(e){z.value=e},class:"title-search",placeholder:"请输入字段名称"},null)]),o(f,null,null),o("div",{class:"table"},[o(_,{type:"ghost",danger:!0,icon:o(v,null,null),class:"table-delete ".concat(0==a.value.length?"hidden":""),onClick:function(){return P(a.value)}},{default:function(){return[c("删除")]}}),o(_,{type:"primary",icon:o(m,null,null),class:"table-add",onClick:K},{default:function(){return[c("新增")]}}),o(_,{type:"ghost",icon:o(p,null,null),class:"table-back",onClick:function(){return n.back()}},{default:function(){return[c("返回")]}}),o(y,{rowKey:"id",bordered:!0,loading:t.state.dataset.loading,columns:V,dataSource:(l=t.state.dataset.datasetsFields,z.value?l.filter((function(e){return e.field_name.indexOf(z.value)>-1})):l),rowSelection:{onChange:function(e){a.value=e}}},null)]),o(b,{title:function(){return o(g,null,[o("strong",null,[D[q.value]]),c(" 字段集配置")])},width:800,visible:"none"!=q.value,closable:!0,onOk:function(){console.log("itemData:: ",O),U().then((function(){O.dataset_id=parseInt(e.query.dataset_id),0==O.id?t.dispatch("dataset/postDatasetField",O):t.dispatch("dataset/putDatasetField",O),q.value="none"})).catch((function(e){F.warn("表单字段填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:function(){return q.value="none"},destroyOnClose:!1},{default:function(){return[o(r,{name:"field"},{default:function(){return[o(h,x({label:"字段名称"},j.field_name),{default:function(){return[o(k,{value:O.field_name,"onUpdate:value":function(e){return O.field_name=e}},null)]}}),o(h,x({label:"字段类型"},j.field_type),{default:function(){return[o(I,{value:O.field_type,"onUpdate:value":function(e){return O.field_type=e}},{default:function(){return[o(w,{value:0},{default:function(){return[c("数字")]}}),o(w,{value:1},{default:function(){return[c("字符串")]}})]}})]}}),o(h,x({label:"字段信息"},j.field_describe),{default:function(){return[o(C,{value:O.field_describe,"onUpdate:value":function(e){return O.field_describe=e}},null)]}})]}})]}})]);var l}}}))}}}));
static/assets/ProductDataSetField.4ae51f79.js 0 → 100644
import{s as e,r as a}from"./index.f608f11e.js";import{d as l,k as t,o as d,J as s,r as i,F as n,c as o,b as r,W as u,X as c,B as f,Y as m,Z as _,$ as p,V as v,Q as y,H as b,K as g,x as h,I as x,O as k,P as I,T as w,m as C}from"./vendor.98dba853.js";const F={add:"新增",modify:"修改",delete:"删除",none:""},D=l({name:"ProductDataSet",setup(){const l=t();console.log("route:: ",l),d((()=>{e.dispatch("dataset/getDatasetFields",{dataset_id:l.query.dataset_id,page_size:10,page_index:1})}));const D=s([]),q=s("none"),z=s(""),O=i({id:0,field_name:"",field_type:0,field_describe:""}),T=i({field_name:[{required:!0,message:"请输入字段名称"}],field_type:[{required:!0,message:"请输入字段类型"}],field_describe:[{required:!0,message:"请输入字段说明"}]}),{resetFields:S,validate:R,validateInfos:U,mergeValidateInfo:j}=n.useForm(O,T),K=()=>{S(),q.value="add"},P=a=>{console.log("delete item:: ",a);const l=a instanceof Array?a:[a.id];y.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:()=>{e.dispatch("dataset/deleteDatasetFields",{list_id:l})}})},V=[{title:"ID",dataIndex:"id",resizable:!0,width:50},{title:"所属数据集",dataIndex:"dataset_id",resizable:!0,width:200,customRender:({record:e})=>o(b,null,[l.query.dataset_name])},{title:"字段名称",dataIndex:"field_name"},{title:"字段类型",dataIndex:"field_type",customRender:({record:e})=>o(b,null,[0==e.field_type?"数字":"字符串"])},{title:"说明",dataIndex:"field_describe"},{title:"创建时间",dataIndex:"create_time",resizable:!0,width:200},{key:"action",title:"操作",width:135,customRender:({record:e})=>o(b,null,[o(f,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:()=>{return a=e,O.field_name=a.field_name,O.field_type=a.field_type,O.field_describe=a.field_describe,O.id=a.id,void(q.value="modify");var a}},{default:()=>[r("修改")]}),o(f,{size:"small",type:"ghost",danger:!0,onClick:()=>P(e)},{default:()=>[r("删除")]})])}];return()=>{return o(b,null,[o("h2",{class:"title"},[r("字段配置管理"),o(u,{onSearch:e=>{z.value=e},class:"title-search",placeholder:"请输入字段名称"},null)]),o(c,null,null),o("div",{class:"table"},[o(f,{type:"ghost",danger:!0,icon:o(m,null,null),class:"table-delete "+(0==D.value.length?"hidden":""),onClick:()=>P(D.value)},{default:()=>[r("删除")]}),o(f,{type:"primary",icon:o(_,null,null),class:"table-add",onClick:K},{default:()=>[r("新增")]}),o(f,{type:"ghost",icon:o(p,null,null),class:"table-back",onClick:()=>a.back()},{default:()=>[r("返回")]}),o(v,{rowKey:"id",bordered:!0,loading:e.state.dataset.loading,columns:V,dataSource:(t=e.state.dataset.datasetsFields,z.value?t.filter((e=>e.field_name.indexOf(z.value)>-1)):t),rowSelection:{onChange:e=>{D.value=e}}},null)]),o(y,{title:()=>o(b,null,[o("strong",null,[F[q.value]]),r(" 字段集配置")]),width:800,visible:"none"!=q.value,closable:!0,onOk:()=>{console.log("itemData:: ",O),R().then((()=>{O.dataset_id=parseInt(l.query.dataset_id),0==O.id?e.dispatch("dataset/postDatasetField",O):e.dispatch("dataset/putDatasetField",O),q.value="none"})).catch((e=>{C.warn("表单字段填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:()=>q.value="none",destroyOnClose:!1},{default:()=>[o(n,{name:"field"},{default:()=>[o(g,h({label:"字段名称"},U.field_name),{default:()=>[o(x,{value:O.field_name,"onUpdate:value":e=>O.field_name=e},null)]}),o(g,h({label:"字段类型"},U.field_type),{default:()=>[o(k,{value:O.field_type,"onUpdate:value":e=>O.field_type=e},{default:()=>[o(I,{value:0},{default:()=>[r("数字")]}),o(I,{value:1},{default:()=>[r("字符串")]})]})]}),o(g,h({label:"字段信息"},U.field_describe),{default:()=>[o(w,{value:O.field_describe,"onUpdate:value":e=>O.field_describe=e},null)]})]})]})]);var t}}});export{D as default};
static/assets/ProductTemplate-legacy.6081af85.js 0 → 100644
System.register(["./index-legacy.90883ee8.js","./vendor-legacy.bbc7855f.js"],(function(e){"use strict";var t,n,a,l,r,u,i,o,c,d,s,f,p,m,v,_,g,h,b,y,x,w,k,z,C,T,S,j,I,U,D;return{setters:[function(e){t=e.s,n=e.U,a=e.D},function(e){l=e.d,r=e.o,u=e.J,i=e.r,o=e.a0,c=e.F,d=e.c,s=e.b,f=e.W,p=e.X,m=e.B,v=e.Y,_=e.Z,g=e.V,h=e.Q,b=e.H,y=e.K,x=e.x,w=e.I,k=e.a1,z=e.a2,C=e.T,T=e.O,S=e.P,j=e.a3,I=e.l,U=e.m,D=e._}],execute:function(){var O={add:"新增",modify:"修改",report:"报告",none:""};e("default",l({name:"ProductDataSet",setup:function(){r((function(){e(1,10)}));var e=function(e,n){t.dispatch("template/getTemplates",{page_size:n,page_index:e})},l=u([]),q=u("none"),Y=u([]),F=i({range:[o().add(-1,"week"),o()],type:0}),R=i({start_time:"",end_time:"",report_type:0}),H=function(e){switch(console.log("v::",e.target.value),e.target.value){case 0:F.range[0]=F.range[1].add(-1,"week");break;case 1:F.range[0]=F.range[1].add(-1,"month");break;case 2:F.range[0]=F.range[1].add(-3,"month");break;case 3:F.range[0]=F.range[1].add(-1,"year")}},L=i({id:0,user_id:0,project_name:"",datasource_name:"",template_name:"",template_path:"",template_describe:"",topic_name:""}),M=function(e){Y.value=e.fileList,"done"==e.file.status&&e.file.response.isHandleSuccess&&(L.template_path=e.file.response.resultData.file_path,Y.value=[])},P=function(){R.start_time=F.range[0].toString(),R.end_time=F.range[1].toString();var e=I.stringify(R),t=document.createElement("a");t.href="/api/report?"+e,t.download=t.href,t.click(),t.remove(),U.success("报告生成成功")},A=i({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{required:!0,message:"请输入数据源名称"}],template_name:[{required:!0,message:"请输入报告模板名称"}],template_path:[{required:!0,message:"请上传报告模板"}],template_describe:[{required:!0,message:"请输入报告模板说明"}]}),B=c.useForm(L,A),J=B.resetFields,K=B.validate,N=B.validateInfos,Q=(B.mergeValidateInfo,function(){J(),Y.value=[],q.value="add"}),V=function(e){console.log("delete item:: ",e);var n=e instanceof Array?e:[e.id];h.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:function(){t.dispatch("template/deleteTemplates",{list_id:n})}})},W=[{title:"NO",dataIndex:"id",resizable:!0,width:50,customRender:function(e){return e.record,e.index+1}},{title:"所属项目",dataIndex:"project_name",resizable:!0,width:200},{title:"数据源",dataIndex:"datasource_name",resizable:!0,width:200},{title:"报告名称",dataIndex:"template_name",resizable:!0,width:200,customRender:function(e){var t=e.record;return d(b,null,[t.template_name,d(D,{title:"点击开始配置报告周期"},{default:function(){return[d(m,{onClick:function(){return e=t,R.template_id=e.id,void(q.value="report");var e},size:"small",type:"primary",ghost:!0},{default:function(){return[s("生成报告")]}})]}})])}},{title:"模板地址",dataIndex:"template_path",resizable:!0,width:200,customRender:function(e){var t=e.record;return d(D,{title:"点击下载模板文件"},{default:function(){return[d("a",{href:a+t.template_path,download:a+t.template_path},[t.template_path])]}})}},{title:"说明",dataIndex:"template_describe",resizable:!0,width:200},{title:"创建时间",dataIndex:"create_time",resizable:!0,width:200},{key:"action",title:"操作",width:80,customRender:function(e){var t=e.record;return d(b,null,[d(m,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:function(){return e=t,Y.value=[],L.project_name=e.project_name,L.datasource_name=e.datasource_name,L.template_name=e.template_name,L.template_path=e.template_path,L.template_describe=e.template_describe,L.id=e.id,void(q.value="modify");var e}},{default:function(){return[s("修改")]}}),d(m,{size:"small",type:"ghost",danger:!0,onClick:function(){return V(t)}},{default:function(){return[s("删除")]}})])}}];return function(){return d(b,null,[d("h2",{class:"title"},[s("报告模板配置管理"),d(f,{onSearch:function(){},class:"title-search",placeholder:"请输入报告模板名称"},null)]),d(p,null,null),d("div",{class:"table"},[d(m,{type:"ghost",danger:!0,icon:d(v,null,null),class:"table-delete ".concat(0==l.value.length?"hidden":""),onClick:function(){return V(l.value)}},{default:function(){return[s("删除")]}}),d(m,{type:"primary",icon:d(_,null,null),class:"table-add",onClick:Q},{default:function(){return[s("新增")]}}),d(g,{rowKey:"id",bordered:!0,loading:t.state.template.loading,columns:W,dataSource:t.state.template.templates.rows,rowSelection:{onChange:function(e){l.value=e}},pagination:{showQuickJumper:!0,showSizeChanger:!0,pageSize:t.state.template.templates.pageSize,total:t.state.template.templates.total,current:t.state.template.templates.pageNo,onChange:function(t,n){return e(t,n)}}},null)]),d(h,{title:function(){return d(b,null,[d("strong",null,[O[q.value]]),s(" 报告模板配置")])},width:800,visible:"add"==q.value||"modify"==q.value,closable:!0,onOk:function(){console.log("itemData:: ",L),K().then((function(){0==L.id?t.dispatch("template/postTemplate",L):t.dispatch("template/putTemplate",L),q.value="none"})).catch((function(e){U.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:function(){return q.value="none"},destroyOnClose:!1},{default:function(){return[d(c,{name:"template"},{default:function(){return[d(y,x({label:"所属项目"},N.project_name),{default:function(){return[d(w,{value:L.project_name,"onUpdate:value":function(e){return L.project_name=e}},null)]}}),d(y,x({label:"数据源"},N.datasource_name),{default:function(){return[d(w,{value:L.datasource_name,"onUpdate:value":function(e){return L.datasource_name=e}},null)]}}),d(y,x({label:"报告名称"},N.template_name),{default:function(){return[d(w,{value:L.template_name,"onUpdate:value":function(e){return L.template_name=e}},null)]}}),d(y,x({label:"模板文件"},N.template_path),{default:function(){return[d(w,{value:L.template_path,"onUpdate:value":function(e){return L.template_path=e},style:{width:"calc( 100% - 86px )"}},null)," ",d(k,{action:n,onChange:M,fileList:Y.value},{default:function(){return[d(m,{icon:d(z,null,null)},{default:function(){return[s("上传")]}})]}})]}}),d(y,x({label:"模板信息"},N.template_describe),{default:function(){return[d(C,{value:L.template_describe,"onUpdate:value":function(e){return L.template_describe=e}},null)]}})]}})]}}),d(h,{title:function(){return"报告配置生成"},visible:"report"==q.value,okText:"生成",cancelText:"取消",onCancel:function(){return q.value="none"},onOk:P,destroyOnClose:!1},{default:function(){return[d("h2",{style:{width:"100%",textAlign:"center",marginBottom:"20px"}},[s("请配置报告的类型及周期")]),d(c,null,{default:function(){return[d(y,{label:"报告类型"},{default:function(){return[d(T,{value:R.report_type,"onUpdate:value":function(e){return R.report_type=e}},{default:function(){return[d(S,{value:0},{default:function(){return[s("Word")]}}),d(S,{value:1},{default:function(){return[s("PDF")]}}),d(S,{value:2},{default:function(){return[s("HTML")]}})]}})]}}),d(y,{label:"报告周期"},{default:function(){return[d(T,{onChange:H,value:F.type,"onUpdate:value":function(e){return F.type=e}},{default:function(){return[d(S,{value:0},{default:function(){return[s("周报")]}}),d(S,{value:1},{default:function(){return[s("月报")]}}),d(S,{value:2},{default:function(){return[s("季报")]}}),d(S,{value:3},{default:function(){return[s("年报")]}}),d(S,{value:4},{default:function(){return[s("自定义")]}})]}})]}}),d(y,{label:"选择日期"},{default:function(){return[d(j,{format:"YYYY-MM-DD",value:F.range,"onUpdate:value":function(e){return F.range=e}},null)]}})]}})]}})])}}}))}}}));
static/assets/ProductTemplate.4e3e265e.js 0 → 100644
import{s as e,U as a,D as t}from"./index.f608f11e.js";import{d as l,o as n,J as d,r as s,a0 as r,F as o,c as i,b as u,W as p,X as m,B as c,Y as _,Z as v,V as f,Q as h,H as g,K as b,x as y,I as x,a1 as w,a2 as k,T as z,O as C,P as T,a3 as j,l as I,m as S,_ as U}from"./vendor.98dba853.js";const D={add:"新增",modify:"修改",report:"报告",none:""},O=l({name:"ProductDataSet",setup(){n((()=>{l(1,10)}));const l=(a,t)=>{e.dispatch("template/getTemplates",{page_size:t,page_index:a})},O=d([]),q=d("none"),Y=d([]),F=s({range:[r().add(-1,"week"),r()],type:0}),R=s({start_time:"",end_time:"",report_type:0}),H=e=>{switch(console.log("v::",e.target.value),e.target.value){case 0:F.range[0]=F.range[1].add(-1,"week");break;case 1:F.range[0]=F.range[1].add(-1,"month");break;case 2:F.range[0]=F.range[1].add(-3,"month");break;case 3:F.range[0]=F.range[1].add(-1,"year")}},L=s({id:0,user_id:0,project_name:"",datasource_name:"",template_name:"",template_path:"",template_describe:"",topic_name:""}),M=e=>{Y.value=e.fileList,"done"==e.file.status&&e.file.response.isHandleSuccess&&(L.template_path=e.file.response.resultData.file_path,Y.value=[])},P=()=>{R.start_time=F.range[0].toString(),R.end_time=F.range[1].toString();const e=I.stringify(R),a=document.createElement("a");a.href="/api/report?"+e,a.download=a.href,a.click(),a.remove(),S.success("报告生成成功")},A=s({project_name:[{required:!0,message:"请输入项目名称"}],datasource_name:[{required:!0,message:"请输入数据源名称"}],template_name:[{required:!0,message:"请输入报告模板名称"}],template_path:[{required:!0,message:"请上传报告模板"}],template_describe:[{required:!0,message:"请输入报告模板说明"}]}),{resetFields:B,validate:J,validateInfos:K,mergeValidateInfo:N}=o.useForm(L,A),Q=()=>{B(),Y.value=[],q.value="add"},V=a=>{console.log("delete item:: ",a);const t=a instanceof Array?a:[a.id];h.confirm({title:"确定要删除吗？",okText:"确定",cancelText:"取消",onOk:()=>{e.dispatch("template/deleteTemplates",{list_id:t})}})},W=[{title:"NO",dataIndex:"id",resizable:!0,width:50,customRender:({record:e,index:a})=>a+1},{title:"所属项目",dataIndex:"project_name",resizable:!0,width:200},{title:"数据源",dataIndex:"datasource_name",resizable:!0,width:200},{title:"报告名称",dataIndex:"template_name",resizable:!0,width:200,customRender:({record:e})=>i(g,null,[e.template_name,i(U,{title:"点击开始配置报告周期"},{default:()=>[i(c,{onClick:()=>{return a=e,R.template_id=a.id,void(q.value="report");var a},size:"small",type:"primary",ghost:!0},{default:()=>[u("生成报告")]})]})])},{title:"模板地址",dataIndex:"template_path",resizable:!0,width:200,customRender:({record:e})=>i(U,{title:"点击下载模板文件"},{default:()=>[i("a",{href:t+e.template_path,download:t+e.template_path},[e.template_path])]})},{title:"说明",dataIndex:"template_describe",resizable:!0,width:200},{title:"创建时间",dataIndex:"create_time",resizable:!0,width:200},{key:"action",title:"操作",width:80,customRender:({record:e})=>i(g,null,[i(c,{size:"small",type:"primary",style:{margin:"0 5px 5px 0"},onClick:()=>{return a=e,Y.value=[],L.project_name=a.project_name,L.datasource_name=a.datasource_name,L.template_name=a.template_name,L.template_path=a.template_path,L.template_describe=a.template_describe,L.id=a.id,void(q.value="modify");var a}},{default:()=>[u("修改")]}),i(c,{size:"small",type:"ghost",danger:!0,onClick:()=>V(e)},{default:()=>[u("删除")]})])}];return()=>i(g,null,[i("h2",{class:"title"},[u("报告模板配置管理"),i(p,{onSearch:()=>{},class:"title-search",placeholder:"请输入报告模板名称"},null)]),i(m,null,null),i("div",{class:"table"},[i(c,{type:"ghost",danger:!0,icon:i(_,null,null),class:"table-delete "+(0==O.value.length?"hidden":""),onClick:()=>V(O.value)},{default:()=>[u("删除")]}),i(c,{type:"primary",icon:i(v,null,null),class:"table-add",onClick:Q},{default:()=>[u("新增")]}),i(f,{rowKey:"id",bordered:!0,loading:e.state.template.loading,columns:W,dataSource:e.state.template.templates.rows,rowSelection:{onChange:e=>{O.value=e}},pagination:{showQuickJumper:!0,showSizeChanger:!0,pageSize:e.state.template.templates.pageSize,total:e.state.template.templates.total,current:e.state.template.templates.pageNo,onChange:(e,a)=>l(e,a)}},null)]),i(h,{title:()=>i(g,null,[i("strong",null,[D[q.value]]),u(" 报告模板配置")]),width:800,visible:"add"==q.value||"modify"==q.value,closable:!0,onOk:()=>{console.log("itemData:: ",L),J().then((()=>{0==L.id?e.dispatch("template/postTemplate",L):e.dispatch("template/putTemplate",L),q.value="none"})).catch((e=>{S.warn("表单数据填写有误，请检查后再次提交")}))},okText:"确定",cancelText:"取消",onCancel:()=>q.value="none",destroyOnClose:!1},{default:()=>[i(o,{name:"template"},{default:()=>[i(b,y({label:"所属项目"},K.project_name),{default:()=>[i(x,{value:L.project_name,"onUpdate:value":e=>L.project_name=e},null)]}),i(b,y({label:"数据源"},K.datasource_name),{default:()=>[i(x,{value:L.datasource_name,"onUpdate:value":e=>L.datasource_name=e},null)]}),i(b,y({label:"报告名称"},K.template_name),{default:()=>[i(x,{value:L.template_name,"onUpdate:value":e=>L.template_name=e},null)]}),i(b,y({label:"模板文件"},K.template_path),{default:()=>[i(x,{value:L.template_path,"onUpdate:value":e=>L.template_path=e,style:{width:"calc( 100% - 86px )"}},null)," ",i(w,{action:a,onChange:M,fileList:Y.value},{default:()=>[i(c,{icon:i(k,null,null)},{default:()=>[u("上传")]})]})]}),i(b,y({label:"模板信息"},K.template_describe),{default:()=>[i(z,{value:L.template_describe,"onUpdate:value":e=>L.template_describe=e},null)]})]})]}),i(h,{title:()=>"报告配置生成",visible:"report"==q.value,okText:"生成",cancelText:"取消",onCancel:()=>q.value="none",onOk:P,destroyOnClose:!1},{default:()=>[i("h2",{style:{width:"100%",textAlign:"center",marginBottom:"20px"}},[u("请配置报告的类型及周期")]),i(o,null,{default:()=>[i(b,{label:"报告类型"},{default:()=>[i(C,{value:R.report_type,"onUpdate:value":e=>R.report_type=e},{default:()=>[i(T,{value:0},{default:()=>[u("Word")]}),i(T,{value:1},{default:()=>[u("PDF")]}),i(T,{value:2},{default:()=>[u("HTML")]})]})]}),i(b,{label:"报告周期"},{default:()=>[i(C,{onChange:H,value:F.type,"onUpdate:value":e=>F.type=e},{default:()=>[i(T,{value:0},{default:()=>[u("周报")]}),i(T,{value:1},{default:()=>[u("月报")]}),i(T,{value:2},{default:()=>[u("季报")]}),i(T,{value:3},{default:()=>[u("年报")]}),i(T,{value:4},{default:()=>[u("自定义")]})]})]}),i(b,{label:"选择日期"},{default:()=>[i(j,{format:"YYYY-MM-DD",value:F.range,"onUpdate:value":e=>F.range=e},null)]})]})]})])}});export{O as default};
static/assets/index-legacy.26db9e69.js 0 → 100644
!function(){function t(t,e){var r=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),r.push.apply(r,n)}return r}function e(e){for(var n=1;n<arguments.length;n++){var c=null!=arguments[n]?arguments[n]:{};n%2?t(Object(c),!0).forEach((function(t){r(e,t,c[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(c)):t(Object(c)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(c,t))}))}return e}function r(t,e,r){return e in t?Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}):t[e]=r,t}System.register(["./vendor-legacy.bbc7855f.js","./index-legacy.90883ee8.js"],(function(t){"use strict";var r,n,c,u,i,o,l,a;return{setters:[function(t){r=t.c,n=t.E,c=t.G,u=t.d,i=t.o,o=t.R,l=t.H},function(t){a=t.s}],execute:function(){var p=[{title:"模板管理",path:"/product-template",icon:r(n,null,null),children:[{title:"模板管理",path:"/product/template"}]},{title:"数据源配置",path:"/product-dataset",icon:r(c,null,null),children:[{title:"API数据集配置",path:"/product/urldataset"},{title:"SQL数据集配置",path:"/product/sqldataset"}]}];t("default",u({name:"app",setup:function(){return i((function(){a.commit("app/set_sider",e(e({},a.state.app.sider),{},{hidden:!1,menus:p,title:"报告配置管理"}))})),function(){return r(l,null,[r(o,null,null)])}}}))}}}))}();
static/assets/index-legacy.6d01cfcd.js 0 → 100644
!function(){var e=document.createElement("style");e.innerHTML='.header{width:100%;height:64px;background:white;box-shadow:0 0 5px 2px #ddd;line-height:64px;display:flex;padding:0 25px;position:fixed;z-index:1;top:0}.header-logo{margin-right:40px}.header-logo-img{width:56px;height:56px;position:relative;margin-right:6px;font-size:45px}.header-logo-title{font-size:24px}.header-menu{display:flex;height:64px;line-height:64px;font-size:18px;margin:0 20px}.header-menu .menuItem{color:#2086ff;margin:0 16px;position:relative;transition:all .4s}.header-menu .menuItem:before{content:" ";position:absolute;width:0;border-radius:4px;height:3px;background-color:#2086ff;box-sizing:border-box;left:0;right:0;top:52px;transition:width .4s;margin:auto}.header-menu .menuItem.active,.header-menu .menuItem:hover{color:#0046ef}.header-menu .menuItem.active:before,.header-menu .menuItem:hover:before{background-color:#0046ef;display:block;width:50%}.header-user{margin-left:auto;font-size:18px;padding:0 15px;min-width:136px;text-align:center}.header-user>a{color:#2086ff;transition:all .3s}.header-user>a:hover{color:#0046ef}.header-user>a:hover>.circle{border-color:#0046ef}.header-user>a>.circle{padding:2px;border-radius:50%;border:2px solid #2086ff;position:relative;margin-right:6px}.header-user-menu>.ant-dropdown-menu-item{padding:6px 40px;color:#2086ff}.header-user-menu>.ant-dropdown-menu-item:hover{color:#0046ef}.siderbar-menu{padding:20px 0}.siderbar-menu .ant-menu{border-right:none;text-align:center;font-weight:400;font-size:15px;transition:all .4s}.siderbar-menu .ant-menu .ant-menu-submenu{margin-bottom:5px}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-submenu-title{padding-right:12px}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-title-content,.siderbar-menu .ant-menu .ant-menu-item .ant-menu-title-content{margin-right:8px}.siderbar-menu .ant-menu .ant-menu-submenu.ant-menu-item-selected,.siderbar-menu .ant-menu .ant-menu-item.ant-menu-item-selected,.siderbar-menu .ant-menu .ant-menu-submenu:hover,.siderbar-menu .ant-menu .ant-menu-item:hover{font-weight:400;background-color:#def}.siderbar-menu .ant-menu .ant-menu-submenu.ant-menu-item-selected .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item.ant-menu-item-selected .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-submenu:hover .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item:hover .ant-menu-item-icon{border:1px solid #2186ff}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item .ant-menu-item-icon{transition:all .3s;border:1px solid gray;font-weight:600;width:30px;height:30px;line-height:30px;border-radius:50%}.siderbar-menu .ant-menu .ant-menu-sub.ant-menu-inline>.ant-menu-item,.siderbar-menu .ant-menu .ant-menu-sub.ant-menu-inline>.ant-menu-submenu>.ant-menu-submenu-title{margin:0}.siderbar-menu-collapsed{position:absolute;bottom:0;right:0}.home{display:flex;padding-top:64px;width:100%;box-sizing:border-box}.home-content{width:80%;overflow-y:auto;overflow-x:hidden;margin-left:20%;padding:8px;min-height:calc(100vh - 68px)}.home-content-div{width:100%;height:100%;background:white;padding:16px}.home-siderbar{width:20%;background:white;text-align:center;height:calc(100vh - 64px);box-sizing:border-box;position:fixed}.home-info{width:20%;margin:12px 12px 12px 0;background:white;padding:24px}.loading-page{width:100%;height:100%;display:flex;flex-direction:column;justify-content:center;align-items:center;padding:20%}\n',document.head.appendChild(e),System.register(["./vendor-legacy.bbc7855f.js","./index-legacy.90883ee8.js"],(function(e){"use strict";var n,t,i,r,a,u,o,d,s,l,m,c,h,p,f,g,b,x,v,y,w,K;return{setters:[function(e){n=e.k,t=e.u,i=e.n,r=e.c,a=e.S,u=e.j,o=e.D,d=e.U,s=e.p,l=e.M,m=e.q,c=e.s,h=e.b,p=e.d,f=e.r,g=e.t,b=e.v,x=e.w,v=e.x,y=e.y,w=e.R},function(e){K=e.s}],execute:function(){var k=function(e){var p=n(),f=t(),g=i((function(){return p.path})),b=i((function(){return"string"==typeof e.logo?r("img",{src:e.logo,alt:"logo",class:"header-logo-img"},null):e.logo}));return r("div",{class:"header"},[r("div",{class:"header-logo"},[b.value||r(a,{class:"header-logo-img"},null),r("span",{class:"header-logo-title"},[e.title])]),r("div",{class:"header-menu"},[e.menus&&e.menus.map((function(e){return r(u,{to:e.path,class:g.value.startsWith(e.path)?"menuItem active":"menuItem"},{default:function(){return[e.title]}})}))]),e.hiddenUser?e.rightNode:r("div",{class:"header-user"},[r(o,{trigger:["click","hover"]},{default:function(){return[r("a",{class:"ant-dropdown-link"},[r(d,{class:"circle"},null),e.user&&e.user.info?e.user.info.username:""," ",r(s,null,null)])]},overlay:function(){return r(l,{class:"header-user-menu"},{default:function(){return[e.userLinks&&e.userLinks.map((function(e){return r(m,{onClick:function(){return f.push(e.path)}},{default:function(){return[e.title]}})})),r(c,null,null),r(m,{onClick:function(){return e.user&&e.user.logout&&e.user.logout()}},{default:function(){return[h("退出")]}})]}})}})])])};var j=p({name:"SiderBar",props:{menus:{type:Object,require:!1},classNames:{type:Array,required:!1},hidden:{type:Boolean,required:!1},width:{type:Number,required:!1},theme:{type:String,required:!1},collapsed:{type:Boolean,required:!1},title:{type:String,required:!1}},setup:function(e,i){var a,u=i.slots;console.log("props:: ",e);var o=t(),d=n(),s=f({rootSubmenuKeys:(null===(a=e.menus)||void 0===a?void 0:a.filter((function(e){return e.children&&e.children.length})).map((function(e){return e.path})))||[],openKeys:[],selectedKeys:[]});g((function(){if(!(s.selectedKeys.length>0||s.openKeys.length>0)){var n=e.menus?function e(n){var t=n.find((function(n){return d.path==n.path||n.children&&n.children.length&&e(n.children)}));if(t)return(null==t?void 0:t.path)==d.path?t:(s.openKeys=[t.path],e(null==t?void 0:t.children))}(e.menus):null,t=n&&n.path?[n.path]:[];t.toString()!=s.selectedKeys.toString()&&(s.selectedKeys=t)}}));var c=function(e){var n=e.find((function(e){return-1===s.openKeys.indexOf(e)}));-1===s.rootSubmenuKeys.indexOf(n)?s.openKeys=e:s.openKeys=n?[n]:[]};console.log("selectedKeys",d.path,s.selectedKeys,e.collapsed);var h=function e(n){var t;if(n&&n.children&&n.children.length)return r(x,{title:n.title,icon:n.icon,key:n.path},"function"==typeof(i=t=n.children.map((function(n){return e(n)})))||"[object Object]"===Object.prototype.toString.call(i)&&!b(i)?t:{default:function(){return[t]}});var i,a=n;return r(m,v({key:a.path},a,{onClick:function(){a.path&&(s.selectedKeys=[a.path],o.push(a.path))}}),{default:function(){return[a.title]}})};return function(){return r("div",{class:"siderbar-menu"+(e.classNames?e.classNames.join(" "):""),style:{width:e.width||200,display:e.hidden?"none":"block"}},[r("h3",{style:{color:"dark"==e.theme?"#ffffff":"#212121"}},[e.title]),e.menus?r(l,{style:{width:e.width},mode:"inline",selectedKeys:s.selectedKeys,"onUpdate:selectedKeys":function(e){return s.selectedKeys=e},openKeys:s.openKeys,"onUpdate:openKeys":function(e){return s.openKeys=e},onOpenChange:c,selectable:!0,theme:e.theme,inlineCollapsed:e.collapsed},{default:function(){return[e.menus&&e.menus.map((function(e){return h(e)}))]}}):null,u.default?u.default():null])}}}),S=function(e){var n=i((function(){var n=e.sider||{};return n.hidden?{content:{width:"100%",marginLeft:0},sider:{display:"none",width:0}}:{content:{width:"calc( 100% - ".concat(n.width||200,"px )"),marginLeft:"".concat(n.width||200,"px")},sider:{display:"block",width:"".concat(n.width||200,"px")}}}));return r("div",{class:e.classNames?e.classNames.join(" "):""},[r(k,e.header,null),r("div",{class:"home"},[r("div",{class:"home-siderbar",style:n.value.sider},[r(j,e.sider,null)]),r("div",{class:"home-content",style:n.value.content},[r("div",{class:"home-content-div"},[r(w,{name:"default"},null)])])])])};e("default",p({setup:function(){return K.dispatch("app/getCurrent"),function(){return K.state.app.loading?r("div",{class:"loading-page"},[r(y,null,null)]):r(S,{sider:K.state.app.sider,header:K.state.app.header},null)}}}))}}}))}();
static/assets/index-legacy.90883ee8.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
static/assets/index-legacy.b5824f36.js 0 → 100644
!function(){function e(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function t(t){for(var n=1;n<arguments.length;n++){var c=null!=arguments[n]?arguments[n]:{};n%2?e(Object(c),!0).forEach((function(e){r(t,e,c[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(c)):e(Object(c)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(c,e))}))}return t}function r(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}System.register(["./index-legacy.90883ee8.js","./vendor-legacy.bbc7855f.js"],(function(e){"use strict";var r,n,c,o,i;return{setters:[function(e){r=e.s},function(e){n=e.d,c=e.c,o=e.b,i=e.H}],execute:function(){e("default",n({name:"app",setup:function(){return r.commit("app/set_sider",t(t({},r.state.app.sider),{},{hidden:!0})),function(){return c(i,null,[c("h1",null,[o("这是About页面")])])}}}))}}}))}();
static/assets/index.2960128b.js 0 → 100644
import{k as e,u as s,n as t,c as l,S as n,j as a,D as d,U as o,p as i,M as r,q as c,s as u,b as p,d as h,r as m,t as y,v as f,w as g,x as v,y as K,R as b}from"./vendor.98dba853.js";import{s as w}from"./index.f608f11e.js";const k=h=>{const m=e(),y=s(),f=t((()=>m.path)),g=t((()=>"string"==typeof h.logo?l("img",{src:h.logo,alt:"logo",class:"header-logo-img"},null):h.logo));return l("div",{class:"header"},[l("div",{class:"header-logo"},[g.value||l(n,{class:"header-logo-img"},null),l("span",{class:"header-logo-title"},[h.title])]),l("div",{class:"header-menu"},[h.menus&&h.menus.map((e=>l(a,{to:e.path,class:f.value.startsWith(e.path)?"menuItem active":"menuItem"},{default:()=>[e.title]})))]),h.hiddenUser?h.rightNode:l("div",{class:"header-user"},[l(d,{trigger:["click","hover"]},{default:()=>[l("a",{class:"ant-dropdown-link"},[l(o,{class:"circle"},null),h.user&&h.user.info?h.user.info.username:""," ",l(i,null,null)])],overlay:()=>l(r,{class:"header-user-menu"},{default:()=>[h.userLinks&&h.userLinks.map((e=>l(c,{onClick:()=>y.push(e.path)},{default:()=>[e.title]}))),l(u,null,null),l(c,{onClick:()=>h.user&&h.user.logout&&h.user.logout()},{default:()=>[p("退出")]})]})})])])};const j=h({name:"SiderBar",props:{menus:{type:Object,require:!1},classNames:{type:Array,required:!1},hidden:{type:Boolean,required:!1},width:{type:Number,required:!1},theme:{type:String,required:!1},collapsed:{type:Boolean,required:!1},title:{type:String,required:!1}},setup:(t,{slots:n})=>{var a;console.log("props:: ",t);const d=s(),o=e(),i=m({rootSubmenuKeys:(null==(a=t.menus)?void 0:a.filter((e=>e.children&&e.children.length)).map((e=>e.path)))||[],openKeys:[],selectedKeys:[]});y((()=>{if(i.selectedKeys.length>0||i.openKeys.length>0)return;const e=s=>{const t=s.find((s=>o.path==s.path||s.children&&s.children.length&&e(s.children)));if(t)return(null==t?void 0:t.path)==o.path?t:(i.openKeys=[t.path],e(null==t?void 0:t.children))},s=t.menus?e(t.menus):null,l=s&&s.path?[s.path]:[];l.toString()!=i.selectedKeys.toString()&&(i.selectedKeys=l)}));const u=e=>{const s=e.find((e=>-1===i.openKeys.indexOf(e)));-1===i.rootSubmenuKeys.indexOf(s)?i.openKeys=e:i.openKeys=s?[s]:[]};console.log("selectedKeys",o.path,i.selectedKeys,t.collapsed);const p=e=>{if(e&&e.children&&e.children.length){let t;return l(g,{title:e.title,icon:e.icon,key:e.path},"function"==typeof(s=t=e.children.map((e=>p(e))))||"[object Object]"===Object.prototype.toString.call(s)&&!f(s)?t:{default:()=>[t]})}{const s=e;return l(c,v({key:s.path},s,{onClick:()=>{s.path&&(i.selectedKeys=[s.path],d.push(s.path))}}),{default:()=>[s.title]})}var s};return()=>l("div",{class:"siderbar-menu"+(t.classNames?t.classNames.join(" "):""),style:{width:t.width||200,display:t.hidden?"none":"block"}},[l("h3",{style:{color:"dark"==t.theme?"#ffffff":"#212121"}},[t.title]),t.menus?l(r,{style:{width:t.width},mode:"inline",selectedKeys:i.selectedKeys,"onUpdate:selectedKeys":e=>i.selectedKeys=e,openKeys:i.openKeys,"onUpdate:openKeys":e=>i.openKeys=e,onOpenChange:u,selectable:!0,theme:t.theme,inlineCollapsed:t.collapsed},{default:()=>[t.menus&&t.menus.map((e=>p(e)))]}):null,n.default?n.default():null])}});const S=e=>{const s=t((()=>{const s=e.sider||{};return s.hidden?{content:{width:"100%",marginLeft:0},sider:{display:"none",width:0}}:{content:{width:`calc( 100% - ${s.width||200}px )`,marginLeft:`${s.width||200}px`},sider:{display:"block",width:`${s.width||200}px`}}}));return l("div",{class:e.classNames?e.classNames.join(" "):""},[l(k,e.header,null),l("div",{class:"home"},[l("div",{class:"home-siderbar",style:s.value.sider},[l(j,e.sider,null)]),l("div",{class:"home-content",style:s.value.content},[l("div",{class:"home-content-div"},[l(b,{name:"default"},null)])])])])};var q=h({setup:()=>(w.dispatch("app/getCurrent"),()=>w.state.app.loading?l("div",{class:"loading-page"},[l(K,null,null)]):l(S,{sider:w.state.app.sider,header:w.state.app.header},null))});export{q as default};
static/assets/index.4d155483.css 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
static/assets/index.60ff5ce2.js 0 → 100644
var t=Object.defineProperty,e=Object.defineProperties,r=Object.getOwnPropertyDescriptors,a=Object.getOwnPropertySymbols,l=Object.prototype.hasOwnProperty,p=Object.prototype.propertyIsEnumerable,o=(e,r,a)=>r in e?t(e,r,{enumerable:!0,configurable:!0,writable:!0,value:a}):e[r]=a;import{c as s,E as n,G as i,d as c,o as d,R as u,H as m}from"./vendor.98dba853.js";import{s as b}from"./index.f608f11e.js";const f=[{title:"模板管理",path:"/product-template",icon:s(n,null,null),children:[{title:"模板管理",path:"/product/template"}]},{title:"数据源配置",path:"/product-dataset",icon:s(i,null,null),children:[{title:"API数据集配置",path:"/product/urldataset"},{title:"SQL数据集配置",path:"/product/sqldataset"}]}],h=c({name:"app",setup:()=>(d((()=>{var t;b.commit("app/set_sider",(t=((t,e)=>{for(var r in e||(e={}))l.call(e,r)&&o(t,r,e[r]);if(a)for(var r of a(e))p.call(e,r)&&o(t,r,e[r]);return t})({},b.state.app.sider),e(t,r({hidden:!1,menus:f,title:"报告配置管理"}))))})),()=>s(m,null,[s(u,null,null)]))});export{h as default};
static/assets/index.63269ceb.css 0 → 100644
.header{width:100%;height:64px;background:white;box-shadow:0 0 5px 2px #ddd;line-height:64px;display:flex;padding:0 25px;position:fixed;z-index:1;top:0}.header-logo{margin-right:40px}.header-logo-img{width:56px;height:56px;position:relative;margin-right:6px;font-size:45px}.header-logo-title{font-size:24px}.header-menu{display:flex;height:64px;line-height:64px;font-size:18px;margin:0 20px}.header-menu .menuItem{color:#2086ff;margin:0 16px;position:relative;transition:all .4s}.header-menu .menuItem:before{content:" ";position:absolute;width:0;border-radius:4px;height:3px;background-color:#2086ff;box-sizing:border-box;left:0;right:0;top:52px;transition:width .4s;margin:auto}.header-menu .menuItem.active,.header-menu .menuItem:hover{color:#0046ef}.header-menu .menuItem.active:before,.header-menu .menuItem:hover:before{background-color:#0046ef;display:block;width:50%}.header-user{margin-left:auto;font-size:18px;padding:0 15px;min-width:136px;text-align:center}.header-user>a{color:#2086ff;transition:all .3s}.header-user>a:hover{color:#0046ef}.header-user>a:hover>.circle{border-color:#0046ef}.header-user>a>.circle{padding:2px;border-radius:50%;border:2px solid #2086ff;position:relative;margin-right:6px}.header-user-menu>.ant-dropdown-menu-item{padding:6px 40px;color:#2086ff}.header-user-menu>.ant-dropdown-menu-item:hover{color:#0046ef}.siderbar-menu{padding:20px 0}.siderbar-menu .ant-menu{border-right:none;text-align:center;font-weight:400;font-size:15px;transition:all .4s}.siderbar-menu .ant-menu .ant-menu-submenu{margin-bottom:5px}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-submenu-title{padding-right:12px}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-title-content,.siderbar-menu .ant-menu .ant-menu-item .ant-menu-title-content{margin-right:8px}.siderbar-menu .ant-menu .ant-menu-submenu.ant-menu-item-selected,.siderbar-menu .ant-menu .ant-menu-item.ant-menu-item-selected,.siderbar-menu .ant-menu .ant-menu-submenu:hover,.siderbar-menu .ant-menu .ant-menu-item:hover{font-weight:400;background-color:#def}.siderbar-menu .ant-menu .ant-menu-submenu.ant-menu-item-selected .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item.ant-menu-item-selected .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-submenu:hover .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item:hover .ant-menu-item-icon{border:1px solid #2186ff}.siderbar-menu .ant-menu .ant-menu-submenu .ant-menu-item-icon,.siderbar-menu .ant-menu .ant-menu-item .ant-menu-item-icon{transition:all .3s;border:1px solid gray;font-weight:600;width:30px;height:30px;line-height:30px;border-radius:50%}.siderbar-menu .ant-menu .ant-menu-sub.ant-menu-inline>.ant-menu-item,.siderbar-menu .ant-menu .ant-menu-sub.ant-menu-inline>.ant-menu-submenu>.ant-menu-submenu-title{margin:0}.siderbar-menu-collapsed{position:absolute;bottom:0;right:0}.home{display:flex;padding-top:64px;width:100%;box-sizing:border-box}.home-content{width:80%;overflow-y:auto;overflow-x:hidden;margin-left:20%;padding:8px;min-height:calc(100vh - 68px)}.home-content-div{width:100%;height:100%;background:white;padding:16px}.home-siderbar{width:20%;background:white;text-align:center;height:calc(100vh - 64px);box-sizing:border-box;position:fixed}.home-info{width:20%;margin:12px 12px 12px 0;background:white;padding:24px}.loading-page{width:100%;height:100%;display:flex;flex-direction:column;justify-content:center;align-items:center;padding:20%}
static/assets/index.c36fa510.js 0 → 100644
var e=Object.defineProperty,r=Object.defineProperties,t=Object.getOwnPropertyDescriptors,a=Object.getOwnPropertySymbols,o=Object.prototype.hasOwnProperty,p=Object.prototype.propertyIsEnumerable,s=(r,t,a)=>t in r?e(r,t,{enumerable:!0,configurable:!0,writable:!0,value:a}):r[t]=a;import{s as n}from"./index.f608f11e.js";import{d as i,c as l,b,H as c}from"./vendor.98dba853.js";const f=i({name:"app",setup(){var e;return n.commit("app/set_sider",(e=((e,r)=>{for(var t in r||(r={}))o.call(r,t)&&s(e,t,r[t]);if(a)for(var t of a(r))p.call(r,t)&&s(e,t,r[t]);return e})({},n.state.app.sider),r(e,t({hidden:!0})))),()=>l(c,null,[l("h1",null,[b("这是About页面")])])}});export{f as default};
static/assets/index.ec330889.css 0 → 100644
.login{width:380px;height:280px;background-color:#fff;border:1px solid #ddd;border-radius:4px;padding:30px 35px;box-shadow:0 0 4px 1px #777;box-sizing:border-box;position:absolute;top:0;bottom:0;right:80px;margin:auto}.login-forget{font-size:14px;float:right;margin-top:15px}.login>h1{font-size:22px;color:#717171;font-weight:700}.login>h1>span{font-weight:500}
static/assets/index.f608f11e.js 0 → 100644
var e=Object.defineProperty,t=Object.defineProperties,a=Object.getOwnPropertyDescriptors,s=Object.getOwnPropertySymbols,o=Object.prototype.hasOwnProperty,n=Object.prototype.propertyIsEnumerable,r=(t,a,s)=>a in t?e(t,a,{enumerable:!0,configurable:!0,writable:!0,value:s}):t[a]=s,i=(e,t)=>{for(var a in t||(t={}))o.call(t,a)&&r(e,a,t[a]);if(s)for(var a of s(t))n.call(t,a)&&r(e,a,t[a]);return e},d=(e,s)=>t(e,a(s));export function __vite_legacy_guard(){import("data:text/javascript,")}import{a as l,N as c,l as p,d as u,c as m,R as h,b as g,e as _,f,m as y,g as D,h as b,o as v,C as S,z as T,i as q,A as P}from"./vendor.98dba853.js";!function(){const e=document.createElement("link").relList;if(!(e&&e.supports&&e.supports("modulepreload"))){for(const e of document.querySelectorAll('link[rel="modulepreload"]'))t(e);new MutationObserver((e=>{for(const a of e)if("childList"===a.type)for(const e of a.addedNodes)"LINK"===e.tagName&&"modulepreload"===e.rel&&t(e)})).observe(document,{childList:!0,subtree:!0})}function t(e){if(e.ep)return;e.ep=!0;const t=function(e){const t={};return e.integrity&&(t.integrity=e.integrity),e.referrerpolicy&&(t.referrerPolicy=e.referrerpolicy),"use-credentials"===e.crossorigin?t.credentials="include":"anonymous"===e.crossorigin?t.credentials="omit":t.credentials="same-origin",t}(e);fetch(e.href,t)}}(),l.defaults.baseURL=window.location.origin,l.defaults.timeout=1e4,l.defaults.headers.post["Content-Type"]="application/json;charset=UTF-8",l.defaults.headers.put["Content-Type"]="application/json;charset=UTF-8",l.defaults.headers.delete["Content-Type"]="application/json;charset=UTF-8",l.interceptors.request.use((e=>{const t=localStorage.getItem("Username");return t&&(e.headers.token=t),e}),(e=>e)),l.interceptors.response.use((e=>(111===e.data.code&&sessionStorage.removeItem("token"),e)));const L={get:(e,t)=>new Promise(((a,s)=>{c.start(),l.get(e+"?"+p.stringify(t)).then((e=>{c.done(),a(e.data)})).catch((e=>{c.done(),s(e.data)}))})),post:(e,t)=>new Promise(((a,s)=>{c.start(),l.post(e,JSON.stringify(t)).then((e=>{c.done(),a(e.data)})).catch((e=>{c.done(),s(e.data)}))})),put:(e,t)=>new Promise(((a,s)=>{c.start(),l.put(e,JSON.stringify(t)).then((e=>{c.done(),a(e.data)})).catch((e=>{c.done(),s(e.data)}))})),delete:(e,t)=>new Promise(((a,s)=>{c.start(),l.delete(e,{data:t}).then((e=>{c.done(),a(e.data)})).catch((e=>{c.done(),s(e.data)}))})),upload:(e,t)=>new Promise(((a,s)=>{c.start(),l.post(e,t,{headers:{"Content-Type":"multipart/form-data"}}).then((e=>{c.done(),a(e.data)})).catch((e=>{c.done(),s(e.data)}))})),download(e){const t=document.createElement("iframe");t.style.display="none",t.src=e,t.onload=function(){document.body.removeChild(t)},document.body.appendChild(t)}};function E(e){return L.post("/auth/login",e)}const F={},I=function(e,t){return t&&0!==t.length?Promise.all(t.map((e=>{if((e=`/${e}`)in F)return;F[e]=!0;const t=e.endsWith(".css"),a=t?'[rel="stylesheet"]':"";if(document.querySelector(`link[href="${e}"]${a}`))return;const s=document.createElement("link");return s.rel=t?"stylesheet":"modulepreload",t||(s.as="script",s.crossOrigin=""),s.href=e,document.head.appendChild(s),t?new Promise(((e,t)=>{s.addEventListener("load",e),s.addEventListener("error",t)})):void 0}))).then((()=>e())):e()};const O=e=>m("div",{class:"auth"},[m("div",{class:"auth-title"},[m("h1",null,[e.slogon]),m("h2",null,[e.subtitle])]),m("div",{class:"auth-form"},[m(h,null,null)]),m("div",{class:"auth-footer"},[e.compony&&e.compony.name,g(" © 版权所有"),m("p",null,[e.beian?m("a",null,[e.beian]):null,g("    ")," ",e.compony&&e.compony.phone?`联系电话 : ${e.compony.phone}`:null])])]);const j=[{path:"/auth",name:"auth",component:u({setup:()=>()=>m(O,V.state.app.basic,null)}),children:[{path:"/login",name:"login",component:()=>I((()=>import("./Login.10fa1365.js")),["assets/Login.10fa1365.js","assets/index.ec330889.css","assets/vendor.98dba853.js"])},{path:"/forget",name:"forget",component:()=>I((()=>import("./Forget.2626476b.js")),["assets/Forget.2626476b.js","assets/index.ec330889.css","assets/vendor.98dba853.js"])}]},{path:"/",name:"index",component:()=>I((()=>import("./index.2960128b.js")),["assets/index.2960128b.js","assets/index.63269ceb.css","assets/vendor.98dba853.js"]),redirect:e=>V.state.app.isLogin?"/product":"/login",children:[{path:"/product",name:"product",component:()=>I((()=>import("./index.60ff5ce2.js")),["assets/index.60ff5ce2.js","assets/vendor.98dba853.js"]),meta:{requiredAuth:!0},children:[{path:"urldataset",name:"URL数据集",component:()=>I((()=>import("./ProductDataSet.2eb44551.js")),["assets/ProductDataSet.2eb44551.js","assets/ProductDataSet.daa66c6c.css","assets/vendor.98dba853.js"])},{path:"sqldataset",name:"SQL数据集",component:()=>I((()=>import("./ProductDataSet.2eb44551.js")),["assets/ProductDataSet.2eb44551.js","assets/ProductDataSet.daa66c6c.css","assets/vendor.98dba853.js"])},{path:"datasetField",name:"datasetField",component:()=>I((()=>import("./ProductDataSetField.4ae51f79.js")),["assets/ProductDataSetField.4ae51f79.js","assets/vendor.98dba853.js"])},{path:"template",name:"template",component:()=>I((()=>import("./ProductTemplate.4e3e265e.js")),["assets/ProductTemplate.4e3e265e.js","assets/vendor.98dba853.js"])}]},{path:"/about",name:"about",component:()=>I((()=>import("./index.c36fa510.js")),["assets/index.c36fa510.js","assets/vendor.98dba853.js"]),meta:{requiredAuth:!0}}]}],w={},H=(e,t)=>{for(const a of e){const e=a.path.startsWith("/")?a.path:t+"/"+a.path;w[e]=a.name,a.children&&a.children.length>0&&H(a.children,e)}};H(j,"");const A=_({history:f(),routes:j});A.beforeEach(((e,t)=>{for(const a of e.matched){const t=a.meta;if(t&&t.requiredAuth&&!V.state.app.isLogin)return{path:"/login",query:{redirect:e.fullPath}}}})),console.log(localStorage.getItem("Username"));const R=e=>L.get("/api/database",e);const U="/api/file?file_path=",C="/api/file";var V=D({modules:{app:{namespaced:!0,state:{loading:!1,sider:{},header:{logo:"",title:"",user:{}},isLogin:!1,basic:{},siderWidth:0},getters:{loading:e=>e.loading,isLogin:e=>e.isLogin,userInfo:e=>e.userInfo},mutations:{set_loading:(e,t)=>{e.loading=t},set_config:(e,t)=>{e.isLogin=!!localStorage.getItem("Username"),e.basic=t.basic,e.header=d(i({},t.header),{theme:t.theme}),e.sider=d(i({},t.sider),{theme:t.theme}),e.siderWidth=t.sider.width||200},set_sider_collapsed:(e,t)=>{console.log("payload",t);const a=t?64:e.siderWidth;e.sider.width=a,e.sider.collapsed=t},set_sider:(e,t)=>{console.log("Sider数据: ",t),e.sider=t},set_header:(e,t)=>{console.log("Header数据: ",t),e.header=t},set_userInfo:(e,t)=>{e.userInfo=t,t&&t.username?(localStorage.setItem("Username",t.username),e.isLogin=!0):(localStorage.removeItem("Username"),e.isLogin=!1)}},actions:{login:({commit:e,state:t,dispatch:a},s)=>{E(s).then((a=>{var s;a.isHandleSuccess?(e("set_userInfo",a.resultData),e("set_header",d(i({},t.header),{user:d(i({},t.header.user),{info:a.resultData})})),y.success("登录成功"),A.replace("/product")):y.warn("Error:: "+(null==(s=a.handleMsg)?void 0:s.toString()))}))},logout:({commit:e,state:t},a)=>{L.delete("/auth/logout").then((a=>{a.isHandleSuccess&&(e("set_userInfo",void 0),e("set_header",d(i({},t.header),{user:d(i({},t.header.user),{info:void 0})})),y.success("退出成功"),setTimeout((()=>{location.reload()}),100))}))},getCurrent:({commit:e,state:t,dispatch:a},s)=>{e("set_loading",!0),L.get("/auth/current").then((s=>{e("set_loading",!1),s.isHandleSuccess&&(e("set_userInfo",s.resultData),e("set_header",d(i({},t.header),{user:d(i({},t.header.user),{info:s.resultData,logout:()=>a("app/logout")})})))}))}}},dataset:{namespaced:!0,state:{queryDataset:{dataset_source_type:0},queryDatasetField:{},loading:!1,datasets:{rows:[],pageNo:1,pageSize:10,total:0},datasetsFields:[]},getters:{loading:e=>e.loading},mutations:{set_loading:(e,t)=>{e.loading=t},set_datasets:(e,t)=>{e.datasets=t},set_queryDataset:(e,t)=>{e.queryDataset=t},set_datasetFields:(e,t)=>{e.datasetsFields=t},set_queryDatasetField:(e,t)=>{e.queryDatasetField=t}},actions:{getDatasets:({commit:e,state:t,dispatch:a},s)=>{var o;e("set_loading",!0),e("set_queryDataset",s),(o=s,L.get("/api/dataset",o)).then((t=>{e("set_loading",!1),t.isHandleSuccess&&e("set_datasets",t.resultData)}))},postDataset:({commit:e,state:t,dispatch:a},s)=>{var o;e("set_loading",!0),(o=s,L.post("/api/dataset",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasets",t.queryDataset,{root:!0}),y.success("增加数据集成功"))}))},putDataset:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.put("/api/dataset",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasets",t.queryDataset,{root:!0}),y.success("修改数据集成功"))}))},deleteDatasets:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.delete("/api/dataset",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasets",t.queryDataset,{root:!0}),y.success("删除数据集成功"))}))},getDatasetFields:({commit:e,state:t,dispatch:a},s)=>{var o;e("set_datasetFields",[]),e("set_loading",!0),e("set_queryDatasetField",s),(o=s,L.get("/api/dataset_field",o)).then((t=>{t.isHandleSuccess&&e("set_datasetFields",t.resultData),e("set_loading",!1)}))},postDatasetField:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.post("/api/dataset_field",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasetFields",t.queryDatasetField,{root:!0}),y.success("增加数据集字段成功"))}))},putDatasetField:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.put("/api/dataset_field",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasetFields",t.queryDatasetField,{root:!0}),y.success("修改数据集字段成功"))}))},deleteDatasetFields:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.delete("/api/dataset_field",o)).then((e=>{e.isHandleSuccess&&(a("dataset/getDatasetFields",t.queryDatasetField,{root:!0}),y.success("删除数据集字段成功"))}))}}},template:{namespaced:!0,state:{queryTemplate:{},loading:!1,templates:{rows:[],pageNo:1,pageSize:10,total:0}},getters:{loading:e=>e.loading},mutations:{set_loading:(e,t)=>{e.loading=t},set_templates:(e,t)=>{e.templates=t},set_queryTemplate:(e,t)=>{e.queryTemplate=t}},actions:{getTemplates:({commit:e,state:t,dispatch:a},s)=>{var o;e("set_loading",!0),e("set_queryTemplate",s),(o=s,L.get("/api/template",o)).then((t=>{e("set_loading",!1),t.isHandleSuccess&&e("set_templates",t.resultData)}))},postTemplate:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.post("/api/template",o)).then((e=>{e.isHandleSuccess&&(a("template/getTemplates",t.queryTemplate,{root:!0}),y.success("增加报告模板成功"))}))},putTemplate:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.put("/api/template",o)).then((e=>{e.isHandleSuccess&&(a("template/getTemplates",t.queryTemplate,{root:!0}),y.success("修改报告模板成功"))}))},deleteTemplates:({commit:e,state:t,dispatch:a},s)=>{var o;(o=s,L.delete("/api/template",o)).then((e=>{e.isHandleSuccess&&(a("template/getTemplates",t.queryTemplate,{root:!0}),y.success("删除报告模板成功"))}))}}}},strict:!1,plugins:[]});const N={header:{title:"报告系统",logo:"/assets/logo.png"},theme:"dark",sider:{width:200},basic:{slogon:"人工智能 数据先行",subtitle:"让信息产生价值，让数字产生效益",beian:"豫ICP: 123456",compony:{name:"郑州数能软件有限公司",phone:"0371-123456"}}},k=u({name:"app",setup:()=>(v((()=>{V.commit("app/set_config",N)})),()=>m("div",{class:"theme-blue"},[m(S,{locale:T},{default:()=>[m(h,null,null)]})]))});(async()=>{const e=q(k);e.use(P),e.use(A),e.use(V),e.mount("#app")})();export{U as D,C as U,E as l,A as r,V as s,R as t};
static/assets/polyfills-legacy.34d9b402.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
static/assets/vendor-legacy.bbc7855f.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
static/assets/vendor.98dba853.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
static/index.html 0 → 100644
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" href="/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite App</title>
<script type="module" crossorigin src="/assets/index.f608f11e.js"></script>
<link rel="modulepreload" href="/assets/vendor.98dba853.js">
<link rel="stylesheet" href="/assets/index.4d155483.css">
<script type="module">!function(){try{new Function("m","return import(m)")}catch(o){console.warn("vite: loading legacy build because dynamic import is unsupported, syntax error above should be ignored");var e=document.getElementById("vite-legacy-polyfill"),n=document.createElement("script");n.src=e.src,n.onload=function(){System.import(document.getElementById('vite-legacy-entry').getAttribute('data-src'))},document.body.appendChild(n)}}();</script>
</head>
<body>
<div id="app"></div>
<script nomodule>!function(){var e=document,t=e.createElement("script");if(!("noModule"in t)&&"onbeforeload"in t){var n=!1;e.addEventListener("beforeload",(function(e){if(e.target===t)n=!0;else if(!e.target.hasAttribute("nomodule")||!n)return;e.preventDefault()}),!0),t.type="module",t.src=".",e.head.appendChild(t),t.remove()}}();</script>
<script nomodule id="vite-legacy-polyfill" src="/assets/polyfills-legacy.34d9b402.js"></script>
<script nomodule id="vite-legacy-entry" data-src="/assets/index-legacy.90883ee8.js">System.import(document.getElementById('vite-legacy-entry').getAttribute('data-src'))</script>
</body>
</html>
transform_doc_to_docx.py 0 → 100644
# -*- coding: utf-8 -*-
# @Time : 2023/2/14 15:26
# @Author : ctt
# @File : transform_doc_to_docx
# @Project : 解析docx
import os
import pythoncom
import win32com.client
def closesoft():
print('''挂载程序关闭中……
''')
wc = win32com.client.constants
try:
wps = win32com.client.gencache.EnsureDispatch('kwps.application')
except:
wps = win32com.client.gencache.EnsureDispatch('wps.application')
else:
wps = win32com.client.gencache.EnsureDispatch('word.application')
try:
wps.Documents.Close()
wps.Documents.Close(wc.wdDoNotSaveChanges)
wps.Quit
except:
pass
def doc2docx(path):
pythoncom.CoInitialize()
w = win32com.client.Dispatch('Word.Application')
w.Visible = 0
w.DisplayAlerts = 0
doc = w.Documents.Open(path)
newpath = os.path.splitext(path)[0] + '.docx'
doc.SaveAs(newpath, 12, False, "", True, "", False, False, False, False)
doc.Close()
w.Quit()
return newpath
if __name__ == '__main__':
closesoft()
doc2docx(r'D:\四川报告\相关代码\从word中提取指定表格\data\特殊教育学校(1).doc')
utils/__init__.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : __init__.py
# @Author : LiuYan
# @Time : 2021/7/31 17:36
utils/database_mysql.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : database
# @Author : LiuYan
# @Time : 2021/9/14 17:51
import pymysql
class DatabaseMySQL(object):
def __init__(self, database_config: dict):
super(DatabaseMySQL, self).__init__()
self._conn = None
self._cursor = None
self._database_config = database_config
self._connect()
def _connect(self) -> None:
self._database_config['port'] = int(self._database_config['port'])
self._database_config['charset'] = 'utf8'
self._database_config['cursorclass'] = pymysql.cursors.DictCursor
self._conn = pymysql.connect(**self._database_config)
self._cursor = self._conn.cursor()
def query(self, sql: str) -> list:
# 获取表单信息
print('SQL: {}'.format(sql))
self._cursor.execute(sql)
list_result = self._cursor.fetchall()
return list_result
def close(self) -> None:
self._cursor.close()
self._conn.close()
def is_connected(self):
"""Check if the server is alive"""
try:
self.conn.ping(reconnect=True)
print("db is connecting")
except Exception:
self.conn = self._connect()
print("db reconnect")
if __name__ == '__main__':
database_config = {
'host': '114.115.159.144',
'port': 3306,
'user': 'root',
'password': 'zzsn9988',
'database': 'clb_project'
}
dbm = DatabaseMySQL(database_config=database_config)
# sql = """select ds.data_source_name,ds.url,ds.params,ds.type,ds.data_name from clb_report_data_source ds inner join clb_report_data_set_source_map m on ds.id = m.data_source_id
# where m.data_set_id = '1641045122365317122'
# """
task_id = '1641261934625521666'
#
sql = '''SELECT ds.id,ds.param_value,te.file_path FROM clb_report_task t inner join clb_report_template te on t.template_id = te.id
inner join clb_report_data_set ds on te.data_set_id = ds.id
where t.id = {};'''.format(task_id)
list_result = dbm.query(sql=sql)
print(list_result)
dbm.close()
utils/database_oracle.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : database_oracle
# @Author : LiuYan
# @Time : 2021/10/13 16:10
import os
import cx_Oracle
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'
class DatabaseOracle(object):
def __init__(self, database_config: dict):
super(DatabaseOracle, self).__init__()
self._conn = None
self._cursor = None
self._database_config = database_config
self._connect()
def _connect(self) -> None:
self._conn = cx_Oracle.connect(
self._database_config['user'],
self._database_config['password'],
self._database_config['host'] + ':' + str(self._database_config['port']) + '/' + self._database_config['database']
)
self._cursor = self._conn.cursor()
def query(self, sql: str) -> list:
# 获取表单信息
print('SQL: {}'.format(sql))
self._cursor.execute(sql)
list_result = self.rows_as_dicts()
return list_result
def rows_as_dicts(self) -> list:
"""
将查询结果转为dict
:return:
"""
col_names = [i[0] for i in self._cursor.description]
return [dict(zip(col_names, row)) for row in self._cursor]
def close(self) -> None:
self._cursor.close()
self._conn.close()
if __name__ == '__main__':
database_config = {
'host': '114.116.91.1',
'port': 1521,
'user': 'cis',
'password': 'cis_zzsn9988',
'database': 'ORCL'
}
dbo = DatabaseOracle(database_config=database_config)
sql = '''
select
TITLE as 标题,
SUMMARY as 摘要,
ORIGIN as 来源
from CIS_ANS_BASEDATA where ROWNUM < 10'''
list_result = dbo.query(sql=sql)
print(list_result)
dbo.close()
utils/log.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : log
# @Author : LiuYan
# @Time : 2020/6/21 21:08
import os
import logging
import logging.handlers
from pathlib import Path
__all__ = ['logger']
# 用户配置部分 ↓
import tqdm
LEVEL_COLOR = {
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red,bg_white',
}
STDOUT_LOG_FMT = '%(log_color)s[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(filename)s:%(lineno)d] %(message)s'
STDOUT_DATE_FMT = '%Y-%m-%d %H:%M:%S'
FILE_LOG_FMT = '[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(filename)s:%(lineno)d] %(message)s'
FILE_DATE_FMT = '%Y-%m-%d %H:%M:%S'
# 用户配置部分 ↑
class ColoredFormatter(logging.Formatter):
COLOR_MAP = {
'black': '30',
'red': '31',
'green': '32',
'yellow': '33',
'blue': '34',
'magenta': '35',
'cyan': '36',
'white': '37',
'bg_black': '40',
'bg_red': '41',
'bg_green': '42',
'bg_yellow': '43',
'bg_blue': '44',
'bg_magenta': '45',
'bg_cyan': '46',
'bg_white': '47',
'light_black': '1;30',
'light_red': '1;31',
'light_green': '1;32',
'light_yellow': '1;33',
'light_blue': '1;34',
'light_magenta': '1;35',
'light_cyan': '1;36',
'light_white': '1;37',
'light_bg_black': '100',
'light_bg_red': '101',
'light_bg_green': '102',
'light_bg_yellow': '103',
'light_bg_blue': '104',
'light_bg_magenta': '105',
'light_bg_cyan': '106',
'light_bg_white': '107',
}
def __init__(self, fmt, datefmt):
super(ColoredFormatter, self).__init__(fmt, datefmt)
def parse_color(self, level_name):
color_name = LEVEL_COLOR.get(level_name, '')
if not color_name:
return ""
color_value = []
color_name = color_name.split(',')
for _cn in color_name:
color_code = self.COLOR_MAP.get(_cn, '')
if color_code:
color_value.append(color_code)
return '\033[' + ';'.join(color_value) + 'm'
def format(self, record):
record.log_color = self.parse_color(record.levelname)
message = super(ColoredFormatter, self).format(record) + '\033[0m'
return message
class TqdmLoggingHandler(logging.Handler):
def __init__(self, level=logging.NOTSET):
super().__init__(level)
def emit(self, record):
try:
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def _get_logger(log_to_file=True, log_filename='default.log', log_level='DEBUG'):
_logger = logging.getLogger(__name__)
stdout_handler = logging.StreamHandler()
stdout_handler.setFormatter(
ColoredFormatter(
fmt=STDOUT_LOG_FMT,
datefmt=STDOUT_DATE_FMT,
)
)
_logger.addHandler(stdout_handler)
# _logger.setLevel(logging.INFO)
# _logger.addHandler(TqdmLoggingHandler())
if log_to_file:
# _tmp_path = os.path.dirname(os.path.abspath(__file__))
# _tmp_path = os.path.join(_tmp_path, '../logs/{}'.format(log_filename))
_project_path = os.path.dirname(os.getcwd())
_tmp_path = os.path.join(_project_path, 'logs')
Path(_tmp_path).mkdir(parents=True, exist_ok=True)
_tmp_path = os.path.join(_tmp_path, log_filename)
file_handler = logging.handlers.TimedRotatingFileHandler(_tmp_path, when='midnight', backupCount=30)
file_formatter = logging.Formatter(
fmt=FILE_LOG_FMT,
datefmt=FILE_DATE_FMT,
)
file_handler.setFormatter(file_formatter)
_logger.addHandler(file_handler)
_logger.setLevel(log_level)
return _logger
logger = _get_logger(log_to_file=False)
utils/tools.py 0 → 100644
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @File : tool
# @Author : LiuYan
# @Time : 2021/6/21 11:22
from __future__ import unicode_literals, print_function, division
import os
import re
import json
import time
import zipfile
import datetime
import xlsxwriter
def read_json(path: str) -> list:
f = open(path, 'r', encoding='utf-8')
examples = []
for line in f.readlines():
examples.append(json.loads(line))
f.close()
return examples
def clean_text(text: str) -> str:
return re.sub('\n+', '\n', text.strip().replace(' ', '').replace('\t', '').replace('\r', ''))
# 打包zip
def make_zip(file_dir: str, zip_path: str) -> None:
zip_f = zipfile.ZipFile(zip_path, 'w')
pre_len = len(os.path.dirname(file_dir))
for parent, dir_names, filenames in os.walk(file_dir):
for filename in filenames:
path_file = os.path.join(parent, filename)
arc_name = path_file[pre_len:].strip(os.path.sep)
zip_f.write(path_file, arc_name)
zip_f.close()
# 删除zip
def delete_zip(zip_path: str) -> None:
os.remove(zip_path)
def timeit(f):
def timed(*args, **kw):
ts = time.time()
print('......begin {0:8s}......'.format(f.__name__))
result = f(*args, **kw)
te = time.time()
print('......finish {0:8s}, took:{1:.4f} sec......'.format(f.__name__, te - ts))
return result
return timed
def list2xlsx(result_list: list, xlsx_path: str):
"""
:param result_list: [
{
'id': 1,
'title': 't',
...
}
...
]
:param xlsx_path: '/home/zzsn/liuyan/result/result.xlsx'
:return:
"""
workbook = xlsxwriter.Workbook(xlsx_path)
worksheet = workbook.add_worksheet('sheet1')
worksheet.write_row(row=0, col=0, data=list(result_list[0].keys()))
for row_index, result_dict in enumerate(result_list):
worksheet.write_row(row=row_index + 1, col=0, data=list(
';'.join(result) if type(result) in [list, set] else result for result in result_dict.values()
))
workbook.close()
def return_json(handle_msg: str, is_handle_success: bool, logs: str or None, result_data: object) -> dict or json:
"""
:param handle_msg: str 处理信息：是否成功 'success' / 'failure'
:param is_handle_success: bool 是否处理成功
:param logs: str or None 处理过程以及结果信息
:param result_data: object 处理结果 数据
:return:
"""
dict_result = {
'handleMsg': handle_msg,
'isHandleSuccess': is_handle_success,
'logs': logs,
'resultData': result_data
}
return dict_result
# return json.dumps(dict_result, ensure_ascii=False)
def return_json_(handle_msg: str, is_handle_success: bool, logs: str or None, result_data: object) -> dict or json:
"""
:param handle_msg: str 处理信息：是否成功 'success' / 'failure'
:param is_handle_success: bool 是否处理成功
:param logs: str or None 处理过程以及结果信息
:param result_data: object 处理结果 数据
:return:
"""
dict_result = {
'handleMsg': handle_msg,
'ynHandleSuccess': is_handle_success,
'logs': logs,
'resultData': result_data
}
return json.dumps(dict_result, ensure_ascii=False)
def return_json_str(handle_msg: str, is_handle_success: bool, logs: str or None, result_data: object) -> dict or json:
"""
:param handle_msg: str 处理信息：是否成功 'success' / 'failure'
:param is_handle_success: bool 是否处理成功
:param logs: str or None 处理过程以及结果信息
:param result_data: object 处理结果 数据
:return:
"""
dict_result = {
'handleMsg': handle_msg,
'isHandleSuccess': is_handle_success,
'logs': logs,
'resultData': result_data
}
return json.dumps(dict_result, ensure_ascii=False)
def class_to_dict(obj):
is_list = obj.__class__ == [].__class__
is_set = obj.__class__ == set().__class__
if is_list or is_set:
obj_arr = []
for o in obj:
dict = {}
a = o.__dict__
if "_sa_instance_state" in a:
del a['_sa_instance_state']
dict.update(a)
obj_arr.append(dict)
return obj_arr
else:
dict = {}
a = obj.__dict__
if "_sa_instance_state" in a:
del a['_sa_instance_state']
dict.update(a)
return dict
def class_to_dict_field(obj):
"""
字段集合转化字典 如 db.session.query(User.created_time).all()
:param obj:
:return:
"""
is_list = obj.__class__ == [].__class__
is_set = obj.__class__ == set().__class__
if is_list or is_set:
obj_arr = []
for o in obj:
for v in dir(o):
if isinstance(o.__getattribute__(v), datetime.datetime) and not v.startswith("__"):
dict = {str(v): o.__getattribute__(v).strftime('%Y-%m-%d %H:%M:%S')}
obj_arr.append(dict)
return obj_arr
def class_to_dict_all(obj):
"""
例如 GlobalRegion.query.all()
:param obj:
:return:
"""
is_list = obj.__class__ == [].__class__
is_set = obj.__class__ == set().__class__
if is_list or is_set:
obj_arr = []
for o in obj:
for k, v in vars(o).items():
if isinstance(v, datetime.datetime):
o.__dict__[k] = v.strftime('%Y-%m-%d %H:%M:%S')
dict = {}
a = o.__dict__
if '_sa_instance_state' in a:
del a['_sa_instance_state']
dict.update(a)
obj_arr.append(dict)
return obj_arr
else:
dict = {}
for k, v in vars(obj).items():
if isinstance(v, datetime.datetime):
obj.__dict__[k] = v.strftime('%Y-%m-%d %H:%M:%S')
a = obj.__dict__
if '_sa_instance_state' in a:
del a['_sa_instance_state']
dict.update(a)
return dict
def formatGMTime(timestamp):
"""
格式化 GMT 时间
:param timestamp:
:return:
"""
GMT_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
return datetime.datetime.strptime(timestamp, GMT_FORMAT) + datetime.timedelta(hours=8)
文件服务器操作.py 0 → 100644
#!/usr/bin/env python
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import traceback
from fdfs_client.client import *
class FDFS(object):
def __init__(self, client_file):
self.client_file = client_file
self.client = self.create_client()
def create_client(self):
try:
client = Fdfs_client(self.client_file)
return client
except Exception as e:
print("FastDFS Create file fail, {0}, {1}".format(e, traceback.print_exc()))
return None
def download(self, file_name, file_id):
try:
ret_download = self.client.download_to_file(file_name, file_id)
print(ret_download)
return True
except Exception as e:
print("FastDFS download file fail, {0}, {1}".format(e, traceback.print_exc()))
return False
def upload(self, file_name):
try:
ret_upload = self.client.upload_by_filename(file_name)
print(ret_upload)
return True
except Exception as e:
print("FastDFS upload file fail, {0}, {1}".format(e, traceback.print_exc()))
return False
def delete(self, file_id):
try:
ret_delete = self.client.delete_file(file_id)
print(ret_delete)
return True
except Exception as e:
print("FastDFS delete file fail, {0}, {1}".format(e, traceback.print_exc()))
return False
if __name__ == "__main__":
client_file = "fdfs_client.conf"
upload_file = "/opt/abc.py"
download_file = "/opt/abc.docx"
file_id = "group1/M00/00/02/rBAAQWCt48iAUX7DAA-s5fjCGo402.docx"
fdfs = FDFS(client_file)
fdfs.download(download_file, file_id)
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论