245 lines
11 KiB
Python
245 lines
11 KiB
Python
|
from fastapi import FastAPI, HTTPException
|
|||
|
|
from pydantic import BaseModel
|
|||
|
|
import os
|
|||
|
|
import utils
|
|||
|
|
import queue
|
|||
|
|
from multiprocessing import Process,Manager
|
|||
|
|
import pdf_title
|
|||
|
|
import main
|
|||
|
|
import time
|
|||
|
|
import threading
|
|||
|
|
import config
|
|||
|
|
import requests
|
|||
|
|
import db_service
|
|||
|
|
import threading
|
|||
|
|
from Mil_unit import create_partition_by_hour
|
|||
|
|
from datetime import datetime, timedelta
|
|||
|
|
from log_config import logger
|
|||
|
|
|
|||
|
|
app = FastAPI()
|
|||
|
|
cpu_count = 4
|
|||
|
|
job_queue = queue.Queue()
|
|||
|
|
|
|||
|
|
# 定义请求体模型
|
|||
|
|
class FileItem(BaseModel):
|
|||
|
|
file_path: str
|
|||
|
|
file_id: str
|
|||
|
|
|
|||
|
|
def run_job():
|
|||
|
|
#判断是否有任务在执行
|
|||
|
|
if_run = True
|
|||
|
|
|
|||
|
|
if job_queue.empty():
|
|||
|
|
logger.info(f"job_queue为空: {file_path}")
|
|||
|
|
if_run = False
|
|||
|
|
|
|||
|
|
if if_run:
|
|||
|
|
job_config = job_queue.get()
|
|||
|
|
page_list = []
|
|||
|
|
file_path = job_config['file_path']
|
|||
|
|
file_id = job_config['file_id']
|
|||
|
|
job_status = True
|
|||
|
|
continue_execution = True
|
|||
|
|
try:
|
|||
|
|
#下载pdf
|
|||
|
|
start_time = time.time()
|
|||
|
|
logger.info(f"开始启动文件解析任务: {file_path}")
|
|||
|
|
if file_path.startswith('http'):
|
|||
|
|
file_path = utils.save_pdf_from_url(file_path, config.FILE_PATH)
|
|||
|
|
try:
|
|||
|
|
file_info = pdf_title.create_text_outline(file_path,file_id)
|
|||
|
|
except Exception as e:
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id,'status': 7})
|
|||
|
|
logger.info(f'通知任务状态url:{file_id}:{response.url}')
|
|||
|
|
logger.info(f'通知任务状态任务:{file_id}:{response.text}')
|
|||
|
|
logger.info(f"{file_id}运行失败: {e}")
|
|||
|
|
continue_execution = False
|
|||
|
|
if continue_execution:
|
|||
|
|
parent_table_pages = file_info['parent_table_pages']
|
|||
|
|
page_num = file_info['page_count']
|
|||
|
|
if page_num < cpu_count:
|
|||
|
|
p_count = page_num
|
|||
|
|
else :
|
|||
|
|
p_count = cpu_count
|
|||
|
|
|
|||
|
|
for i in range(p_count):
|
|||
|
|
page_list.append({
|
|||
|
|
'type': 'table',
|
|||
|
|
'page_num': file_info['split_parts']['table_split_parts'][i],
|
|||
|
|
# 'page_num': page_nums[i],
|
|||
|
|
'path': file_path,
|
|||
|
|
'file_id': file_id,
|
|||
|
|
'parent_table_pages': parent_table_pages,
|
|||
|
|
'page_count': file_info['page_count'],
|
|||
|
|
'tables_range': {},
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 通知开始解析
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id,'status': 5})
|
|||
|
|
logger.info(f'通知pdf开始解析url:{file_id}:{response.url}')
|
|||
|
|
logger.info(f'通知pdf开始解析状态:{file_id}:{response.text}')
|
|||
|
|
parser_start_time = time.time()
|
|||
|
|
processes = []
|
|||
|
|
time_dispatch_job = time.time()
|
|||
|
|
|
|||
|
|
for job_info in page_list:
|
|||
|
|
p = Process(target=main.dispatch_job, args=(job_info,))
|
|||
|
|
processes.append(p)
|
|||
|
|
p.start()
|
|||
|
|
|
|||
|
|
logger.info(f'等待所有子任务完成，任务ID:{file_id}')
|
|||
|
|
for p in processes:
|
|||
|
|
p.join()
|
|||
|
|
logger.info(f'pdf解析任务完成任务完成，任务ID:{file_id}')
|
|||
|
|
time_dispatch_job_end = time.time()
|
|||
|
|
process_time = time_dispatch_job_end - time_dispatch_job
|
|||
|
|
db_service.process_time(file_id,'1',process_time,time_dispatch_job,time_dispatch_job_end)
|
|||
|
|
parser_end_time = time.time()
|
|||
|
|
logger.info(f"解析任务 {file_id} 完成，耗时{(parser_end_time - parser_start_time):.2f} 秒。")
|
|||
|
|
#这里做一步判断，看看是否还要继续。
|
|||
|
|
if db_service.file_type_check(file_id):
|
|||
|
|
logger.info(f"文本较真表格生成已结束")
|
|||
|
|
else:
|
|||
|
|
# 通知抽取指标
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id,'status': 6})
|
|||
|
|
logger.info(f'通知开始抽取指标url:{file_id}:{response.url}')
|
|||
|
|
logger.info(f'通知开始抽取指标状态:{file_id}:{response.text}')
|
|||
|
|
|
|||
|
|
parser_start_time = time.time()
|
|||
|
|
logger.info(f'开始表格指标抽取，任务ID:{file_id}')
|
|||
|
|
time_start = time.time()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 获取当前时间
|
|||
|
|
now = datetime.now()
|
|||
|
|
current_hour = now.strftime("%Y%m%d%H")
|
|||
|
|
partition_name = f"partition_{current_hour}"
|
|||
|
|
# 判断是否创建新的分区
|
|||
|
|
create_partition_by_hour(current_hour)
|
|||
|
|
time.sleep(10)
|
|||
|
|
# 判断是否为3季报
|
|||
|
|
|
|||
|
|
if db_service.file_type_check_v2(file_id) == 3:
|
|||
|
|
main.start_table_measure_job(file_id,partition_name)
|
|||
|
|
time_start_end = time.time()
|
|||
|
|
process_time = time_start_end - time_start
|
|||
|
|
db_service.process_time(file_id,'2',process_time,time_start,time_start_end)
|
|||
|
|
logger.info(f'表格指标抽取完成，任务ID:{file_id}')
|
|||
|
|
parser_end_time = time.time()
|
|||
|
|
logger.info(f"表格指标抽取 {file_id} 完成，耗时{(parser_end_time - parser_start_time):.2f} 秒。")
|
|||
|
|
|
|||
|
|
logger.info(f'启动这个指标归一化任务ID-修改测试:{file_id}')
|
|||
|
|
time_update = time.time()
|
|||
|
|
main.update_measure_data(file_id,file_path,parent_table_pages,partition_name)
|
|||
|
|
|
|||
|
|
logger.info(f'归一化完成任务ID:{file_id}')
|
|||
|
|
end_time = time.time()
|
|||
|
|
logger.info(f"任务 {file_id} 完成，耗时{(end_time - start_time):.2f} 秒。")
|
|||
|
|
time_update_end = time.time()
|
|||
|
|
process_time = time_update_end - time_update
|
|||
|
|
db_service.process_time(file_id,'3',process_time,time_update,time_update_end)
|
|||
|
|
# 不是三季报就直接按照年报和半年报走
|
|||
|
|
else:
|
|||
|
|
main.start_table_measure_job(file_id,partition_name)
|
|||
|
|
time_start_end = time.time()
|
|||
|
|
process_time = time_start_end - time_start
|
|||
|
|
db_service.process_time(file_id,'2',process_time,time_start,time_start_end)
|
|||
|
|
logger.info(f'表格指标抽取完成，任务ID:{file_id}')
|
|||
|
|
parser_end_time = time.time()
|
|||
|
|
logger.info(f"表格指标抽取 {file_id} 完成，耗时{(parser_end_time - parser_start_time):.2f} 秒。")
|
|||
|
|
|
|||
|
|
logger.info(f'启动这个指标归一化任务ID-修改测试:{file_id}')
|
|||
|
|
time_update = time.time()
|
|||
|
|
main.update_measure_data(file_id,file_path,parent_table_pages,partition_name)
|
|||
|
|
|
|||
|
|
logger.info(f'归一化完成任务ID:{file_id}')
|
|||
|
|
end_time = time.time()
|
|||
|
|
logger.info(f"任务 {file_id} 完成，耗时{(end_time - start_time):.2f} 秒。")
|
|||
|
|
time_update_end = time.time()
|
|||
|
|
process_time = time_update_end - time_update
|
|||
|
|
db_service.process_time(file_id,'3',process_time,time_update,time_update_end)
|
|||
|
|
#通知任务完成
|
|||
|
|
response_time = time.time()
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id,'status': 1})
|
|||
|
|
logger.info(f'通知任务状态url:{file_id}:{response.url}')
|
|||
|
|
logger.info(f'通知任务状态任务:{file_id}:{response.text}')
|
|||
|
|
response_time_end = time.time()
|
|||
|
|
process_time = response_time_end - response_time
|
|||
|
|
db_service.process_time(file_id,'4',process_time,response_time,response_time_end)
|
|||
|
|
except Exception as e:
|
|||
|
|
#通知任务完成
|
|||
|
|
response_time = time.time()
|
|||
|
|
if "integer division or modulo by zero" in str(e):
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id, 'status': 4})
|
|||
|
|
else:
|
|||
|
|
response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id, 'status': 4})
|
|||
|
|
#response = requests.get(config.NOTIFY_ADDR, params={'fileId': file_id,'status': 4})
|
|||
|
|
response_time_end = time.time()
|
|||
|
|
process_time = response_time_end - response_time
|
|||
|
|
db_service.process_time(file_id,'4',process_time,response_time,response_time_end)
|
|||
|
|
logger.info(f'通知任务状态url:{file_id}:{response.url}')
|
|||
|
|
logger.info(f'通知任务状态任务:{file_id}:{response.text}')
|
|||
|
|
logger.info(f"Response status code: {response.status_code}")
|
|||
|
|
logger.info(f"{file_id}运行失败: {e}")
|
|||
|
|
finally:
|
|||
|
|
logger.info(f"任务 {file_id} 完成，运行状态：{job_status}")
|
|||
|
|
|
|||
|
|
#pdf_company_0824.name_code_fix(file_id,file_path)
|
|||
|
|
#print('公司名与编码填充完毕')
|
|||
|
|
else:
|
|||
|
|
logger.info(f"有任务运行中，需要等待.....")
|
|||
|
|
|
|||
|
|
def parse_pdf_route(fileItem: FileItem):
|
|||
|
|
|
|||
|
|
# 创建一个队列,保证每次只执行一个文件解析任务
|
|||
|
|
job_queue.put({
|
|||
|
|
'file_path' : fileItem.file_path,
|
|||
|
|
'file_id' : fileItem.file_id
|
|||
|
|
})
|
|||
|
|
logger.info(f"增加 {fileItem.file_id} 到队列.")
|
|||
|
|
|
|||
|
|
threading.Thread(target=run_job, args=()).start()
|
|||
|
|
|
|||
|
|
return {"success": True, "msg": "文件解析开始"}
|
|||
|
|
|
|||
|
|
app.post("/parser/start",
|
|||
|
|
tags=["parser"],
|
|||
|
|
summary="解析Pdf文件",
|
|||
|
|
)(parse_pdf_route)
|
|||
|
|
|
|||
|
|
def get_local_ip():
|
|||
|
|
try:
|
|||
|
|
# 创建一个 UDP 套接字
|
|||
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|||
|
|
# 连接到一个外部地址（这里使用 Google 的公共 DNS 服务器）
|
|||
|
|
s.connect(("8.8.8.8", 80))
|
|||
|
|
# 获取本地套接字的 IP 地址
|
|||
|
|
local_ip = s.getsockname()[0]
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.info(f"获取内网 IP 失败: {e}")
|
|||
|
|
local_ip = "127.0.0.1" # 如果失败，返回本地回环地址
|
|||
|
|
finally:
|
|||
|
|
s.close() # 关闭套接字
|
|||
|
|
return local_ip
|
|||
|
|
|
|||
|
|
# 运行 FastAPI 应用
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# 服务器启动服务
|
|||
|
|
import uvicorn
|
|||
|
|
uvicorn.run(app, host="0.0.0.0", port=config.PORT)
|
|||
|
|
try:
|
|||
|
|
# 获取内网IP
|
|||
|
|
ip = get_local_ip()
|
|||
|
|
response = requests.get(f"/api/tenant/report/restart?address={ip}:{config.PORT}")
|
|||
|
|
except KeyboardInterrupt:
|
|||
|
|
logger.info("Shutdown server")
|
|||
|
|
|
|||
|
|
# 本地调试任务
|
|||
|
|
# job_queue.put({
|
|||
|
|
# 'file_path' : '1.pdf',
|
|||
|
|
# 'file_id' : '2122'
|
|||
|
|
# })
|
|||
|
|
#
|
|||
|
|
# run_job()
|