from datetime import datetime import re,os,json import utils import ast import time import redis_service from multiprocessing import Process from config import MILVUS_CLIENT,MYSQL_HOST,MYSQL_USER,MYSQL_PASSWORD,MYSQL_DB,REDIS_HOST,REDIS_PORT,REDIS_PASSWORD,MEASURE_COUNT,MYSQL_HOST_APP,MYSQL_USER_APP,MYSQL_PASSWORD_APP,MYSQL_DB_APP from pymilvus import MilvusClient import mysql.connector import threading import redis measure_name_keywords = ["营业","季度","利润","归属于","扣非","经营","现金","活动","损益","收益","资产","费用","销售","管理","财务","研发","货币资金","应收账款","存货","固定资产","在建工程","商誉","短期借款","应付账款","合同负债","长期借款","营业成本"] # 解析大模型抽取的指标，并插入到数据库 def parse_llm_measure_to_db(measure_info,type,conn,cursor): create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") check_query = ''' select id from ori_measure_list WHERE file_id = %s and type = %s and page_number = %s and ori_measure_value = %s ''' # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO ori_measure_list (file_id, file_name, type, page_number, table_index, ori_measure_id, ori_measure_name, ori_measure_value, create_time, update_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ''' file_id = measure_info['file_id'] file_name = measure_info['path'] llm_measure = measure_info['llm_measure'] page_num = measure_info['page_num'] table_index = '0' if type == 'table': table_index = measure_info['table_index'] for measure_obj in llm_measure: measure_obj = measure_obj.replace('\n', '').replace('\r', '').replace(' ', '').replace('：', ':') if ':' in measure_obj: ori_measure_name = measure_obj.split(':')[0].replace('-', '') if len(ori_measure_name) > 30 : continue ori_measure_value = measure_obj.split(':')[1].replace('+', '').replace(',', '').replace('元', '').replace('%', '') if '-' in ori_measure_value: ori_measure_value = "-" if '.' in ori_measure_name: ori_measure_name = ori_measure_name.split('.')[1] ori_measure_id = utils.get_md5(ori_measure_name) if re.match(r'^[+-]?(\d+(\.\d*)?|\.\d+)(%?)$', ori_measure_value): # 判断数据库中是否有数据 check_query_data = (file_id, 'text', int(page_num), ori_measure_value) cursor.execute(check_query, check_query_data) check_records = cursor.fetchall() if(len(check_records)) > 0: continue data_to_insert = (file_id, file_name, type, int(page_num), int(table_index), ori_measure_id, ori_measure_name, ori_measure_value, create_time, create_time) cursor.execute(insert_query, data_to_insert) conn.commit() def insert_measure_parser_info(parser_info,conn,cursor): create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO measure_parser_info (file_id, type, content, create_time) VALUES (%s, %s, %s, %s) ''' file_id = parser_info['file_id'] type = parser_info['type'] content = parser_info['content'] data_to_insert = (file_id, type, content, create_time) cursor.execute(insert_query, data_to_insert) conn.commit() def insert_measure_parser_info_measure(parser_info, conn, cursor, line_text): create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") insert_query = ''' INSERT INTO measure_parser_info_linetext (file_id, type, content,text, create_time) VALUES (%s, %s, %s, %s,%s) ''' file_id = parser_info['file_id'] type = parser_info['type'] content = parser_info['content'] text = line_text data_to_insert = (file_id, type, content,text, create_time) cursor.execute(insert_query, data_to_insert) conn.commit() def insert_table_unit_info(table_info,conn,cursor): # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO table_unit_info (file_id, page_num, table_index, unit) VALUES (%s, %s, %s, %s) ''' file_id = table_info['file_id'] page_num = int(table_info['page_num']) table_index = int(table_info['table_index']) unit = table_info['unit'] data_to_insert = (file_id, page_num, table_index, unit) cursor.execute(insert_query, data_to_insert) conn.commit() def insert_table_unit_info_v1(table_info, conn, cursor): """ 插入数据到 table_unit_info 表之前，检查是否存在相同的 file_id, page_num 和 table_index。 如果存在且 unit 不同，更新现有记录，否则插入新记录。 """ file_id = table_info['file_id'] page_num = int(table_info['page_num']) table_index = int(table_info['table_index']) unit = table_info['unit'] # 查询现有记录 check_query = ''' SELECT unit FROM table_unit_info WHERE file_id = %s AND page_num = %s AND table_index = %s ''' cursor.execute(check_query, (file_id, page_num, table_index)) existing_record = cursor.fetchone() if existing_record: existing_unit = existing_record[0] if unit != existing_unit: # 更新现有记录 update_query = ''' UPDATE table_unit_info SET unit = %s WHERE file_id = %s AND page_num = %s AND table_index = %s ''' cursor.execute(update_query, (unit, file_id, page_num, table_index)) #print(f'Updated existing record with file_id={file_id}, page_num={page_num}, table_index={table_index}.') else: print(f'No change needed. Existing unit={existing_unit} is the same as new unit={unit}.') else: # 插入新的记录 insert_query = ''' INSERT INTO table_unit_info (file_id, page_num, table_index, unit) VALUES (%s, %s, %s, %s) ''' data_to_insert = (file_id, page_num, table_index, unit) cursor.execute(insert_query, data_to_insert) #print(f'Inserted new record with file_id={file_id}, page_num={page_num}, table_index={table_index}, unit={unit}.') conn.commit() def insert_table_text_info(table_info,conn,cursor): # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO table_text_info (file_id, page_num, table_index, text) VALUES (%s, %s, %s, %s) ''' file_id = table_info['file_id'] page_num = int(table_info['page_num']) table_index = int(table_info['table_index']) text = table_info['text_info'] data_to_insert = (file_id, page_num, table_index, text) cursor.execute(insert_query, data_to_insert) conn.commit() def update_ori_measure(conn,cursor,file_id): select_year_select = f"""select report_type,year from report_check where id = {file_id}""" cursor.execute(select_year_select) record_select = cursor.fetchall() report_type = record_select[0][0] report_year = record_select[0][1] # 执行SQL语句，更新数据 update_query = ''' UPDATE ori_measure_list SET measure_id = %s, measure_name = %s WHERE ori_measure_id = %s and file_id = %s ''' select_query = ''' SELECT t2.measure_id,t2.measure_name,t1.ori_measure_id FROM ori_measure_list t1 left join measure_config t2 on t1.ori_measure_id = t2.ori_measure_id where t2.measure_id is not null and (t1.measure_id is null or t1.measure_id ='') and t1.file_id = '{file_id}' and t2.year = '{year}' '''.format(file_id=file_id, year=report_year) select_query_half_year = ''' SELECT t2.measure_id,t2.measure_name,t1.ori_measure_id FROM ori_measure_list t1 left join measure_config_half_year t2 on t1.ori_measure_id = t2.ori_measure_id where t2.measure_id is not null and (t1.measure_id is null or t1.measure_id ='') and t1.file_id = '{file_id}' and t2.year = '{year}' '''.format(file_id=file_id, year=report_year) select_query_thrid = ''' SELECT t2.measure_id,t2.measure_name,t1.ori_measure_id FROM ori_measure_list t1 left join measure_config_third_quarter t2 on t1.ori_measure_id = t2.ori_measure_id where t2.measure_id is not null and (t1.measure_id is null or t1.measure_id ='') and t1.file_id = '{file_id}' and t2.year = '{year}' '''.format(file_id=file_id, year=report_year) if report_type == 1: start_time = time.time() cursor.execute(select_query_half_year) records = cursor.fetchall() end_time = time.time() print(f"更新数据查询 {(end_time - start_time):.2f} 秒。") print(f'update_ori_measure方法走的是半年报') elif report_type == 3: start_time = time.time() cursor.execute(select_query_thrid) records = cursor.fetchall() end_time = time.time() print(f"更新数据查询 {(end_time - start_time):.2f} 秒。") print(f'update_ori_measure方法走的是三季报') else: start_time = time.time() cursor.execute(select_query) records = cursor.fetchall() end_time = time.time() print(f"更新数据查询 {(end_time - start_time):.2f} 秒。") print(f'update_ori_measure方法走的是全年报') start_time = time.time() for record in records: data_to_update = (record[0], record[1], record[2], file_id) cursor.execute(update_query, data_to_update) conn.commit() end_time = time.time() print(f"更新数据更新 {(end_time - start_time):.2f} 秒。") #更新measure_list表，增加此次文件的显示指标 start_time = time.time() create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if report_type == 0: insert_query = ''' INSERT INTO measure_list (measure_id, measure_name, create_time, update_time, file_id) select distinct measure_id,measure_name, %s,%s,%s from measure_config where year = '{year}' '''.format(year=report_year) elif report_type == 3: insert_query = ''' INSERT INTO measure_list (measure_id, measure_name, create_time, update_time, file_id) select distinct measure_id,measure_name, %s,%s,%s from measure_config_third_quarter where year = '{year}' '''.format(year=report_year) else: insert_query = ''' INSERT INTO measure_list (measure_id, measure_name, create_time, update_time, file_id) select distinct measure_id,measure_name, %s,%s,%s from measure_config_half_year where year = '{year}' '''.format(year=report_year) data_to_update = (create_time, create_time, file_id) cursor.execute(insert_query, data_to_update) conn.commit() end_time = time.time() print(f"更新数据写入 {(end_time - start_time):.2f} 秒。") def insert_table_from_vector_mul_process(parent_table_pages,file_id,file_name,records,record_range,black_array,partition_name,): create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") print('Run task %s (%s)...' % (record_range, os.getpid())) print(f"插入数据 {len(records)}") conn = mysql.connector.connect( host = MYSQL_HOST, user = MYSQL_USER, password = MYSQL_PASSWORD, database = MYSQL_DB ) redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=6) # 创建一个cursor对象来执行SQL语句 cursor = conn.cursor(buffered=True) conn_app = mysql.connector.connect( host = MYSQL_HOST_APP, user = MYSQL_USER_APP, password = MYSQL_PASSWORD_APP, database = MYSQL_DB_APP ) cursor_app = conn_app.cursor(buffered=True) select_year_select = f"""select report_type,year from report_check where id = {file_id}""" cursor.execute(select_year_select) record_select = cursor.fetchall() report_type = record_select[0][0] report_year = record_select[0][1] check_query = ''' select id from ori_measure_list WHERE file_id = %s and measure_name = %s and page_number = %s and table_index = %s and ori_measure_value = %s ''' insert_query = ''' INSERT INTO ori_measure_list (file_id, file_name, type, page_number, table_index, ori_measure_id, ori_measure_name, ori_measure_value, create_time, update_time, distance, pdf_measure,measure_id,measure_name,unit) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ''' #获取表格上方文字包含母公司字样的页码 select_parent_query = ''' select distinct content from measure_parser_info WHERE file_id = '{file_id}' and type='parent_com' '''.format(file_id=file_id) #获取表格上方文字黑名单关键词的页码和表格下标 select_table_index_query = ''' select distinct content from measure_parser_info WHERE file_id = '{file_id}' and type='table_index' '''.format(file_id=file_id) # #获取表格上方文字黑名单关键词的页码和表格下标----标题下的详细指标 select_measure_index_query = ''' SELECT content FROM measure_parser_info_linetext WHERE file_id = %s AND type = 'measure_index' ''' unit_query = ''' select unit from table_unit_info WHERE file_id = %s and page_num = %s and table_index = %s ''' cursor_app.execute(select_parent_query) parent_records = cursor_app.fetchall() #print(f"before: {parent_table_pages}") for parent_record in parent_records: parent_id = parent_record[0] parent_table_pages.append(int(parent_id)) #print(f"after: {parent_table_pages}") #表格上方文字黑名单关键词的页码和表格下标转成数组 table_index_array = [] cursor_app.execute(select_table_index_query) table_index_records = cursor_app.fetchall() for table_index_record in table_index_records: table_index_array.append(table_index_record[0]) # #仿照写法，指标的黑名单转化 measure_index_array = [] cursor_app.execute(select_measure_index_query, (file_id,)) measure_index_records = cursor_app.fetchall() print("Executing SQL:", select_measure_index_query) print("With file_id:", file_id) for measure_index_record in measure_index_records: measure_index_array.append(measure_index_record[0]) print(f'黑名单的值是{parent_table_pages}和{table_index_array}以及新增的{measure_index_array}') #print(f'黑名单的值是{parent_table_pages}和{table_index_array}') record_start = record_range.split('-')[0] record_end = record_range.split('-')[1] client = MilvusClient( uri=MILVUS_CLIENT, ) try: for index in range(int(record_start),int(record_end)): record = records[index] ori_measure_name = record[0] measure_name = record[1] distance = record[2] ori_measure_id = record[3] measure_id = record[4] measure_vector = redis_service.read_from_redis(redis_client,ori_measure_id) measure_list = ast.literal_eval(measure_vector) data = [measure_list] filter_str = 'file_id == "'+file_id+'"' res = client.search( collection_name="pdf_measure_v4", # Replace with the actual name of your collection # Replace with your query vector data=data, limit=3, # Max. number of search results to return search_params={"metric_type": "COSINE", "params": {}}, # Search parameters output_fields=["measure_name", "measure_value", "table_num", "table_index", "measure_unit"], filter=filter_str, partition_name=partition_name ) # Convert the output to a formatted JSON string # for i in range(len(res[0])): for i in range(len(res[0])): vector_distance = float(res[0][i]["distance"]) pdf_measure = res[0][i]["entity"]["measure_name"] measure_value = res[0][i]["entity"]["measure_value"] table_num = res[0][i]["entity"]["table_num"] table_index = res[0][i]["entity"]["table_index"] unit = res[0][i]["entity"]["measure_unit"] # if pdf_measure == '2023年6月30日货币资金合计': # print(f'{pdf_measure} 的相似度是 {vector_distance}，其值为 {measure_value}，页码在 {table_num}') #先过滤页码为0的情况，暂时不知道原因 if table_num == 0: continue #过滤表格上方文字黑名单关键词的页码和表格下标 if f"{table_num}_{table_index}" in table_index_array: continue #if f"{table_num}_{table_index}" in table_index_array and pdf_measure in (): #过滤指标中包含黑名单关键词 if utils.check_pdf_measure_black_list(pdf_measure): continue if f"{table_num}_{table_index}" in measure_index_array and utils.check_pdf_measure_black_list_v3(file_id,table_num,table_index,pdf_measure,conn_app,cursor_app): #if utils.check_pdf_measure_black_list_v3(file_id,table_num,table_index,pdf_measure,conn_app,cursor_app): print(f'经过第三层规则去除了{table_num}页的{pdf_measure}指标') continue if vector_distance > distance and table_num not in parent_table_pages: #检测规则开始 #判断抽取指标和财报指标周期是否相同 ori_period = utils.get_period_type(ori_measure_name, report_year) pdf_period = utils.get_period_type(pdf_measure, report_year) if pdf_measure == '2023年6月30日货币资金合计': print(f'第1处{ori_period}和{pdf_period}') if(ori_period != pdf_period): continue #判断抽取指标和财报指标是否期初指标 start_ori_period = utils.get_start_period_type(ori_measure_name) start_pdf_period = utils.get_start_period_type(pdf_measure) if pdf_measure == '2023年6月30日货币资金合计': print(f'第2处{start_ori_period}和{start_pdf_period}') if(start_ori_period != start_pdf_period): continue #判断抽取指标和财报指标类型是否相同，是否都是季度 ori_season_type = utils.get_season_flag(ori_measure_name) pdf_season_type = utils.get_season_flag(pdf_measure) if pdf_measure == '2023年6月30日货币资金合计': print(f'第3处{ori_season_type}和{pdf_season_type}') if(ori_season_type != pdf_season_type): continue #判断是否都是扣非指标 ori_kf_type = utils.get_kf_flag(ori_measure_name) pdf_kf_type = utils.get_kf_flag(pdf_measure) if pdf_measure == '2023年6月30日货币资金合计': print(f'第4处{ori_kf_type}和{pdf_kf_type}') if(ori_kf_type != pdf_kf_type): continue #判断抽取指标和财报指标类型是否相同，是否都是百分比 ori_type = utils.get_percent_flag(ori_measure_name) pdf_type = utils.get_percent_flag(pdf_measure) if pdf_measure == '2023年6月30日货币资金合计': print(f'第5处{ori_type}和{pdf_type}') if(ori_type != pdf_type): continue #判断抽取指标和财报指标类型是否相同，是否都是占比同比变动类 ori_growth_type = utils.get_percent_growth(ori_measure_name) pdf_growth_type = utils.get_percent_growth(pdf_measure) if pdf_measure == '2023年6月30日货币资金合计': print(f'第6处{ori_growth_type}和{pdf_growth_type}') if(ori_growth_type != pdf_growth_type): continue #解决指标语义是比率，但值为非比率的情况 if ori_growth_type == '1': check_measure_value = abs(float(measure_value)) if(check_measure_value > 10000): continue # 判断数据库中是否有数据 check_query_data = (file_id, measure_name, int(table_num), int(table_index), measure_value) cursor.execute(check_query, check_query_data) check_records = cursor.fetchall() if(len(check_records)) > 0: continue #判断是否包含黑名单 if(utils.check_black_list(measure_name,pdf_measure,black_array)): continue if(utils.check_white_list(measure_name,pdf_measure)): continue #判断抽取指标和财报指标类型是否都是增长类，比如同比变动为增长类 ori_change_type = utils.get_change_rate_flag(ori_measure_name) pdf_change_type = utils.get_change_rate_flag(pdf_measure) if(ori_change_type != pdf_change_type): continue #处理调整前，调整前、后同时出现，如果有调整前过滤 if pdf_measure.find('调整前') != -1 or pdf_measure.find('重述前') != -1: continue #判断指标是否报告期初 ori_report_start = utils.get_report_start(ori_measure_name) pdf_report_start = utils.get_report_start(pdf_measure) # if pdf_measure == '2023年6月30日货币资金合计': # print(f'第7处{ori_report_start}和{pdf_report_start}') if(ori_report_start != pdf_report_start): continue # #表格描述文字黑名单判断 # text_query_data = (file_id, int(table_num), int(table_index)) # cursor.execute(text_query, text_query_data) # text_records = cursor.fetchall() # if(len(text_records)) > 0: # text_info = '' # for text_record in text_records: # text_info += text_record[0] # if(utils.check_title_black_list(measure_name,text_info)): # continue #检测规则结束 #获取指标单位数据，除了百分比 if(utils.get_percent_flag(measure_name) == '0'): unit_query_data = (file_id, int(table_num), int(table_index)) cursor.execute(unit_query, unit_query_data) unit_records = cursor.fetchall() if unit != '' : pass elif unit == '' and (len(unit_records)) > 0: unit = unit_records[0][0] else: unit = '元' data_to_insert = (file_id, file_name, "table", int(table_num), int(table_index), ori_measure_id, ori_measure_name, measure_value, create_time, create_time, vector_distance, pdf_measure,measure_id,measure_name,unit) cursor.execute(insert_query, data_to_insert) conn.commit() except Exception as e: print(e) finally: parent_table_pages = [] client.close() redis_client.close() cursor.close() conn.close() def insert_table_measure_from_vector_async_process(cursor,parent_table_pages,file_id,file_name, partition_name): select_year_select = f"""select report_type,year from report_check where id = {file_id}""" cursor.execute(select_year_select) record_select = cursor.fetchall() report_type = record_select[0][0] report_year = record_select[0][1] select_query = ''' SELECT ori_measure_name,measure_name,distance,ori_measure_id,measure_id FROM measure_config where year = '{year}' '''.format(year=report_year) select_query_half_year = ''' SELECT ori_measure_name,measure_name,distance,ori_measure_id,measure_id FROM measure_config_half_year where year = '{year}' '''.format(year=report_year) select_query_thrid = ''' SELECT ori_measure_name,measure_name,distance,ori_measure_id,measure_id FROM measure_config_third_quarter where year = '{year}' '''.format(year=report_year) select_black_array_query = ''' SELECT measure_name, keywords FROM measure_black_list where isdel = 0 and find_in_set('{year}',year) and find_in_set('{flag}',flag) '''.format(year=report_year,flag = report_type) black_array = [] cursor.execute(select_black_array_query) results = cursor.fetchall() for row in results: category = row[0] keywords = row[1].split(',') black_array.append(f"{category}:{','.join(keywords)}") if report_type == 1: start_time = time.time() cursor.execute(select_query_half_year) records = cursor.fetchall() end_time = time.time() print(f"向量配置数据查询 {(end_time - start_time):.2f} 秒。") print('insert_table_measure_from_vector_async_process方法走的半年报') start_time = time.time() records_range_parts = utils.get_range(len(records),MEASURE_COUNT) processes = [] for record_range in records_range_parts: p = Process(target=insert_table_from_vector_mul_process, args=(parent_table_pages,file_id,file_name,records,record_range,black_array, partition_name)) processes.append(p) p.start() elif report_type == 3: start_time = time.time() cursor.execute(select_query_thrid) records = cursor.fetchall() end_time = time.time() print(f"向量配置数据查询 {(end_time - start_time):.2f} 秒。") print('insert_table_measure_from_vector_async_process方法走的三季报') start_time = time.time() records_range_parts = utils.get_range(len(records),MEASURE_COUNT) processes = [] for record_range in records_range_parts: p = Process(target=insert_table_from_vector_mul_process, args=(parent_table_pages,file_id,file_name,records,record_range,black_array,partition_name)) processes.append(p) p.start() # p.apply_async(insert_table_from_vector_mul, args=(parent_table_pages,file_id,file_name,records,record_range,)) else: start_time = time.time() cursor.execute(select_query) records = cursor.fetchall() end_time = time.time() print(f"向量配置数据查询 {(end_time - start_time):.2f} 秒。") print('insert_table_measure_from_vector_async_process方法走的全年报') start_time = time.time() records_range_parts = utils.get_range(len(records),MEASURE_COUNT) processes = [] for record_range in records_range_parts: p = Process(target=insert_table_from_vector_mul_process, args=(parent_table_pages,file_id,file_name,records,record_range,black_array,partition_name)) processes.append(p) p.start() print('等待所有子任务完成，任务ID:', file_id) for p in processes: p.join() print('所有子任务完成，任务ID:', file_id) print('启动指标归一化任务ID:', file_id) end_time = time.time() print(f"向量更新时间 {(end_time - start_time):.2f} 秒。") def insert_table_measure_from_vector(conn,cursor,client,parent_table_pages,file_id,file_name): create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") select_query = ''' SELECT ori_measure_name,measure_name,distance,ori_measure_id,measure_id FROM measure_config ''' check_query = ''' select id from ori_measure_list WHERE file_id = %s and measure_name = %s and page_number = %s and table_index = %s and ori_measure_value = %s ''' insert_query = ''' INSERT INTO ori_measure_list (file_id, file_name, type, page_number, table_index, ori_measure_id, ori_measure_name, ori_measure_value, create_time, update_time, distance, pdf_measure,measure_id,measure_name,unit) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ,%s) ''' select_year_select = f"""select report_type,year from report_check where id = {file_id}""" cursor.execute(select_year_select) record_select = cursor.fetchall() report_type = record_select[0][0] report_year = record_select[0][1] start_time = time.time() cursor.execute(select_query) records = cursor.fetchall() end_time = time.time() print(f"向量配置数据查询 {(end_time - start_time):.2f} 秒。") start_time = time.time() try: for record in records: ori_measure_name = record[0] measure_name = record[1] distance = record[2] ori_measure_id = record[3] measure_id = record[4] measure_vector = redis_service.read_from_redis(ori_measure_id) measure_list = ast.literal_eval(measure_vector) data = [measure_list] filter_str = 'file_id == "'+file_id+'"' res = client.search( collection_name="pdf_measure_v4", # Replace with the actual name of your collection # Replace with your query vector data=data, limit=3, # Max. number of search results to return search_params={"metric_type": "COSINE", "params": {}}, # Search parameters output_fields=["measure_name","measure_value","table_num","table_index","measure_unit"], filter=filter_str ) # Convert the output to a formatted JSON string for i in range(len(res[0])): vector_distance = float(res[0][i]["distance"]) pdf_measure = res[0][i]["entity"]["measure_name"] measure_value = res[0][i]["entity"]["measure_value"] table_num = res[0][i]["entity"]["table_num"] table_index = res[0][i]["entity"]["table_index"] measure_unit = res[0][i]["entity"]["measure_unit"] if vector_distance > distance and table_num not in parent_table_pages: #检测规则开始 #判断抽取指标和财报指标周期是否相同 ori_period = utils.get_period_type(ori_measure_name, report_year) pdf_period = utils.get_period_type(pdf_measure, report_year) if(ori_period != pdf_period): continue #判断抽取指标和财报指标类型是否相同，是否都是百分比 ori_type = utils.get_percent_flag(ori_measure_name) pdf_type = utils.get_percent_flag(pdf_measure) if(ori_type != pdf_type): continue # 判断数据库中是否有数据 check_query_data = (file_id, measure_name, int(table_num), int(table_index), measure_value) cursor.execute(check_query, check_query_data) check_records = cursor.fetchall() if(len(check_records)) > 0: continue #检测规则结束 data_to_insert = (file_id, file_name, "table", int(table_num), int(table_index), ori_measure_id, ori_measure_name, measure_value, create_time, create_time, vector_distance, pdf_measure,measure_id,measure_name,measure_unit) cursor.execute(insert_query, data_to_insert) conn.commit() except Exception as e: print(e) end_time = time.time() print(f"向量更新数据时间 {(end_time - start_time):.2f} 秒。") start_time = time.time() def insert_measure_data_to_milvus(client,partition_name,table_info,cursor,conn): insert_query = ''' INSERT INTO measure_parse_process (file_id, page_num, content) VALUES (%s, %s, %s) ''' for table in table_info: try: data=[] table_num = table['page_num'].split("_")[0] file_id = table['file_id'] table_index = table['page_num'].split("_")[1] measure_list = table['measure_list'] for measure in measure_list: measure_name = measure['measure_name'] measure_value = measure['measure_value'].replace("(", "").replace(")", "") measure_name = utils.get_clean_text(measure_name) measure_name = measure_name.replace('2023','2023年').replace('2022','2022年').replace('（','').replace('）','')#这个真绝了，怎么都删不掉 #measure_name_1 = measure_name.replace('调整后','') quarters = ['第一季度', '第二季度', '第三季度', '第四季度','增减','2023年','2022年','2021年','年'] for quarter in quarters: measure_name = measure_name.replace(quarter * 2, quarter) pattern_dup = re.compile(r'(\w{3,})\1+')#去掉任意超过两个字且重复的字符 matches = pattern_dup.findall(measure_name) # for match in matches: # print(f"被删除的字符: {match * 2}") measure_name = pattern_dup.sub(r'\1', measure_name) measure_name_1 = measure_name.replace('调整后','').replace('上年期末数','上年期末').replace('上年期末','上年年末') measure_unit = measure['measure_unit'] if re.match(r'^[+-]?(\d+(\.\d*)?|\.\d+)(%?)$', measure_value) and any(key_word in measure_name for key_word in measure_name_keywords): vector_obj = utils.embed_with_str(measure_name_1) vector = vector_obj.output["embeddings"][0]["embedding"] measure_data = {} measure_data['vector'] = vector measure_data['table_num'] = int(table_num) measure_data['table_index'] = int(table_index) measure_data['measure_name'] = measure_name measure_data['measure_value'] = measure_value measure_data['measure_unit'] = measure_unit measure_data['file_id'] = file_id data.append(measure_data) # 指标数据写入指标解析过程表，用于前端展示 content = f"{measure_name}:{measure_value}" data_to_insert = (file_id, table_num, content) cursor.execute(insert_query, data_to_insert) conn.commit() elif re.match(r'(增加|减少|下降|上升)[了]?(\d+\.\d+)[个]?百分点', measure_value) and any(key_word in measure_name for key_word in measure_name_keywords): #特殊处理指标值为增加了/减少了 XXX 个百分点 unit_pattern = re.compile(r'(增加|减少|下降|上升)[了]?(\d+\.\d+)[个]?百分点') match = unit_pattern.search(measure_value) if match and len(match.groups()) == 2: crease_type = match.group(1) measure_value = match.group(2) if crease_type == '减少' or crease_type == '下降': measure_value = f'-{match.group(2)}' vector_obj = utils.embed_with_str(measure_name_1) vector = vector_obj.output["embeddings"][0]["embedding"] measure_data = {} measure_data['vector'] = vector measure_data['table_num'] = int(table_num) measure_data['table_index'] = int(table_index) measure_data['measure_name'] = measure_name measure_data['measure_value'] = measure_value measure_data['measure_unit'] = measure_unit measure_data['file_id'] = file_id data.append(measure_data) # 指标数据写入指标解析过程表，用于前端展示 content = f"{measure_name}:{measure_value}" data_to_insert = (file_id, table_num, content) cursor.execute(insert_query, data_to_insert) conn.commit() else: pass#print(f"数据值的格式错误:{measure_value}。或者字段名不在名单内{measure_name}") res = client.insert( collection_name="pdf_measure_v4", data=data, partition_name=partition_name ) except Exception as e: print(e) def runing_job(): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) # 创建一个cursor对象来执行SQL语句 cursor = conn.cursor(buffered=True) select_query = ''' SELECT * FROM report_check where status = 0 and isdel=0 ''' cursor.execute(select_query) records = cursor.fetchall() if(len(records)) > 1: return True return False def insert_pdf_parse_process(parser_info,conn,cursor): # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO pdf_parse_process (file_id, page_num, page_count, content, type) VALUES (%s, %s, %s, %s, %s) ''' file_id = parser_info['file_id'] page_num = int(parser_info['page_num']) page_count = int(parser_info['page_count']) content = json.dumps(parser_info['content'], ensure_ascii=False) type = parser_info['type'] data_to_insert = (file_id, page_num, page_count, content, type) cursor.execute(insert_query, data_to_insert) conn.commit() def delete_database(conn,cursor,file_id): try: truncate_query = [ "delete from measure_parse_process where file_id = %s;", "delete from measure_parser_info where file_id = %s;", "delete from pdf_parse_process where file_id = %s;", "delete from table_unit_info where file_id = %s;", # "delete from a where file_id = %s;", # "delete from b where file_id = %s;", ] #file_id = file_id for truncate in truncate_query: cursor.execute(truncate,(file_id,)) conn.commit() except Exception as e: print(f'删除失败,原因是{e}') def delete_to_run(conn,cursor,file_id): try: truncate_query = [ "delete from ori_measure_list where file_id = %s;", "delete from measure_list where file_id = %s;", "delete from check_measure_list where file_id = %s;", "delete from check_measure_detail_list where file_id = %s;", # "delete from table_unit_info where file_id = %s;", # "delete from pdf_parse_process where file_id = %s;", # "delete from table_unit_info where file_id = %s;", # "delete from a where file_id = %s;", # "delete from b where file_id = %s;", ] #file_id = file_id for truncate in truncate_query: cursor.execute(truncate,(file_id,)) conn.commit() except Exception as e: print(f'删除失败,原因是{e}') def insert_pdf_text_info(table_info,conn,cursor): # 执行SQL语句，插入数据 insert_query = ''' INSERT INTO pdf_text_info (file_id, page_num, text) VALUES (%s, %s, %s) ''' file_id = table_info['file_id'] page_num = int(table_info['page_num']) text = table_info['text'] data_to_insert = (file_id, page_num, text) cursor.execute(insert_query, data_to_insert) conn.commit() def process_time(file_id,type,time,start_time,end_time): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) cursor = conn.cursor(buffered=True) time = round(time, 2) start_time = datetime.fromtimestamp(start_time).strftime('%Y-%m-%d %H:%M:%S') end_time = datetime.fromtimestamp(end_time).strftime('%Y-%m-%d %H:%M:%S') insert_query = ''' insert into pdf_process_time (file_id,type,time,start_time,end_time) values (%s, %s, %s,%s,%s) ''' data_insert = (file_id,type,time,start_time,end_time) cursor.execute(insert_query,data_insert) conn.commit() def batch_insert_page_text_nocheck(table_info, conn, cursor): file_id = table_info['file_id'] page_num = int(table_info['page_num']) text_lines = table_info['text'] insert_query = ''' INSERT INTO pdf_text_info (file_id, page_num, text) VALUES (%s, %s, %s) ''' data_to_insert = [(file_id, page_num, text) for text in text_lines] cursor.executemany(insert_query, data_to_insert) conn.commit() def batch_insert_page_text(table_info, conn, cursor): file_id = table_info['file_id'] page_num = int(table_info['page_num']) text_lines = table_info['text'] # 1. 检查表是否为空 check_if_empty_query = f"SELECT COUNT(*) FROM pdf_text_info where file_id = '{file_id}' and page_num = {page_num}" cursor.execute(check_if_empty_query) is_table_empty = cursor.fetchone()[0] == 0 if is_table_empty: # 表为空，直接插入数据 insert_query = ''' INSERT INTO pdf_text_info (file_id, page_num, text) VALUES (%s, %s, %s) ''' data_to_insert = [(file_id, page_num, text) for text in text_lines] cursor.executemany(insert_query, data_to_insert) else: pass conn.commit() def file_type_check(file_id): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) cursor = conn.cursor(buffered=True) try: select_query = ''' SELECT report_type FROM report_check WHERE id = %s ''' cursor.execute(select_query, (file_id,)) record = cursor.fetchone() if record and record[0] == 5: return True return False finally: cursor.close() conn.close() def file_type_check_v2(file_id): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) cursor = conn.cursor(buffered=True) try: select_query = ''' SELECT report_type FROM report_check WHERE id = %s ''' cursor.execute(select_query, (file_id,)) record = cursor.fetchone() return record[0] # if record and == 5: # return True # return False finally: cursor.close() conn.close() def pdf_title_insert_mysql(file_id,title_array): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) cursor = conn.cursor(buffered=True) for item in title_array: # 注意这里的缩进，它是循环体的一部分 cursor.execute("INSERT INTO pdf_title_info (file_id,title, page_num, depth) VALUES (%s,%s, %s, %s)", (file_id,item['title'], item['page_num'], item['depth'])) conn.commit() cursor.close() conn.close() def get_file_info_from_mysql(file_id): conn = mysql.connector.connect( host= MYSQL_HOST, user= MYSQL_USER, password= MYSQL_PASSWORD, database= MYSQL_DB ) #cursor = conn.cursor(buffered=True) cursor = conn.cursor(dictionary=True) select_query = """ SELECT title, page_num, depth FROM pdf_title_info WHERE file_id = %s """ cursor.execute(select_query, (file_id,)) result = cursor.fetchall() cursor.close() conn.close() return result