155 lines
6.1 KiB
Python
155 lines
6.1 KiB
Python
#coding=utf-8
|
||
import sys,ast
|
||
from pdfminer.high_level import extract_text
|
||
from pdfminer.pdfparser import PDFParser
|
||
from pdfminer.pdfdocument import PDFDocument
|
||
from pdfminer.pdfpage import PDFPage
|
||
import utils
|
||
import mysql.connector
|
||
from pymilvus import connections,MilvusClient
|
||
import json
|
||
import db_service
|
||
import ast
|
||
import numpy as np
|
||
import config
|
||
import redis_service
|
||
from config import MILVUS_CLIENT,MYSQL_HOST,MYSQL_USER,MYSQL_PASSWORD,MYSQL_DB
|
||
import main
|
||
import redis
|
||
|
||
def measure_config_to_db(conn,cursor):
|
||
insert_query = '''
|
||
INSERT INTO measure_config
|
||
(measure_id, measure_name, ori_measure_id, ori_measure_name)
|
||
VALUES (%s, %s, %s, %s)
|
||
'''
|
||
check_query = '''
|
||
select ori_measure_id from measure_config
|
||
'''
|
||
# 打开文本文件
|
||
with open('/Users/zhengfei/work/zzb_data/measure_config_all.txt', 'r') as file:
|
||
# 读取所有行到一个列表中
|
||
lines = file.readlines()
|
||
|
||
# 打印每一行
|
||
for line in lines:
|
||
config_list = line.strip().split(',')
|
||
measure = config_list[0]
|
||
ori_measure = config_list[1]
|
||
ori_measure_id = utils.get_md5(ori_measure)
|
||
# 判断数据库中是否有数据
|
||
# cursor.execute(check_query.format(ori_measure_id=ori_measure_id))
|
||
# check_records = cursor.fetchall()
|
||
# if(len(check_records)) > 0:
|
||
# continue
|
||
data_to_insert = (utils.get_md5(measure), measure, ori_measure_id, ori_measure)
|
||
cursor.execute(insert_query, data_to_insert)
|
||
conn.commit()
|
||
|
||
def insert_measure_vector(conn,cursor):
|
||
|
||
redis_client = redis.Redis(host='192.168.0.172', port=6379, password='Xgf_redis', db=6)
|
||
# 执行SQL语句，更新数据
|
||
select_query = '''
|
||
SELECT ori_measure_id,ori_measure_name FROM measure_config
|
||
'''
|
||
cursor.execute(select_query)
|
||
records = cursor.fetchall()
|
||
for record in records:
|
||
if redis_client.hexists('measure_config', record[0]):
|
||
measure_vector = redis_client.hget('measure_config', record[0])
|
||
else:
|
||
print('新增指标',record[1])
|
||
vector_obj = utils.embed_with_str(record[1])
|
||
measure_vector = str(vector_obj.output["embeddings"][0]["embedding"])
|
||
|
||
redis_client.hset('measure_config', record[0], measure_vector)
|
||
redis_client.close()
|
||
conn.close()
|
||
|
||
def contains_financial_indicators(text):
|
||
import re
|
||
# 正则表达式模式匹配千分位格式的数字和百分比
|
||
pattern = r"\d{1,3}(,\d{3})+(\.\d{1,3})?"
|
||
|
||
pattern1 = r"\d+(.\d+)+%?"
|
||
# 使用 re.search 函数查找匹配项
|
||
match = re.search(pattern1, text)
|
||
|
||
# 如果找到匹配项，返回 True，否则返回 False
|
||
return bool(match)
|
||
|
||
def get_clean_text(text):
|
||
import re
|
||
pattern = r"\（[^)]*?\）"
|
||
matches = re.findall(pattern, text)
|
||
for match in matches:
|
||
# 使用 re.findall 函数查找括号内的内容中是否包含月份或关键词
|
||
month_keywords_found = re.search(r"归属于|扣非", match)
|
||
if not month_keywords_found:
|
||
# 如果包含，则从文本中删除该部分
|
||
text = re.sub(pattern,"", text)
|
||
else:
|
||
# 如果不包含，删除所有标点符号和中文数字
|
||
text = re.sub(r"[^\w\s]", "", text)
|
||
print(text)
|
||
|
||
def insert_and_update(conn,cursor,client,parent_table_pages,file_id,path):
|
||
# #通过向量查询指标
|
||
db_service.insert_table_measure_from_vector(conn,cursor,client,parent_table_pages,file_id,path)
|
||
|
||
# #指标归一化处理
|
||
db_service.update_ori_measure(conn,cursor,file_id)
|
||
|
||
def print_measure_data(cursor,client):
|
||
select_query = '''
|
||
SELECT ori_measure_name,measure_name,ori_measure_id FROM measure_config
|
||
where measure_id not in(select distinct measure_id from ori_measure_list where file_id='64')
|
||
'''
|
||
cursor.execute(select_query)
|
||
records = cursor.fetchall()
|
||
for record in records:
|
||
ori_measure_name = record[0]
|
||
measure_name = record[1]
|
||
ori_measure_id = record[2]
|
||
measure_vector = redis_service.read_from_redis(ori_measure_id)
|
||
|
||
measure_list = ast.literal_eval(measure_vector)
|
||
data = [measure_list]
|
||
res = client.search(
|
||
collection_name="pdf_measure_v4", # Replace with the actual name of your collection
|
||
# Replace with your query vector
|
||
data=data,
|
||
limit=2, # Max. number of search results to return
|
||
search_params={"metric_type": "COSINE", "params": {}}, # Search parameters
|
||
output_fields=["measure_name","measure_value","table_num","table_index"],
|
||
filter = 'file_id == "64"'
|
||
)
|
||
vector_str = measure_name+":"+ori_measure_name
|
||
# Convert the output to a formatted JSON string
|
||
for i in range(len(res[0])):
|
||
|
||
vector_distance = float(res[0][i]["distance"])
|
||
vector_measure_name = res[0][i]["entity"]["measure_name"]
|
||
measure_value = res[0][i]["entity"]["measure_value"]
|
||
table_num = res[0][i]["entity"]["table_num"]
|
||
table_index = res[0][i]["entity"]["table_index"]
|
||
table_num_list = [106]
|
||
print(vector_str +":"+vector_measure_name+":"+str(vector_distance) +":"+measure_value +":"+str(table_num) +":"+str(table_index))
|
||
# if vector_distance > 0.89 and table_num not in table_num_list:
|
||
# print(vector_str +":"+vector_measure_name+":"+str(vector_distance) +":"+measure_value +":"+str(table_num) +":"+str(table_index)+":"+str(0.94))
|
||
# if vector_distance > distance and table_num not in table_num_list:
|
||
# print(vector_str +":"+vector_measure_name +":"+measure_value +":"+str(table_num) +":"+str(table_index)+":"+str(vector_distance)+":"+str(distance))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
conn = mysql.connector.connect(
|
||
host=MYSQL_HOST,
|
||
user=MYSQL_USER,
|
||
password=MYSQL_PASSWORD,
|
||
database=MYSQL_DB
|
||
)
|
||
cursor = conn.cursor()
|
||
|
||
insert_measure_vector(conn,cursor)
|