269 lines
10 KiB
Python
269 lines
10 KiB
Python
|
from config import MYSQL_HOST,MYSQL_USER,MYSQL_PASSWORD,MYSQL_DB
|
|||
|
|
import mysql.connector
|
|||
|
|
from http import HTTPStatus
|
|||
|
|
import dashscope
|
|||
|
|
import random,re
|
|||
|
|
from pdfminer.high_level import extract_pages
|
|||
|
|
from pdfminer.layout import LTTextBoxHorizontal
|
|||
|
|
import PyPDF2
|
|||
|
|
dashscope.api_key='sk-63c02fbb9b7d4b0494a3200bec1ae286'
|
|||
|
|
|
|||
|
|
def get_company_name(file_path):
|
|||
|
|
line_text = ''
|
|||
|
|
# 我们从PDF中提取页面,page_numbers=[4,5,6]
|
|||
|
|
for pagenum, page in enumerate(extract_pages(file_path)):
|
|||
|
|
if pagenum > 1:
|
|||
|
|
break
|
|||
|
|
# 找到所有的元素
|
|||
|
|
page_elements = [(element.y1, element) for element in page._objs]
|
|||
|
|
# 查找组成页面的元素
|
|||
|
|
for i,component in enumerate(page_elements):
|
|||
|
|
# 提取页面布局的元素
|
|||
|
|
element = component[1]
|
|||
|
|
# 检查该元素是否为文本元素
|
|||
|
|
if isinstance(element, LTTextBoxHorizontal):
|
|||
|
|
# 检查文本是否出现在表中
|
|||
|
|
line_text += element.get_text()
|
|||
|
|
|
|||
|
|
return llm_service(line_text)
|
|||
|
|
def get_company_code(file_path):
|
|||
|
|
line_text = ''
|
|||
|
|
# 我们从PDF中提取页面,page_numbers=[4,5,6]
|
|||
|
|
for pagenum, page in enumerate(extract_pages(file_path)):
|
|||
|
|
if pagenum > 1:
|
|||
|
|
break
|
|||
|
|
# 找到所有的元素
|
|||
|
|
page_elements = [(element.y1, element) for element in page._objs]
|
|||
|
|
# 查找组成页面的元素
|
|||
|
|
for i,component in enumerate(page_elements):
|
|||
|
|
# 提取页面布局的元素
|
|||
|
|
element = component[1]
|
|||
|
|
# 检查该元素是否为文本元素
|
|||
|
|
if isinstance(element, LTTextBoxHorizontal):
|
|||
|
|
# 检查文本是否出现在表中
|
|||
|
|
line_text += element.get_text()
|
|||
|
|
|
|||
|
|
return llm_service_code(line_text)
|
|||
|
|
#获取公司简介的那一页
|
|||
|
|
# def get_code_page(pdf_path):
|
|||
|
|
# with open(pdf_path, 'rb') as file:
|
|||
|
|
# reader = PyPDF2.PdfReader(file)
|
|||
|
|
# outlines = reader.outline
|
|||
|
|
# company_profile_page = None
|
|||
|
|
|
|||
|
|
# def find_page_from_outlines(outlines):
|
|||
|
|
# nonlocal company_profile_page
|
|||
|
|
# for item in outlines:
|
|||
|
|
# if isinstance(item, list): # 如果是子目录，则递归
|
|||
|
|
# find_page_from_outlines(item)
|
|||
|
|
# else:
|
|||
|
|
# title = item.title
|
|||
|
|
# if title is not None and '公司简介' in title:
|
|||
|
|
# # 获取页面的实际页码
|
|||
|
|
# page_num = reader.get_destination_page_number(item)
|
|||
|
|
# company_profile_page = page_num
|
|||
|
|
# return
|
|||
|
|
# # 处理没有标题的情况
|
|||
|
|
# elif item.page is not None:
|
|||
|
|
# page_num = reader.get_destination_page_number(item)
|
|||
|
|
# if page_num is not None:
|
|||
|
|
# pass
|
|||
|
|
|
|||
|
|
# find_page_from_outlines(outlines)
|
|||
|
|
|
|||
|
|
# return company_profile_page
|
|||
|
|
|
|||
|
|
# def get_company_code(file_path):
|
|||
|
|
# line_text = ''
|
|||
|
|
# # 我们从PDF中提取页面,page_numbers=[4,5,6]
|
|||
|
|
# for pagenum, page in enumerate(extract_pages(file_path)):
|
|||
|
|
# print(f'页码是{get_code_page(file_path)+1}')
|
|||
|
|
# if pagenum > 1 and pagenum != get_code_page(file_path)+1:
|
|||
|
|
# break
|
|||
|
|
# # 找到所有的元素
|
|||
|
|
# #print(pagenum)
|
|||
|
|
# page_elements = [(element.y1, element) for element in page._objs]
|
|||
|
|
# # 查找组成页面的元素
|
|||
|
|
# # for i,component in enumerate(page_elements):
|
|||
|
|
# # # 提取页面布局的元素
|
|||
|
|
# # element = component[1]
|
|||
|
|
# # # 检查该元素是否为文本元素
|
|||
|
|
# # if isinstance(element, LTTextBoxHorizontal):
|
|||
|
|
# # # 检查文本是否出现在表中
|
|||
|
|
# # line_text += element.get_text()
|
|||
|
|
# for _, element in page_elements:
|
|||
|
|
# if isinstance(element, LTTextBoxHorizontal):
|
|||
|
|
# # 提取文本并添加到 line_text
|
|||
|
|
# line_text += element.get_text()
|
|||
|
|
|
|||
|
|
# return llm_service_code(line_text)
|
|||
|
|
def llm_service(user_prompt):
|
|||
|
|
|
|||
|
|
system_prompt = '''
|
|||
|
|
从以下数据报告中提取公司全称，只需要提取中文公司全称，不要增加其他内容，如果提取不到公司全称，请返回-，不要返回其他任何内容。
|
|||
|
|
<数据报告>
|
|||
|
|
<user_prompt>
|
|||
|
|
</数据报告>
|
|||
|
|
'''
|
|||
|
|
system_prompt = system_prompt.replace('<user_prompt>', user_prompt)
|
|||
|
|
response = dashscope.Generation.call(
|
|||
|
|
model='qwen-plus',
|
|||
|
|
prompt = system_prompt,
|
|||
|
|
seed=random.randint(1, 10000),
|
|||
|
|
top_p=0.8,
|
|||
|
|
result_format='message',
|
|||
|
|
enable_search=False,
|
|||
|
|
max_tokens=1500,
|
|||
|
|
temperature=0.85,
|
|||
|
|
repetition_penalty=1.0
|
|||
|
|
)
|
|||
|
|
if response.status_code == HTTPStatus.OK:
|
|||
|
|
result = response['output']['choices'][0]['message']['content']
|
|||
|
|
return result
|
|||
|
|
else:
|
|||
|
|
print('Request id: %s, Status code: %s, error code: %s, error message: %s' % (
|
|||
|
|
response.request_id, response.status_code,
|
|||
|
|
response.code, response.message
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
return "llm_error"
|
|||
|
|
def llm_service_code(user_prompt):
|
|||
|
|
|
|||
|
|
system_prompt = '''
|
|||
|
|
从以下数据报告中提取6位数字的股票代码，只需要提取股票代码，如果有多个则以','隔开，不要增加其他内容，如果提取不到股票代码，请返回-,不要返回其他任何内容。
|
|||
|
|
<数据报告>
|
|||
|
|
<user_prompt>
|
|||
|
|
</数据报告>
|
|||
|
|
'''
|
|||
|
|
system_prompt = system_prompt.replace('<user_prompt>', user_prompt)
|
|||
|
|
response = dashscope.Generation.call(
|
|||
|
|
model='qwen-plus',
|
|||
|
|
prompt = system_prompt,
|
|||
|
|
seed=random.randint(1, 10000),
|
|||
|
|
top_p=0.8,
|
|||
|
|
result_format='message',
|
|||
|
|
enable_search=False,
|
|||
|
|
max_tokens=1500,
|
|||
|
|
temperature=0.85,
|
|||
|
|
repetition_penalty=1.0
|
|||
|
|
)
|
|||
|
|
if response.status_code == HTTPStatus.OK:
|
|||
|
|
result = response['output']['choices'][0]['message']['content']
|
|||
|
|
return result
|
|||
|
|
else:
|
|||
|
|
print('Request id: %s, Status code: %s, error code: %s, error message: %s' % (
|
|||
|
|
response.request_id, response.status_code,
|
|||
|
|
response.code, response.message
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
return "llm_error"
|
|||
|
|
def update_company_name(file_id, company_name,company_code, cursor, conn):
|
|||
|
|
update_sql = f'''
|
|||
|
|
UPDATE report_check
|
|||
|
|
SET c_name = '{company_name}',c_code = '{company_code}'
|
|||
|
|
WHERE id = {file_id}
|
|||
|
|
'''
|
|||
|
|
cursor.execute(update_sql)
|
|||
|
|
conn.commit()
|
|||
|
|
def name_code_fix(file_id,file_path):
|
|||
|
|
conn = mysql.connector.connect(
|
|||
|
|
host = MYSQL_HOST,
|
|||
|
|
user = MYSQL_USER,
|
|||
|
|
password = MYSQL_PASSWORD,
|
|||
|
|
database = MYSQL_DB
|
|||
|
|
)
|
|||
|
|
# 创建一个cursor对象来执行SQL语句
|
|||
|
|
cursor = conn.cursor()
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# file_id = data[0]
|
|||
|
|
# #生产环境地址
|
|||
|
|
# file_path = f'/usr/local/zhanglei/financial{data[1]}'
|
|||
|
|
# #测试环境地址
|
|||
|
|
# # file_path_1 = f'/root/pdf_parser/pdf/{data[1]}'
|
|||
|
|
# # file_path = file_path_1.replace('/upload/file/','')
|
|||
|
|
# print(f'财报{file_id}开始解析')
|
|||
|
|
# #file_id = '305'
|
|||
|
|
# #file_path = r"F:\11_pdf\7874.pdf"
|
|||
|
|
company_name = get_company_name(file_path)
|
|||
|
|
contains_newline = '\n' in company_name
|
|||
|
|
if contains_newline:
|
|||
|
|
lines = company_name.splitlines(True)
|
|||
|
|
company_name = lines[0]
|
|||
|
|
|
|||
|
|
company_code = get_company_code(file_path)
|
|||
|
|
contains_newline1 = '\n' in company_code
|
|||
|
|
if contains_newline1:
|
|||
|
|
lines = company_code.splitlines(True)
|
|||
|
|
company_code = lines[0]
|
|||
|
|
|
|||
|
|
if company_name != "llm_error" or company_code != "llm_error":
|
|||
|
|
#print(company_code)
|
|||
|
|
pattern = re.compile(r'^(\d{6}|\d{6}(,\d{6})*)$')
|
|||
|
|
if not pattern.match(company_code):
|
|||
|
|
company_code = '-'
|
|||
|
|
if len(company_name) > 15 or company_name == '-':
|
|||
|
|
company_name = ''
|
|||
|
|
update_company_name(file_id, company_name,company_code, cursor, conn)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f'财报解析失败',e)
|
|||
|
|
|
|||
|
|
cursor.close()
|
|||
|
|
conn.close()
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
conn = mysql.connector.connect(
|
|||
|
|
host = MYSQL_HOST,
|
|||
|
|
user = MYSQL_USER,
|
|||
|
|
password = MYSQL_PASSWORD,
|
|||
|
|
database = MYSQL_DB
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 创建一个cursor对象来执行SQL语句
|
|||
|
|
cursor = conn.cursor()
|
|||
|
|
|
|||
|
|
data_query = '''
|
|||
|
|
SELECT id,file_path FROM report_check where c_code is null
|
|||
|
|
'''
|
|||
|
|
|
|||
|
|
cursor.execute(data_query)
|
|||
|
|
data_list = cursor.fetchall()
|
|||
|
|
|
|||
|
|
for data in data_list:
|
|||
|
|
try:
|
|||
|
|
file_id = data[0]
|
|||
|
|
#生产环境地址
|
|||
|
|
file_path = f'/usr/local/zhanglei/financial{data[1]}'
|
|||
|
|
#测试环境地址
|
|||
|
|
# file_path_1 = f'/root/pdf_parser/pdf/{data[1]}'
|
|||
|
|
# file_path = file_path_1.replace('/upload/file/','')
|
|||
|
|
print(f'财报{file_id}开始解析')
|
|||
|
|
#file_id = '305'
|
|||
|
|
#file_path = r"F:\11_pdf\7874.pdf"
|
|||
|
|
|
|||
|
|
company_name = get_company_name(file_path)
|
|||
|
|
contains_newline = '\n' in company_name
|
|||
|
|
if contains_newline:
|
|||
|
|
lines = company_name.splitlines(True)
|
|||
|
|
company_name = lines[0]
|
|||
|
|
|
|||
|
|
company_code = get_company_code(file_path)
|
|||
|
|
contains_newline1 = '\n' in company_code
|
|||
|
|
if contains_newline1:
|
|||
|
|
lines = company_code.splitlines(True)
|
|||
|
|
company_code = lines[0]
|
|||
|
|
|
|||
|
|
if company_name != "llm_error" or company_code != "llm_error":
|
|||
|
|
#print(company_code)
|
|||
|
|
pattern = re.compile(r'^(\d{6}|\d{6}(,\d{6})*)$')
|
|||
|
|
if not pattern.match(company_code):
|
|||
|
|
company_code = '-'
|
|||
|
|
update_company_name(file_id, company_name,company_code, cursor, conn)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f'财报解析失败',e)
|
|||
|
|
|
|||
|
|
cursor.close()
|
|||
|
|
conn.close()
|