240 lines
9.9 KiB
Python
240 lines
9.9 KiB
Python
import PyPDF2
|
||
import re
|
||
import os,threading
|
||
from config import REDIS_HOST,REDIS_PORT,REDIS_PASSWORD
|
||
import redis
|
||
import db_service
|
||
def get_tree_pages(root, info, depth=0,title_array=[]):
|
||
"""
|
||
Recursively iterate the outline tree
|
||
Find the pages pointed by the outline item
|
||
and get the assigned physical order id
|
||
|
||
Decrement with padding if necessary
|
||
"""
|
||
|
||
if isinstance(root, dict):
|
||
# print(root)
|
||
page = root['/Page'].get_object()
|
||
# print(id(page))
|
||
t = root['/Title']
|
||
title = t
|
||
if isinstance(t, PyPDF2.generic.ByteStringObject):
|
||
title = t.original_bytes.decode('utf8')
|
||
title = title.strip()
|
||
title = title.replace('\n', '')
|
||
title = title.replace('\r', '')
|
||
|
||
page_num = info['all_pages'].get(id(page), 0)
|
||
if page_num == 0:
|
||
print('Not found page number for /Page!', page)
|
||
elif page_num < info['padding']:
|
||
page_num = 0
|
||
else:
|
||
page_num -= info['padding']
|
||
|
||
|
||
# str_val = '%-5d' % page_num
|
||
# str_val += '\t' * depth
|
||
# str_val += title + '\t' + '%3d' % page_num
|
||
# print(str_val)
|
||
title_array.append({
|
||
'title': title,
|
||
'page_num': page_num,
|
||
'depth': depth
|
||
})
|
||
for elem in root:
|
||
get_tree_pages(elem, info, depth+1,title_array)
|
||
return title_array
|
||
|
||
|
||
def recursive_numbering(obj, info):
|
||
"""
|
||
Recursively iterate through all the pages in order and assign them a physical
|
||
order number
|
||
"""
|
||
# print(id(obj), obj)
|
||
if obj['/Type'] == '/Page':
|
||
obj_id = id(obj)
|
||
if obj_id not in info['all_pages']:
|
||
info['all_pages'][obj_id] = info['current_page_id']
|
||
info['current_page_id'] += 1
|
||
return
|
||
elif obj['/Type'] == '/Pages':
|
||
for page in obj['/Kids']:
|
||
recursive_numbering(page.get_object(), info)
|
||
|
||
def get_numbers_between(numbers_between,start, end):
|
||
# 初始化一个空列表来存储两个数字之间的所有数字
|
||
|
||
# 遍历从开始数字到结束数字之间的每个数字
|
||
for i in range(start, end + 1):
|
||
# 将每个数字添加到列表中
|
||
numbers_between.append(i)
|
||
return numbers_between
|
||
|
||
def get_page_end(start, depth, title_array):
|
||
page_end = -1
|
||
for i in range(start, len(title_array)):
|
||
if title_array[i]['depth'] == depth:
|
||
page_end = title_array[i]['page_num']
|
||
break
|
||
return page_end
|
||
|
||
def get_file_split(page_count):
|
||
# 获取 CPU 核数
|
||
cpu_count = os.cpu_count()
|
||
if page_count < cpu_count:
|
||
cpu_count = page_count
|
||
# 使用 divmod() 函数计算除法结果和余数
|
||
quotient, remainder = divmod(page_count, cpu_count)
|
||
table_split_parts = []
|
||
text_split_parts = []
|
||
for i in range(cpu_count):
|
||
start_num = i * quotient
|
||
if i < cpu_count-1:
|
||
start_num = i * quotient
|
||
end_num = start_num+quotient
|
||
else:
|
||
end_num = page_count
|
||
table_split_parts.append(f'{start_num}-{end_num}')
|
||
text_split_parts.append(get_numbers_between([],start_num, end_num))
|
||
|
||
# 返回除法结果和余数
|
||
return {
|
||
'table_split_parts': table_split_parts,
|
||
'text_split_parts': text_split_parts
|
||
}
|
||
|
||
def create_text_outline(pdf_path, file_id):
|
||
# print('Running the script for [%s] with padding [%d]' % (pdf_path, page_number_padding))
|
||
# creating an object
|
||
with open(pdf_path, 'rb') as file:
|
||
file_info = {}
|
||
fileReader = PyPDF2.PdfReader(file)
|
||
page_count = len(fileReader.pages)
|
||
|
||
redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=6)
|
||
redis_client.set(f'page_count_{file_id}', page_count)
|
||
|
||
info = {
|
||
'page_count': page_count,
|
||
'all_pages': {},
|
||
'current_page_id': 1,
|
||
'padding': 0
|
||
}
|
||
|
||
print('Number of pages: %d' % info['page_count'])
|
||
|
||
pages = fileReader.trailer['/Root']['/Pages'].get_object()
|
||
recursive_numbering(pages, info)
|
||
#for page_num, page in enumerate(pages['/Kids']):
|
||
# page_obj = page.getObject()
|
||
# all_pages[id(page_obj)] = page_num + 1 # who starts counting from 0 anyways?
|
||
title_array = get_tree_pages(fileReader.outline, info, 0, [])
|
||
db_service.pdf_title_insert_mysql(file_id,title_array)
|
||
title_array = db_service.get_file_info_from_mysql(file_id)
|
||
|
||
parent_table_pages_local = {}
|
||
parent_table_pages_local[file_id] = []
|
||
print(f'{file_id}:{len(title_array)}')
|
||
for i in range(len(title_array)):
|
||
title_obj = title_array[i]
|
||
title = title_obj['title']
|
||
#print(f'标题分别是{title}')
|
||
if len(re.findall('母公司|现金流量表补充|重要会计政策|会计估计变更|公允价值的披露|合营安排或联营企业中的权益|与金融工具相关的风险|税项|主要控股参股公司|结构化主体情况|公司股份总数及股东结构变动及公司资产和负债结构的变动情况|所有权或使用权受到限制的资产|在建工程|固定资产|其他主体中的权益|分部信息|与金融工具相关的风险|其他关联交易|公司子公司重大事项', title)) >0 :
|
||
page_start = title_obj['page_num']
|
||
depth = title_obj['depth']
|
||
if i < len(title_array) - 1:
|
||
page_end = title_array[i+1]['page_num']
|
||
if title_array[i]['depth'] in [1,2]:
|
||
page_end = get_page_end(i+1, depth, title_array)
|
||
else:
|
||
page_end = page_count
|
||
print(f'目录识别时被丢弃的页码：{page_start}-{page_end}')
|
||
|
||
#当标题为母公司财务报表主要项目注释时，最后一页不过滤，避免核心roe指标无法召回
|
||
if len(re.findall('财务报表主要项目注释', title)) == 0:
|
||
page_end = page_end - 1
|
||
# print(title,page_start,page_end)
|
||
for i in range(page_start, page_end + 1):
|
||
# 将每个数字添加到列表中
|
||
parent_table_pages_local[file_id].append(i)
|
||
file_info['page_count'] = page_count
|
||
file_info['parent_table_pages'] = parent_table_pages_local[file_id]
|
||
file_info['split_parts'] = get_file_split(page_count)
|
||
|
||
redis_client.close()
|
||
|
||
return file_info
|
||
|
||
def create_text_outline_disclosure(pdf_path, file_id):
|
||
# print('Running the script for [%s] with padding [%d]' % (pdf_path, page_number_padding))
|
||
# creating an object
|
||
with open(pdf_path, 'rb') as file:
|
||
file_info = {}
|
||
fileReader = PyPDF2.PdfReader(file)
|
||
page_count = len(fileReader.pages)
|
||
|
||
redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=6)
|
||
redis_client.set(f'page_count_{file_id}', page_count)
|
||
|
||
info = {
|
||
'page_count': page_count,
|
||
'all_pages': {},
|
||
'current_page_id': 1,
|
||
'padding': 0
|
||
}
|
||
|
||
print('Number of pages: %d' % info['page_count'])
|
||
|
||
pages = fileReader.trailer['/Root']['/Pages'].get_object()
|
||
recursive_numbering(pages, info)
|
||
#for page_num, page in enumerate(pages['/Kids']):
|
||
# page_obj = page.getObject()
|
||
# all_pages[id(page_obj)] = page_num + 1 # who starts counting from 0 anyways?
|
||
title_array = get_tree_pages(fileReader.outline, info, 0, [])
|
||
#db_service.pdf_title_insert_mysql(file_id,title_array)
|
||
#title_array = db_service.get_file_info_from_mysql(file_id)
|
||
|
||
parent_table_pages_local = {}
|
||
parent_table_pages_local[file_id] = []
|
||
print(f'{file_id}:{len(title_array)}')
|
||
for i in range(len(title_array)):
|
||
title_obj = title_array[i]
|
||
title = title_obj['title']
|
||
#print(f'标题分别是{title}')
|
||
if len(re.findall('母公司|现金流量表补充|重要会计政策|会计估计变更|公允价值的披露|合营安排或联营企业中的权益|与金融工具相关的风险|税项|主要控股参股公司|结构化主体情况|公司股份总数及股东结构变动及公司资产和负债结构的变动情况|所有权或使用权受到限制的资产|在建工程|固定资产|其他主体中的权益|分部信息|与金融工具相关的风险|其他关联交易|公司子公司重大事项', title)) >0 :
|
||
page_start = title_obj['page_num']
|
||
depth = title_obj['depth']
|
||
if i < len(title_array) - 1:
|
||
page_end = title_array[i+1]['page_num']
|
||
if title_array[i]['depth'] in [1,2]:
|
||
page_end = get_page_end(i+1, depth, title_array)
|
||
else:
|
||
page_end = page_count
|
||
print(f'目录识别时被丢弃的页码：{page_start}-{page_end}')
|
||
|
||
#当标题为母公司财务报表主要项目注释时，最后一页不过滤，避免核心roe指标无法召回
|
||
if len(re.findall('财务报表主要项目注释', title)) == 0:
|
||
page_end = page_end - 1
|
||
# print(title,page_start,page_end)
|
||
for i in range(page_start, page_end + 1):
|
||
# 将每个数字添加到列表中
|
||
parent_table_pages_local[file_id].append(i)
|
||
file_info['page_count'] = page_count
|
||
file_info['parent_table_pages'] = parent_table_pages_local[file_id]
|
||
file_info['split_parts'] = get_file_split(page_count)
|
||
|
||
redis_client.close()
|
||
|
||
return file_info
|
||
if __name__ == '__main__':
|
||
import time
|
||
path = "/Users/zhengfei/Desktop/cb/2023年报检测/安妮股份.pdf"
|
||
|
||
threading.Thread(target=create_text_outline, args=(path,'111')).start()
|
||
time.sleep(5)
|
||
threading.Thread(target=create_text_outline, args=(path,'222')).start()
|
||
|