251 lines
9.6 KiB
Python
251 lines
9.6 KiB
Python
|
import camelot
|
|||
|
|
import time
|
|||
|
|
import re
|
|||
|
|
import numpy as np
|
|||
|
|
from pdfminer.high_level import extract_pages
|
|||
|
|
from pdfminer.layout import LTTextBoxHorizontal
|
|||
|
|
import pdfplumber
|
|||
|
|
import json
|
|||
|
|
import utils
|
|||
|
|
|
|||
|
|
def chunks(l, n):
|
|||
|
|
"""Yield successive n-sized chunks from l."""
|
|||
|
|
for i in range(0, len(l), n):
|
|||
|
|
yield l[i : i + n]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_tables(filepath, pages_num, chunk_num=50, export_path=".", params={}):
|
|||
|
|
"""
|
|||
|
|
Divide the extraction work into n chunks. At the end of every chunk,
|
|||
|
|
save data on disk and free RAM.
|
|||
|
|
|
|||
|
|
filepath : str
|
|||
|
|
Filepath or URL of the PDF file.
|
|||
|
|
pages : str, optional (default: '1')
|
|||
|
|
Comma-separated page numbers.
|
|||
|
|
Example: '1,3,4' or '1,4-end' or 'all'.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
# get list of pages from camelot.handlers.PDFHandler
|
|||
|
|
handler = camelot.handlers.PDFHandler(filepath)
|
|||
|
|
page_list = handler._get_pages(pages=pages_num)
|
|||
|
|
|
|||
|
|
# chunk pages list
|
|||
|
|
page_chunks = list(chunks(page_list, chunk_num))
|
|||
|
|
|
|||
|
|
# extraction and export
|
|||
|
|
for chunk in page_chunks:
|
|||
|
|
pages_string = str(chunk).replace("[", "").replace("]", "")
|
|||
|
|
tables = camelot.read_pdf(filepath, pages=pages_string, strip_text=' ,\n', copy_text=['h'])
|
|||
|
|
tables.export(f"{export_path}/tables.csv")
|
|||
|
|
|
|||
|
|
# 读取pdf中的表格,并将表格中指标和表头合并，eg: 2022年1季度营业收入为xxxxx
|
|||
|
|
def get_pdf_info(file_path, pages):
|
|||
|
|
tables = camelot.read_pdf(file_path, pages=pages, strip_text=' ,\n', copy_text=['h'])
|
|||
|
|
|
|||
|
|
pdf_info = []
|
|||
|
|
tables_range = {}
|
|||
|
|
|
|||
|
|
for table_num, t in enumerate(tables):
|
|||
|
|
|
|||
|
|
top = t._bbox[3]
|
|||
|
|
buttom = t._bbox[1]
|
|||
|
|
page_num = int(t.page)
|
|||
|
|
table_index = int(t.order)
|
|||
|
|
arr = np.array(t.data)
|
|||
|
|
if not tables_range.get(page_num):
|
|||
|
|
tables_range[page_num] = []
|
|||
|
|
|
|||
|
|
tables_range[page_num].append({
|
|||
|
|
'top' : top,
|
|||
|
|
'buttom' : buttom,
|
|||
|
|
'table_index' : table_index,
|
|||
|
|
'page_num' : page_num,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
pdf_info.append({
|
|||
|
|
'top' : top,
|
|||
|
|
'buttom' : buttom,
|
|||
|
|
'page_num' : page_num,
|
|||
|
|
'table_index' : table_index,
|
|||
|
|
"type" : "table",
|
|||
|
|
"data" : t.data,
|
|||
|
|
'sort_num' : page_num*1000 - top
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
for pagenum, page in enumerate(extract_pages(file_path)):
|
|||
|
|
page_elements = [(element.y1, element) for element in page._objs]
|
|||
|
|
# 查找组成页面的元素
|
|||
|
|
for i,component in enumerate(page_elements):
|
|||
|
|
|
|||
|
|
text_type = 'text'
|
|||
|
|
# 提取页面布局的元素
|
|||
|
|
element = component[1]
|
|||
|
|
# 检查该元素是否为文本元素
|
|||
|
|
if isinstance(element, LTTextBoxHorizontal):
|
|||
|
|
# 检查文本是否出现在表中
|
|||
|
|
line_text = element.get_text().replace('\n','')
|
|||
|
|
line_text = re.sub(r"\s", "", line_text)
|
|||
|
|
|
|||
|
|
element_top = element.bbox[3]
|
|||
|
|
element_buttom = element.bbox[1]
|
|||
|
|
|
|||
|
|
# 检查该文本是否出现在表中
|
|||
|
|
if tables_range.get(pagenum+1):
|
|||
|
|
for range in tables_range[pagenum+1]:
|
|||
|
|
# print(f"{range['top']}: {range['buttom']}: {range['table_index']}")
|
|||
|
|
if element_top < range['top'] and element_top > range['buttom']:
|
|||
|
|
pass
|
|||
|
|
else:
|
|||
|
|
if element_top - range['top'] < 100 and element_top - range['top'] > 5 and not text_in_table(element_top, tables_range, pagenum+1):
|
|||
|
|
if i == 0:
|
|||
|
|
text_type = get_text_type(line_text)
|
|||
|
|
if text_type == 'page_header':
|
|||
|
|
break
|
|||
|
|
if utils.check_table_title_black_list(line_text):
|
|||
|
|
print(line_text)
|
|||
|
|
|
|||
|
|
pdf_info.append({
|
|||
|
|
'top' : element_top,
|
|||
|
|
'buttom' : element_buttom,
|
|||
|
|
'page_num' : range['page_num'],
|
|||
|
|
'table_index' : range['table_index'],
|
|||
|
|
"type" : text_type,
|
|||
|
|
'content' : line_text,
|
|||
|
|
'sort_num' : range['page_num']*1000 - element_top
|
|||
|
|
})
|
|||
|
|
break
|
|||
|
|
#处理母公司表格标题在页面底部，完整表格在下一页
|
|||
|
|
if element_buttom < 150 and not text_in_table(element_top, tables_range, pagenum+1):
|
|||
|
|
text_type = get_text_type(line_text)
|
|||
|
|
|
|||
|
|
if text_type == 'page_footer':
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
pdf_info.append({
|
|||
|
|
'top' : element_top,
|
|||
|
|
'buttom' : element_buttom,
|
|||
|
|
'page_num' : pagenum+1,
|
|||
|
|
"type" : text_type,
|
|||
|
|
'content' : line_text,
|
|||
|
|
'sort_num' : (pagenum+1)*1000 - element_top
|
|||
|
|
})
|
|||
|
|
# print(f'{element_top}: {element_buttom}: {line_text}')
|
|||
|
|
sorted_pdf_info = sorted(pdf_info, key=lambda k: k['sort_num'])
|
|||
|
|
for info in sorted_pdf_info:
|
|||
|
|
print(info)
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def text_in_table(top, tables_range, page_num):
|
|||
|
|
if tables_range.get(page_num):
|
|||
|
|
for range in tables_range[page_num]:
|
|||
|
|
if top < range['top'] and top > range['buttom']:
|
|||
|
|
return True
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def get_text_type(text: str):
|
|||
|
|
first_re = '年度报告'
|
|||
|
|
page_number_pattern = re.compile(r'^\d+(/\d+)?$')
|
|||
|
|
|
|||
|
|
if re.search(first_re, text.strip()):
|
|||
|
|
return 'page_header'
|
|||
|
|
|
|||
|
|
if page_number_pattern.match(text.strip()):
|
|||
|
|
return 'page_footer'
|
|||
|
|
|
|||
|
|
return 'text'
|
|||
|
|
|
|||
|
|
def find_continuous_numbers(numbers):
|
|||
|
|
# 首先对数组进行排序
|
|||
|
|
numbers.sort()
|
|||
|
|
|
|||
|
|
# 初始化新数组和连续序列的开始索引
|
|||
|
|
new_numbers = []
|
|||
|
|
start_index = 0
|
|||
|
|
|
|||
|
|
# 遍历排序后的数组
|
|||
|
|
for i in range(1, len(numbers)):
|
|||
|
|
# 检查当前数字是否与前一个数字不连续
|
|||
|
|
if numbers[i] != numbers[i-1] + 1:
|
|||
|
|
# 如果当前数字与前一个数字不连续，处理连续序列
|
|||
|
|
if i - start_index > 1:
|
|||
|
|
# 如果连续序列长度大于1，将最小数和最大数用"-"连接
|
|||
|
|
new_numbers.append(f"{numbers[start_index]}-{numbers[i-1]}")
|
|||
|
|
else:
|
|||
|
|
# 如果连续序列长度为1，直接添加数字
|
|||
|
|
new_numbers.append(str(numbers[start_index]))
|
|||
|
|
if start_index == i - 1:
|
|||
|
|
new_numbers.append(str(numbers[i-1]))
|
|||
|
|
|
|||
|
|
# 更新连续序列的开始索引
|
|||
|
|
start_index = i
|
|||
|
|
|
|||
|
|
# 处理数组末尾的连续序列
|
|||
|
|
if len(numbers) - start_index > 1:
|
|||
|
|
new_numbers.append(f"{numbers[start_index]}-{numbers[-1]}")
|
|||
|
|
else:
|
|||
|
|
new_numbers.append(str(numbers[start_index]))
|
|||
|
|
if start_index < len(numbers) - 1:
|
|||
|
|
new_numbers.append(str(numbers[-1]))
|
|||
|
|
|
|||
|
|
return new_numbers
|
|||
|
|
|
|||
|
|
def merge_consecutive_arrays(file_path):
|
|||
|
|
merged_objects = []
|
|||
|
|
temp_array = {}
|
|||
|
|
|
|||
|
|
# 打开文件并读取每一行
|
|||
|
|
with open(file_path, 'r') as file:
|
|||
|
|
for line in file:
|
|||
|
|
# 去除行尾的换行符
|
|||
|
|
line = line.strip()
|
|||
|
|
# 尝试将行转换成JSON格式
|
|||
|
|
try:
|
|||
|
|
obj = eval(line)
|
|||
|
|
if obj['type'] == 'table':
|
|||
|
|
# 如果对象是数组，将其元素添加到临时列表中
|
|||
|
|
if not temp_array.get('page_num'):
|
|||
|
|
temp_array = obj
|
|||
|
|
else:
|
|||
|
|
temp_array['data'].extend(obj['data'])
|
|||
|
|
else:
|
|||
|
|
# 如果对象不是数组，检查临时列表是否为空
|
|||
|
|
if temp_array:
|
|||
|
|
# 将临时列表中的元素合并成一个数组，并添加到新的对象列表中
|
|||
|
|
merged_objects.append(temp_array)
|
|||
|
|
temp_array = {} # 重置临时列表
|
|||
|
|
except json.JSONDecodeError as e:
|
|||
|
|
print(f"Error decoding JSON line: {e}")
|
|||
|
|
|
|||
|
|
if temp_array:
|
|||
|
|
merged_objects.append(temp_array)
|
|||
|
|
|
|||
|
|
# 关闭文件
|
|||
|
|
file.close()
|
|||
|
|
|
|||
|
|
return merged_objects
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
# print(get_text_type('6/223 '.strip()))
|
|||
|
|
# start = time.time()
|
|||
|
|
get_pdf_info('/Users/zhengfei/Desktop/0609/zhangjun-430489-2023-nb-nb_2824bc6302e9442285aed64eed760d99/zhangjun-430489-2023-nb-nb_2824bc6302e9442285aed64eed760d99_1.pdf','all')
|
|||
|
|
# end = time.time()
|
|||
|
|
# print('Task %s runs %0.2f seconds.' % ('223', (end - start)))
|
|||
|
|
# 示例数组
|
|||
|
|
# numbers = [1, 2, 3, 5, 7, 9, 10, 12, 13, 14, 17, 18, 19, 20, 22, 23, 24, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39]
|
|||
|
|
# # 调用函数并打印结果
|
|||
|
|
# print(find_continuous_numbers(numbers))
|
|||
|
|
# 示例数组对象列表
|
|||
|
|
# 分别对两个表格进行列命名和索引指定等操作，最后将两个表格进行合并，执行代码如下：
|
|||
|
|
# df1 = tables[0].df
|
|||
|
|
# df2 = df1.rename(columns=df1.iloc[0]).drop(df1.index[0]) ##将第0行作为行索引
|
|||
|
|
# df3 = tables[1].df
|
|||
|
|
# df4 = df3.rename(columns=df3.iloc[0]).drop(df3.index[0])
|
|||
|
|
# df__2= df2.append(df4,ignore_index=True) ##将两个数据进行合并，ignore_index=True,表根据列名对齐合并，生成新的index
|
|||
|
|
# print(df__2)
|
|||
|
|
|
|||
|
|
# 调用函数并打印结果
|
|||
|
|
# print(merge_consecutive_arrays('/Users/zhengfei/work/zzb_data/tables.txt'))
|