57 lines
2.1 KiB
Python
57 lines
2.1 KiB
Python
|
from docx import Document
|
|||
|
|
from pymilvus import MilvusClient
|
|||
|
|
import requests
|
|||
|
|
import json,time,os
|
|||
|
|
|
|||
|
|
directory_path = '/Users/zhengfei/Desktop/大模型/书籍/第二批'
|
|||
|
|
|
|||
|
|
|
|||
|
|
client = MilvusClient(
|
|||
|
|
uri= 'http://114.55.128.195:19530'
|
|||
|
|
)
|
|||
|
|
# 遍历目录
|
|||
|
|
for filename in os.listdir(directory_path):
|
|||
|
|
# 构建完整的文件路径
|
|||
|
|
try:
|
|||
|
|
file_path = os.path.join(directory_path, filename)
|
|||
|
|
|
|||
|
|
# 加载Word文档
|
|||
|
|
print(file_path)
|
|||
|
|
doc = Document(file_path)
|
|||
|
|
|
|||
|
|
text = ''
|
|||
|
|
# 读取文档中的所有段落
|
|||
|
|
i=0
|
|||
|
|
data = []
|
|||
|
|
# for para in doc.paragraphs:
|
|||
|
|
for num in range(200, len(doc.paragraphs)-200):
|
|||
|
|
# 将段落文本添加到当前段落片段中
|
|||
|
|
try:
|
|||
|
|
text += doc.paragraphs[num].text
|
|||
|
|
# 当当前段落片段长度超过500时，将其添加到数组中，并重置当前段落片段
|
|||
|
|
if len(text) > 500:
|
|||
|
|
i += 1
|
|||
|
|
response = requests.post("http://114.55.128.195:8001/get_embedding/", json={"text": [text]}, headers={"Content-Type": "application/json"})
|
|||
|
|
res_json = json.loads(response.text)
|
|||
|
|
if res_json["code"] == 200:
|
|||
|
|
vector = res_json["data"][0]
|
|||
|
|
|
|||
|
|
measure_data = {}
|
|||
|
|
measure_data['vector'] = vector
|
|||
|
|
measure_data['text'] = text
|
|||
|
|
measure_data['source'] = '/projects/ai_chat/knowledge_base/ydkf/content/骨盆和骶骼关节功能解剖 手法操作指南 详解局部解剖和功能 涵盖评估分析 运动 肌肉能量技术及替代_14533413.docx'
|
|||
|
|
data.append(measure_data)
|
|||
|
|
text = ''
|
|||
|
|
|
|||
|
|
if (i > 20 or num == len(doc.paragraphs)-200-1):
|
|||
|
|
res = client.insert(
|
|||
|
|
collection_name="ydkf",
|
|||
|
|
data=data
|
|||
|
|
)
|
|||
|
|
i = 0
|
|||
|
|
data = []
|
|||
|
|
except Exception as e:
|
|||
|
|
print(e)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(e)
|
|||
|
|
|