Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 55 additions & 3 deletions apps/common/handle/impl/common_handle.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,40 @@

from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
PILImage.MAX_IMAGE_PIXELS = None

# 全局图片解码像素上限(不再禁用 Pillow 的解压炸弹保护)。
# 超过该上限 Pillow 会告警,超过 2 倍会直接抛错,避免超大图片耗尽 worker 内存。
PILImage.MAX_IMAGE_PIXELS = 50_000_000

# 内嵌图片解码保护(防解压炸弹 / 超大尺寸图片导致共享 worker OOM)。
MAX_EMBED_IMAGE_PIXELS = 16_000_000
MAX_EMBED_IMAGE_AGGREGATE_PIXELS = 64_000_000

# XLSX(zip) 压缩包防护,限制成员数 / 解压后总大小 / 解压膨胀比。
MAX_EMBED_ARCHIVE_MEMBERS = 10_000
MAX_EMBED_ARCHIVE_UNCOMPRESSED_BYTES = 1024 * 1024 * 1024
MAX_EMBED_ARCHIVE_EXPANSION_RATIO = 50


def validate_xlsx_archive(archive: ZipFile):
infolist = archive.infolist()
if len(infolist) > MAX_EMBED_ARCHIVE_MEMBERS:
raise ValueError(f"XLSX archive member count exceeds limit: {len(infolist)}")
total_uncompressed = sum(info.file_size for info in infolist)
total_compressed = sum(info.compress_size for info in infolist)
if total_uncompressed > MAX_EMBED_ARCHIVE_UNCOMPRESSED_BYTES:
raise ValueError("XLSX archive uncompressed size exceeds limit")
if total_compressed > 0 and total_uncompressed > total_compressed * MAX_EMBED_ARCHIVE_EXPANSION_RATIO:
raise ValueError("XLSX archive expansion ratio exceeds limit")


def validate_xlsx_buffer(buffer):
archive = ZipFile(buffer)
try:
validate_xlsx_archive(archive)
finally:
archive.close()


def parse_element(element) -> {}:
data = {}
Expand Down Expand Up @@ -87,6 +120,7 @@ def handle_images(deps, archive: ZipFile) -> []:

def xlsx_embed_cells_images(buffer) -> {}:
archive = ZipFile(buffer)
validate_xlsx_archive(archive)
# 解析cellImage.xml文件
deps = get_dependents(archive, get_rels_path("xl/cellimages.xml"))
image_rel = handle_images(deps=deps, archive=archive)
Expand All @@ -104,6 +138,7 @@ def xlsx_embed_cells_images(buffer) -> {}:
for cnv, embed in cell_images_xml.items():
cell_images_xml[cnv] = cell_images_rel.get(embed)
result = {}
total_pixels = 0
for key, img in cell_images_xml.items():
all_cells = [
cell
Expand All @@ -123,8 +158,25 @@ def xlsx_embed_cells_images(buffer) -> {}:
image_excel_id = image_excel_id_list[-1]
f = archive.open(img.target)
img_byte = io.BytesIO()
im = PILImage.open(f).convert('RGB')
im.save(img_byte, format='JPEG')
try:
with PILImage.open(f) as im:
width, height = im.size
pixels = width * height
if pixels > MAX_EMBED_IMAGE_PIXELS:
maxkb_logger.warning(
f"Skip oversized embedded image {img.path}: {width}x{height} pixels exceeds limit"
)
continue
total_pixels += pixels
if total_pixels > MAX_EMBED_IMAGE_AGGREGATE_PIXELS:
maxkb_logger.warning(
f"Skip embedded images in archive: aggregate pixels exceed limit"
)
break
im.convert('RGB').save(img_byte, format='JPEG')
except Exception as e:
maxkb_logger.error(f"Error decoding image {img.target}: {e}, {traceback.format_exc()}")
continue
image = File(id=uuid.uuid7(), file_name=img.path, meta={'debug': False, 'content': img_byte.getvalue()})
result['=' + image_excel_id] = image
archive.close()
Expand Down
3 changes: 2 additions & 1 deletion apps/common/handle/impl/qa/xlsx_parse_qa_handle.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import openpyxl

from common.handle.base_parse_qa_handle import BaseParseQAHandle, get_title_row_index_dict, get_row_value
from common.handle.impl.common_handle import xlsx_embed_cells_images
from common.handle.impl.common_handle import xlsx_embed_cells_images, validate_xlsx_buffer
from common.utils.logger import maxkb_logger


Expand Down Expand Up @@ -56,6 +56,7 @@ def support(self, file, get_buffer):
def handle(self, file, get_buffer, save_image):
buffer = get_buffer(file)
try:
validate_xlsx_buffer(io.BytesIO(buffer))
workbook = openpyxl.load_workbook(io.BytesIO(buffer))
try:
image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer))
Expand Down
3 changes: 2 additions & 1 deletion apps/common/handle/impl/table/xlsx_parse_table_handle.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from openpyxl import load_workbook

from common.handle.base_parse_table_handle import BaseParseTableHandle
from common.handle.impl.common_handle import xlsx_embed_cells_images
from common.handle.impl.common_handle import xlsx_embed_cells_images, validate_xlsx_buffer
from common.handle.impl.xlsx_utils import iter_sheet_content_rows
from common.utils.logger import maxkb_logger

Expand Down Expand Up @@ -61,6 +61,7 @@ def fill_merged_cells(self, sheet, image_dict):
def handle(self, file, get_buffer, save_image):
buffer = get_buffer(file)
try:
validate_xlsx_buffer(io.BytesIO(buffer))
wb = load_workbook(io.BytesIO(buffer))
try:
image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer))
Expand Down
3 changes: 2 additions & 1 deletion apps/common/handle/impl/text/xlsx_split_handle.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from openpyxl import load_workbook

from common.handle.base_split_handle import BaseSplitHandle
from common.handle.impl.common_handle import xlsx_embed_cells_images
from common.handle.impl.common_handle import xlsx_embed_cells_images, validate_xlsx_buffer
from common.handle.impl.xlsx_utils import iter_sheet_content_rows
from common.utils.logger import maxkb_logger

Expand Down Expand Up @@ -109,6 +109,7 @@ def fill_merged_cells(self, sheet, image_dict):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
buffer = get_buffer(file)
try:
validate_xlsx_buffer(io.BytesIO(buffer))
if type(limit) is str:
limit = int(limit)
workbook = openpyxl.load_workbook(io.BytesIO(buffer))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
:data="formValue.treeData"
:multiple="formValue.multiple"
:render-after-expand="false"
filterable
style="width: 100%"
/>
</el-form-item>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
:multiple="cond._treeMultiple"
:render-after-expand="false"
clearable
filterable
:placeholder="$t('workflow.nodes.conditionNode.valueMessage')"
/>

Expand Down
34 changes: 18 additions & 16 deletions ui/src/components/select-knowledge-document/index.vue
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
node-key="id"
lazy
:load="loadTree"
filterable
:placeholder="$t('views.chatLog.selectKnowledgePlaceholder')"
@change="changeKnowledge"
>
Expand Down Expand Up @@ -60,16 +61,19 @@ import type { FormInstance, FormRules } from 'element-plus'
import { loadSharedApi } from '@/utils/dynamics-api/shared-api'
import useStore from '@/stores'
import { t } from '@/locales'
const props = withDefaults(defineProps<{
data?: any,
postKnowledgeHandler?: (knowledge_list:Array<any>) => Array<any>
apiType: 'systemShare' | 'workspace' | 'systemManage'
isApplication?: boolean,
workspaceId?: string,
}>(), {
postKnowledgeHandler: (k_l: Array<any>) => k_l,
data: () => null
})
const props = withDefaults(
defineProps<{
data?: any
postKnowledgeHandler?: (knowledge_list: Array<any>) => Array<any>
apiType: 'systemShare' | 'workspace' | 'systemManage'
isApplication?: boolean
workspaceId?: string
}>(),
{
postKnowledgeHandler: (k_l: Array<any>) => k_l,
data: () => null,
},
)

const { user } = useStore()

Expand Down Expand Up @@ -101,21 +105,19 @@ const defaultProps = {

const loadTree = async (node: any, resolve: any) => {
if (node.isLeaf) return resolve([])
const folder_id = node.level === 0
? (props.workspaceId || user.getWorkspaceId())
: node.data.id
const folder_id = node.level === 0 ? props.workspaceId || user.getWorkspaceId() : node.data.id
const obj =
props.apiType === 'systemManage'
props.apiType === 'systemManage'
? {
workspace_id: props.workspaceId,
folder_id: folder_id,
folder_id: folder_id,
}
: {
folder_id: folder_id,
}
await loadSharedApi({ type: 'knowledge', systemType: props.apiType })
.getKnowledgeList(obj, optionLoading)
.then((ok: any)=>ok.data)
.then((ok: any) => ok.data)
.then(props.postKnowledgeHandler)
.then((res: any) => {
resolve(res)
Expand Down
Loading