From 3d6e51eaeeec39755fd83122e17bf5be41da18ad Mon Sep 17 00:00:00 2001 From: uname <2986773479@qq.com> Date: Thu, 11 Dec 2025 16:49:56 +0800 Subject: [PATCH 1/3] fix: fixed the issue where an error would be reported when only setting the proportioning quantity when creating a proportioning task --- runtime/datamate-python/app/module/ratio/service/ratio_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/datamate-python/app/module/ratio/service/ratio_task.py b/runtime/datamate-python/app/module/ratio/service/ratio_task.py index 943d23b00..73d08ef34 100644 --- a/runtime/datamate-python/app/module/ratio/service/ratio_task.py +++ b/runtime/datamate-python/app/module/ratio/service/ratio_task.py @@ -61,7 +61,7 @@ async def create_task( 'label': { "label":item.get("filter_conditions").label.label, "value":item.get("filter_conditions").label.value, - }, + } if item.get("filter_conditions").label else None, }) ) logger.info(f"Relation created: {relation.id}, {relation}, {item}, {config}") From 9379cffbb39e14057be9c8cab253004c9e5cbd11 Mon Sep 17 00:00:00 2001 From: uname <2986773479@qq.com> Date: Thu, 11 Dec 2025 17:03:10 +0800 Subject: [PATCH 2/3] fix: prevent adding the same file multiple times --- .../app/module/ratio/service/ratio_task.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/runtime/datamate-python/app/module/ratio/service/ratio_task.py b/runtime/datamate-python/app/module/ratio/service/ratio_task.py index 73d08ef34..680e02b12 100644 --- a/runtime/datamate-python/app/module/ratio/service/ratio_task.py +++ b/runtime/datamate-python/app/module/ratio/service/ratio_task.py @@ -147,6 +147,7 @@ async def handle_ratio_relations(relations: list[RatioRelation], session, target select(DatasetFiles.file_path).where(DatasetFiles.dataset_id == target_ds.id) ) existing_paths = set(p for p in existing_path_rows.scalars().all() if p) + source_paths = set() added_count = 0 added_size = 0 @@ -164,10 +165,13 @@ async def handle_ratio_relations(relations: list[RatioRelation], session, target chosen = random.sample(files, pick_n) if pick_n < len(files) else files # Copy into target dataset with de-dup by target path - for f in chosen: - await RatioTaskService.handle_selected_file(existing_paths, f, session, target_ds) + for file in chosen: + if file.file_path in source_paths: + continue + await RatioTaskService.handle_selected_file(existing_paths, file, session, target_ds) + source_paths.add(file.file_path) added_count += 1 - added_size += int(f.file_size or 0) + added_size += int(file.file_size or 0) # Periodically flush to avoid huge transactions await session.flush() From 9199c2de2cf223d34dbb0131a8db736ff2d27d86 Mon Sep 17 00:00:00 2001 From: uname <2986773479@qq.com> Date: Thu, 11 Dec 2025 17:26:07 +0800 Subject: [PATCH 3/3] fix: implement a more flexible matching strategy, allowing only the tag name to be configured for matching --- .../app/module/dataset/schema/dataset_file.py | 2 +- .../app/module/ratio/service/ratio_task.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/runtime/datamate-python/app/module/dataset/schema/dataset_file.py b/runtime/datamate-python/app/module/dataset/schema/dataset_file.py index f31bba84b..e387f9a0f 100644 --- a/runtime/datamate-python/app/module/dataset/schema/dataset_file.py +++ b/runtime/datamate-python/app/module/dataset/schema/dataset_file.py @@ -46,7 +46,7 @@ def get_tags(self) -> List[str]: tags.append(tag_values) # 如果 from_name 不为空，添加前缀 if self.from_name: - tags = [f"{self.from_name}@{tag}" for tag in tags] + tags = [{"label": self.from_name, "value": tag} for tag in tags] return tags diff --git a/runtime/datamate-python/app/module/ratio/service/ratio_task.py b/runtime/datamate-python/app/module/ratio/service/ratio_task.py index 680e02b12..be8d001db 100644 --- a/runtime/datamate-python/app/module/ratio/service/ratio_task.py +++ b/runtime/datamate-python/app/module/ratio/service/ratio_task.py @@ -290,8 +290,15 @@ def _filter_file(file: DatasetFiles, conditions: FilterCondition) -> bool: return False try: # tags could be a list of strings or list of objects with 'name' - tag_names = RatioTaskService.get_all_tags(tags) - return f"{conditions.label.label}@{conditions.label.value}" in tag_names + all_tags = RatioTaskService.get_all_tags(tags) + for tag in all_tags: + if conditions.label.label and tag.get("label") != conditions.label.label: + continue + if conditions.label.value is not None: + return True + if tag.get("value") == conditions.label.value: + return True + return False except Exception as e: logger.exception(f"Failed to get tags for {file}", e) return False @@ -299,9 +306,9 @@ def _filter_file(file: DatasetFiles, conditions: FilterCondition) -> bool: return True @staticmethod - def get_all_tags(tags) -> set[str]: + def get_all_tags(tags) -> list[dict]: """获取所有处理后的标签字符串列表""" - all_tags = set() + all_tags = list() if not tags: return all_tags @@ -318,5 +325,5 @@ def get_all_tags(tags) -> set[str]: for file_tag in file_tags: for tag_data in file_tag.get_tags(): - all_tags.add(tag_data) + all_tags.append(tag_data) return all_tags