Skip to content

rdm: added ts collection rules and model #546

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kpsherva merged 4 commits into from
Jul 21, 2026
Merged

rdm: added ts collection rules and model #546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
99a89ee
rdm: added ts collection rules and model
0einstein0 Jul 2, 2026
d3d7cc9
change(ts_dep): adapt rules to the content
kpsherva Jul 21, 2026
50830cc
change(load): fail if request data found when request not enabled
kpsherva Jul 21, 2026
ffa5f9e
chore(test): add local experiments vocab
kpsherva Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cds_migrator_kit/rdm/records/load/load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,13 @@ def _after_publish(self, identity, published_record, entry, version, uow):
self._after_publish_load_parent_access_grants(published_record, version, entry)
request_data = entry["record"].get("_request_data", {})

if request_data and not self.create_inclusion_request:
raise ManualImportRequired(message="Detected request data, enable the requests",
field="validation",
stage="load",
recid=entry["record"]["recid"],
priority="warning",
subfield=None,)
if self.create_inclusion_request and request_data:
self._after_publish_add_inclusion_request(
request_data, published_record, entry, uow
Expand Down
3 changes: 2 additions & 1 deletion cds_migrator_kit/rdm/records/transform/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,9 @@
"desy",
"medline",
"aip",
"jacow"
]
KEYWORD_SCHEMES_TO_DROP = ["proquest", "disxa"]
KEYWORD_SCHEMES_TO_DROP = ["proquest", "disxa", "inspeq"]

ALLOWED_THESIS_COLLECTIONS = [
"thesis",
Expand Down
112 changes: 112 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/antares.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2025 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM CMS note model."""
from cds_migrator_kit.rdm.records.transform.models.research import (
research_model,
)
from cds_migrator_kit.transform.overdo import CdsOverdo


class ANTARESResearchModel(CdsOverdo):
"""Translation model for research."""

__query__ = '980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'

__ignore_keys__ = {
"923__r", # place of photo, present in posters but redundant
"0248_a",
"0248_p",
"0248_q",
"0247_9", # provenance of the DOI
"030__a", # TODO coden designation to drop?
"035__h", # oai identifiers in 1215391
"035__d", # oai identifiers in 1215391
"035__t", # oai identifiers in 1215391
"035__u", # oai identifiers in 1215391
"035__m", # oai identifiers in 1215391
"035__z", # oai identifiers in 1215391
"336__a", # redundant field
"500__9", # provenance of the note
"520__9", # provenance of the description
"520__h", # provenance of the description
"852__c", # holdings will be taken separately
"852__h",
"037__c", # arxiv subject
"100__m", # email of contributor
"245__9", # title provenance
"270__m", # document contact email
"300__a", # number of pages
"340__a", # TODO ignore material?
"540__3", # TODO still ignore the material of the license?
"540__9", # TODO still ignore the material of the license?
"542__3", # TODO still ignore the material of the license?
"595__i", # TODO ??
"695__e", # some inspire tag
"700__m", # email of contributor
"700__q", # TODO ignore? aliteration of the name, used for searching
"700__v", # TODO drop?
"773__x", # INSPIRE publication note
"773__t", # INSPIRE publication note
"773__0", # from SIS: can be ignored
"773__o", # from SIS: can be ignored
"773__z", # from SIS: can be ignored
"8564_8", # file id
"8564_s", # bibdoc id
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump, sometimes these are used for open access calculation
"8564_w", # system field
"8564_z", # file comment, migrated via file metadata
"913__y", # citation
"913__v", # citation
"913__t", # citation
"913__a", # citation
"913__c", # citation
"916__y", # year, redundant value
"937__c", # last modified by
"937__s", # last modification date
"960__a", # base number
"961__c", #
"961__h", #
"961__l", #
"961__x", #
"964__a", # TODO: ignore?
"981__a", # duplicate record id
"999C50",
"999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en
"999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en
"999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en
"999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en
"999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en
"999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en
"999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en
"999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en
"999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en
"999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en
}

_default_fields = {
"custom_fields": {},
}


antares_research_model = ANTARESResearchModel(
bases=(research_model,),
entry_point_group="cds_migrator_kit.migrator.rdm.rules.small_exp",
)
112 changes: 112 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/lcd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2025 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM CMS note model."""
from cds_migrator_kit.rdm.records.transform.models.research import (
research_model,
)
from cds_migrator_kit.transform.overdo import CdsOverdo


class ANTARESResearchModel(CdsOverdo):
"""Translation model for research."""

__query__ = '980__:LCD-Notes OR 980__:LCD-NOTES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'

__ignore_keys__ = {
"710__b",
"0248_a",
"0248_p",
"0248_q",
"0247_9", # provenance of the DOI
"030__a", # TODO coden designation to drop?
"035__h", # oai identifiers in 1215391
"035__d", # oai identifiers in 1215391
"035__t", # oai identifiers in 1215391
"035__u", # oai identifiers in 1215391
"035__m", # oai identifiers in 1215391
"035__z", # oai identifiers in 1215391
"336__a", # redundant field
"500__9", # provenance of the note
"520__9", # provenance of the description
"520__h", # provenance of the description
"852__c", # holdings will be taken separately
"852__h",
"037__c", # arxiv subject
"100__m", # email of contributor
"245__9", # title provenance
"270__m", # document contact email
"300__a", # number of pages
"340__a", # TODO ignore material?
"540__3", # TODO still ignore the material of the license?
"540__9", # TODO still ignore the material of the license?
"542__3", # TODO still ignore the material of the license?
"595__i", # TODO ??
"695__e", # some inspire tag
"700__m", # email of contributor
"700__q", # TODO ignore? aliteration of the name, used for searching
"700__v", # TODO drop?
"773__x", # INSPIRE publication note
"773__t", # INSPIRE publication note
"773__0", # from SIS: can be ignored
"773__o", # from SIS: can be ignored
"773__z", # from SIS: can be ignored
"8564_8", # file id
"8564_s", # bibdoc id
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump, sometimes these are used for open access calculation
"8564_w", # system field
"8564_z", # file comment, migrated via file metadata
"913__y", # citation
"913__v", # citation
"913__t", # citation
"913__a", # citation
"913__c", # citation
"916__y", # year, redundant value
"937__c", # last modified by
"937__s", # last modification date
"960__a", # base number
"961__c", #
"961__h", #
"961__l", #
"961__x", #
"964__a", # TODO: ignore?
"981__a", # duplicate record id
"999C50",
"999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en
"999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en
"999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en
"999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en
"999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en
"999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en
"999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en
"999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en
"999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en
"999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en
"999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en
}

_default_fields = {
"custom_fields": {},
}


lcd_research_model = ANTARESResearchModel(
bases=(research_model,),
entry_point_group="cds_migrator_kit.migrator.rdm.rules.small_exp",
)
2 changes: 1 addition & 1 deletion cds_migrator_kit/rdm/records/transform/models/lep.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class LEPResearchModel(ResearchModel):
"775__c", # year of volume
"596__a", # multivolume tag
"300__x", # drop the physical description
"8564_z", # file comment, migrated via file metadata
"0248_a",
"0248_p",
"0248_q",
Expand Down Expand Up @@ -64,7 +65,6 @@ class LEPResearchModel(ResearchModel):
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump, sometimes these are used for open access calculation
"8564_w", # system field
"8564_z", # file comment, migrated via file metadata
"913__y", # citation
"913__v", # citation
"913__t", # citation
Expand Down
3 changes: 2 additions & 1 deletion cds_migrator_kit/rdm/records/transform/models/research.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
class ResearchModel(CdsOverdo):
"""Translation model for research."""

__query__ = '980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'
__query__ = '693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'

__ignore_keys__ = {
"0248_a",
Expand Down Expand Up @@ -59,6 +59,7 @@ class ResearchModel(CdsOverdo):
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump, sometimes these are used for open access calculation
"8564_w", # system field
"8564_z", # file comment, migrated via file metadata
"913__y", # citation
"913__v", # citation
"913__t", # citation
Expand Down
71 changes: 71 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/technical_support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2025 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM CMS note model."""
from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
rdm_base_publication_model,
)
from cds_migrator_kit.transform.overdo import CdsOverdo


class TechnicalSupportModel(CdsOverdo):
"""Translation model for MoUs."""

__query__ = """980__:ARTICLE OR
980__:PREPRINT OR
980__:DEMSUPPLIERS OR
980__:INTNOTETSPUBL OR 690C_:INTNOTETSPUBL
AND 710__.5:TS OR 710__.5:ST OR 710__.5:MT OR 710__.5:EST OR 710__.5:SB
"""

__ignore_keys__ = {
"030__a",
"690c_a",
"595__i", # "INSPEC" id
"500__9", # source of notes
"035__z",
"0248_a",
"0248_p",
"0248_q",
"100__m", # email of contributor
"110__c", # Location of corporate author
"110__g", # Type of corporate author
"340__a", # Physical medium
"037__c", # arxiv subject
"300__a", # number of pages
"700__m", # email of contributor
"520__9",
"852__a", # Physical Location
"852__c", # holdings will be taken separately
"852__h",
"8564_8", # file id
"8564_s", # bibdoc id
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump
"913__t", # Citation field
"913__y", # Citation field
"913__v", # Citation field
"913__c", # Citation field
"916__y", # year, redundant value
"937__c", # last modified by
"937__s", # last modification date
"960__a", # base number
"961__c", # CDS modification tag # TODO
"961__h", # CDS modification tag # TODO
"961__l", # CDS modification tag # TODO
"961__x", # CDS modification tag # TODO
"981__a", # duplicate record id
"964__a", # number of physical copies
}

_default_fields = {"custom_fields": {},}


technical_support_model = TechnicalSupportModel(
bases=(rdm_base_publication_model,),
entry_point_group="cds_migrator_kit.migrator.rules.technical_support",
)
Loading
Loading