diff --git a/cds_migrator_kit/rdm/records/load/load.py b/cds_migrator_kit/rdm/records/load/load.py index 1f8b581c..fdb25568 100644 --- a/cds_migrator_kit/rdm/records/load/load.py +++ b/cds_migrator_kit/rdm/records/load/load.py @@ -585,6 +585,13 @@ def _after_publish(self, identity, published_record, entry, version, uow): self._after_publish_load_parent_access_grants(published_record, version, entry) request_data = entry["record"].get("_request_data", {}) + if request_data and not self.create_inclusion_request: + raise ManualImportRequired(message="Detected request data, enable the requests", + field="validation", + stage="load", + recid=entry["record"]["recid"], + priority="warning", + subfield=None,) if self.create_inclusion_request and request_data: self._after_publish_add_inclusion_request( request_data, published_record, entry, uow diff --git a/cds_migrator_kit/rdm/records/transform/config.py b/cds_migrator_kit/rdm/records/transform/config.py index 864eb2f6..aeca8b90 100644 --- a/cds_migrator_kit/rdm/records/transform/config.py +++ b/cds_migrator_kit/rdm/records/transform/config.py @@ -63,8 +63,9 @@ "desy", "medline", "aip", + "jacow" ] -KEYWORD_SCHEMES_TO_DROP = ["proquest", "disxa"] +KEYWORD_SCHEMES_TO_DROP = ["proquest", "disxa", "inspeq"] ALLOWED_THESIS_COLLECTIONS = [ "thesis", diff --git a/cds_migrator_kit/rdm/records/transform/models/antares.py b/cds_migrator_kit/rdm/records/transform/models/antares.py new file mode 100644 index 00000000..849bc173 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/antares.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM CMS note model.""" +from cds_migrator_kit.rdm.records.transform.models.research import ( + research_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class ANTARESResearchModel(CdsOverdo): + """Translation model for research.""" + + __query__ = '980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + + __ignore_keys__ = { + "923__r", # place of photo, present in posters but redundant + "0248_a", + "0248_p", + "0248_q", + "0247_9", # provenance of the DOI + "030__a", # TODO coden designation to drop? + "035__h", # oai identifiers in 1215391 + "035__d", # oai identifiers in 1215391 + "035__t", # oai identifiers in 1215391 + "035__u", # oai identifiers in 1215391 + "035__m", # oai identifiers in 1215391 + "035__z", # oai identifiers in 1215391 + "336__a", # redundant field + "500__9", # provenance of the note + "520__9", # provenance of the description + "520__h", # provenance of the description + "852__c", # holdings will be taken separately + "852__h", + "037__c", # arxiv subject + "100__m", # email of contributor + "245__9", # title provenance + "270__m", # document contact email + "300__a", # number of pages + "340__a", # TODO ignore material? + "540__3", # TODO still ignore the material of the license? + "540__9", # TODO still ignore the material of the license? + "542__3", # TODO still ignore the material of the license? + "595__i", # TODO ?? + "695__e", # some inspire tag + "700__m", # email of contributor + "700__q", # TODO ignore? aliteration of the name, used for searching + "700__v", # TODO drop? + "773__x", # INSPIRE publication note + "773__t", # INSPIRE publication note + "773__0", # from SIS: can be ignored + "773__o", # from SIS: can be ignored + "773__z", # from SIS: can be ignored + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump, sometimes these are used for open access calculation + "8564_w", # system field + "8564_z", # file comment, migrated via file metadata + "913__y", # citation + "913__v", # citation + "913__t", # citation + "913__a", # citation + "913__c", # citation + "916__y", # year, redundant value + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "961__c", # + "961__h", # + "961__l", # + "961__x", # + "964__a", # TODO: ignore? + "981__a", # duplicate record id + "999C50", + "999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en + "999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en + "999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en + "999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en + "999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en + "999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en + "999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en + "999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en + } + + _default_fields = { + "custom_fields": {}, + } + + +antares_research_model = ANTARESResearchModel( + bases=(research_model,), + entry_point_group="cds_migrator_kit.migrator.rdm.rules.small_exp", +) diff --git a/cds_migrator_kit/rdm/records/transform/models/lcd.py b/cds_migrator_kit/rdm/records/transform/models/lcd.py new file mode 100644 index 00000000..22564fec --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/lcd.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM CMS note model.""" +from cds_migrator_kit.rdm.records.transform.models.research import ( + research_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class ANTARESResearchModel(CdsOverdo): + """Translation model for research.""" + + __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + + __ignore_keys__ = { + "710__b", + "0248_a", + "0248_p", + "0248_q", + "0247_9", # provenance of the DOI + "030__a", # TODO coden designation to drop? + "035__h", # oai identifiers in 1215391 + "035__d", # oai identifiers in 1215391 + "035__t", # oai identifiers in 1215391 + "035__u", # oai identifiers in 1215391 + "035__m", # oai identifiers in 1215391 + "035__z", # oai identifiers in 1215391 + "336__a", # redundant field + "500__9", # provenance of the note + "520__9", # provenance of the description + "520__h", # provenance of the description + "852__c", # holdings will be taken separately + "852__h", + "037__c", # arxiv subject + "100__m", # email of contributor + "245__9", # title provenance + "270__m", # document contact email + "300__a", # number of pages + "340__a", # TODO ignore material? + "540__3", # TODO still ignore the material of the license? + "540__9", # TODO still ignore the material of the license? + "542__3", # TODO still ignore the material of the license? + "595__i", # TODO ?? + "695__e", # some inspire tag + "700__m", # email of contributor + "700__q", # TODO ignore? aliteration of the name, used for searching + "700__v", # TODO drop? + "773__x", # INSPIRE publication note + "773__t", # INSPIRE publication note + "773__0", # from SIS: can be ignored + "773__o", # from SIS: can be ignored + "773__z", # from SIS: can be ignored + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump, sometimes these are used for open access calculation + "8564_w", # system field + "8564_z", # file comment, migrated via file metadata + "913__y", # citation + "913__v", # citation + "913__t", # citation + "913__a", # citation + "913__c", # citation + "916__y", # year, redundant value + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "961__c", # + "961__h", # + "961__l", # + "961__x", # + "964__a", # TODO: ignore? + "981__a", # duplicate record id + "999C50", + "999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en + "999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en + "999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en + "999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en + "999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en + "999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en + "999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en + "999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en + } + + _default_fields = { + "custom_fields": {}, + } + + +lcd_research_model = ANTARESResearchModel( + bases=(research_model,), + entry_point_group="cds_migrator_kit.migrator.rdm.rules.small_exp", +) diff --git a/cds_migrator_kit/rdm/records/transform/models/lep.py b/cds_migrator_kit/rdm/records/transform/models/lep.py index ba7d7564..b81b438e 100644 --- a/cds_migrator_kit/rdm/records/transform/models/lep.py +++ b/cds_migrator_kit/rdm/records/transform/models/lep.py @@ -23,6 +23,7 @@ class LEPResearchModel(ResearchModel): "775__c", # year of volume "596__a", # multivolume tag "300__x", # drop the physical description + "8564_z", # file comment, migrated via file metadata "0248_a", "0248_p", "0248_q", @@ -64,7 +65,6 @@ class LEPResearchModel(ResearchModel): "8564_x", # icon thumbnails sizes "8564_y", # file description - done by files dump, sometimes these are used for open access calculation "8564_w", # system field - "8564_z", # file comment, migrated via file metadata "913__y", # citation "913__v", # citation "913__t", # citation diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index 821f5a35..5bef9aa4 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -15,7 +15,7 @@ class ResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = '693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693__:UA5 OR 693__:UA8 OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' __ignore_keys__ = { "0248_a", @@ -59,6 +59,7 @@ class ResearchModel(CdsOverdo): "8564_x", # icon thumbnails sizes "8564_y", # file description - done by files dump, sometimes these are used for open access calculation "8564_w", # system field + "8564_z", # file comment, migrated via file metadata "913__y", # citation "913__v", # citation "913__t", # citation diff --git a/cds_migrator_kit/rdm/records/transform/models/technical_support.py b/cds_migrator_kit/rdm/records/transform/models/technical_support.py new file mode 100644 index 00000000..71166d83 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/technical_support.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM CMS note model.""" +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class TechnicalSupportModel(CdsOverdo): + """Translation model for MoUs.""" + + __query__ = """980__:ARTICLE OR + 980__:PREPRINT OR + 980__:DEMSUPPLIERS OR + 980__:INTNOTETSPUBL OR 690C_:INTNOTETSPUBL + AND 710__.5:TS OR 710__.5:ST OR 710__.5:MT OR 710__.5:EST OR 710__.5:SB + """ + + __ignore_keys__ = { + "030__a", + "690c_a", + "595__i", # "INSPEC" id + "500__9", # source of notes + "035__z", + "0248_a", + "0248_p", + "0248_q", + "100__m", # email of contributor + "110__c", # Location of corporate author + "110__g", # Type of corporate author + "340__a", # Physical medium + "037__c", # arxiv subject + "300__a", # number of pages + "700__m", # email of contributor + "520__9", + "852__a", # Physical Location + "852__c", # holdings will be taken separately + "852__h", + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump + "913__t", # Citation field + "913__y", # Citation field + "913__v", # Citation field + "913__c", # Citation field + "916__y", # year, redundant value + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "961__c", # CDS modification tag # TODO + "961__h", # CDS modification tag # TODO + "961__l", # CDS modification tag # TODO + "961__x", # CDS modification tag # TODO + "981__a", # duplicate record id + "964__a", # number of physical copies + } + + _default_fields = {"custom_fields": {},} + + +technical_support_model = TechnicalSupportModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rules.technical_support", +) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index f0bb349c..210c4752 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -66,12 +66,21 @@ class VocabularyCache: """Vocabulary lookup cache loaded once from YAML files at startup.""" - def __init__(self, vocab_dir): - """Load all vocabularies from the given directory into memory.""" + def __init__(self, default_dir, override_dir=None): + """Load all vocabularies into memory. + + For each vocabulary file, ``override_dir`` (e.g. a test-local + directory overriding just a subset of files) is preferred when it + contains that file, falling back to ``default_dir`` otherwise. + """ self._cache = {} - vocab_dir = Path(vocab_dir) + default_dir = Path(default_dir) + override_dir = Path(override_dir) if override_dir else None for vocab_type, filename in _VOCAB_FILENAMES.items(): - self._cache[vocab_type] = self._load(vocab_dir / filename) + filepath = default_dir / filename + if override_dir and (override_dir / filename).exists(): + filepath = override_dir / filename + self._cache[vocab_type] = self._load(filepath) @staticmethod def _load(filepath): @@ -99,14 +108,11 @@ def get(self, term, vocab_type): def _get_vocabulary_cache(): global _vocabulary_cache if _vocabulary_cache is None: - vocab_dir = current_app.config.get("CDS_MIGRATOR_KIT_VOCABULARIES_DIR") - if vocab_dir is None: - import cds_rdm + import cds_rdm - vocab_dir = Path(cds_rdm.__file__).parent / "app_data" / "vocabularies" - else: - vocab_dir = Path(vocab_dir) - _vocabulary_cache = VocabularyCache(vocab_dir) + default_dir = Path(cds_rdm.__file__).parent / "app_data" / "vocabularies" + override_dir = current_app.config.get("CDS_MIGRATOR_KIT_VOCABULARIES_DIR") + _vocabulary_cache = VocabularyCache(default_dir, override_dir) return _vocabulary_cache @@ -562,11 +568,11 @@ def field_experiments(record_json, custom_fields_dict): subj = json_output["metadata"].get("subjects", []) subj.append({"subject": experiment}) json_output["metadata"]["subjects"] = subj - raise RecordFlaggedCuration( + raise UnexpectedValue( subfield="u", value=experiment, field="author", - message=f"Experiment {experiment} not found, added as a subject", + message=f"Experiment {experiment} not found", stage="vocabulary match", ) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 1629a4b1..6682cb8a 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -851,6 +851,10 @@ def related_identifiers_787(self, key, value): "relation_type": {"id": "references"}, "resource_type": {"id": "publication-article"}, }, + "paper": { + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-article"}, + }, "corresponding video": { "relation_type": {"id": "references"}, "resource_type": {"id": "audio"}, diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index fddbf611..b6df2ef0 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -124,7 +124,7 @@ def corpo_author(self, key, value): return author raise IgnoreKey("creators") - +1 @model.over("imprint_info", "(^250__)") @for_each_value @require(["a"]) @@ -265,6 +265,7 @@ def journal(self, key, value): _custom_fields = self.get("custom_fields", {}) journal_fields = _custom_fields.get("journal:journal", {}) year = StringValue(value.get("y", "")).parse() + conference_url = StringValue(value.get("u", "")).parse() if "z" in value: try: @@ -294,10 +295,12 @@ def journal(self, key, value): if conference_cnum or conference_acronym: session = StringValue(value.get("c", "")).parse() new_meeting = {} + identifiers = [] + if conference_url: + identifiers.append({"scheme": "URL", "identifier": conference_url}) if conference_cnum: - new_meeting["identifiers"] = [ - {"scheme": "inspire", "identifier": conference_cnum} - ] + identifiers.append({"scheme": "inspire", "identifier": conference_cnum}) + new_meeting["identifiers"] = identifiers if conference_acronym: new_meeting["acronym"] = conference_acronym if session: @@ -638,8 +641,13 @@ def resource_type(self, key, value): "slintnote", "indico", "re29_papers", + "privimxgam", ] + # handles occurrences where document type should have been an experiment + add_experiment_for = {"lhcbcerntalk": "LHCb", "lhcb_misc": "LHCb"} + + committees = { "scicommpubldrdc": "DRDC", "scicommpubleec": "EEC", @@ -661,10 +669,12 @@ def resource_type(self, key, value): "scicommpubltcc": "TCC", } - if (value_a and value_a.lower() in committees.keys()) or ( - value_b and value_b in committees - ): + value_a = value_a.lower() + value_b = value_b.lower() + if (value_a in committees.keys()) or ( + value_b in committees + ): custom_fields = self.get("custom_fields", {}) comm_cf = custom_fields.get("cern:committees", []) if value_a: @@ -673,11 +683,20 @@ def resource_type(self, key, value): comm_cf.append({"id": committees[value_b.lower()]}) self["custom_fields"]["cern:committees"] = comm_cf raise IgnoreKey("resource_type") - if (value_a and value_a.lower() in ignore_res_types) or ( - value_b and value_b in ignore_res_types + + if (value_a.lower() in ignore_res_types) or ( + value_b in ignore_res_types ): raise IgnoreKey("resource_type") + + if value_a in add_experiment_for.keys(): + custom_field = self.get("custom_fields", {}) + experiments = custom_field.get("cern:experiments", []) + experiments.append(add_experiment_for[value_a]) + self["custom_fields"]["cern:experiments"] = experiments + raise IgnoreKey("resource_type") + # first has highest priority priority = { v: i @@ -696,8 +715,8 @@ def resource_type(self, key, value): "intnotealephpriv", "intnoteeppubl", "intnotehsepubl", + "intnote", "note", - "lcd-notes", "software", ] ) @@ -725,7 +744,13 @@ def resource_type(self, key, value): "proceedings": {"id": "publication-conferenceproceeding"}, "article": {"id": "publication-article"}, "note": {"id": "publication-technicalnote"}, + "intnote": {"id": "publication-technicalnote"}, "lcd-notes": {"id": "publication-technicalnote"}, + "privantares": {"id": "publication-technicalnote"}, + "privantaresplot": {"id": "image-plot"}, + "internalplot": {"id": "image-plot"}, + "internalnote": {"id": "publication-technicalnote"}, + "intnoteharpcdppubl": {"id": "publication-technicalnote"}, "brochure": {"id": "publication-brochure"}, "itcerntalk": {"id": "presentation"}, "talk": {"id": "presentation"}, @@ -749,6 +774,7 @@ def resource_type(self, key, value): "conferencenote": {"id": "publication-conferencenote"}, "slide": {"id": "presentation"}, "faser_papers": {"id": "publication-article"}, + "demsuppliers": {"id": "other"}, } try: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py new file mode 100644 index 00000000..2f19ca44 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/small_experiments.py @@ -0,0 +1,39 @@ +from dateutil.parser import ParserError, parse +from dojson.errors import IgnoreKey + +from cds_migrator_kit.errors import UnexpectedValue + +from .base import normalize + +from cds_migrator_kit.rdm.records.transform.models.antares import antares_research_model as model + +@model.over("publication_date", "(^225__)", override_tag=True) +def imprint_info(self, key, value): + """Translates imprint - WARNING - also publisher and publication_date. + + In case of summer student notes this field contains only date + but it needs to be reimplemented for the base set of rules - + it will contain also imprint place + """ + _custom_fields = self.get("custom_fields", {}) + imprint = _custom_fields.get("imprint:imprint", {}) + + publication_date_str = value.get("c") + _publisher = value.get("b") + place = value.get("a") + if _publisher and not self.get("publisher"): + self["publisher"] = _publisher + if place: + imprint["place"] = place + self["custom_fields"]["imprint:imprint"] = imprint + if publication_date_str: + try: + publication_date = normalize(publication_date_str) + return publication_date + except (ParserError, TypeError) as e: + raise UnexpectedValue( + field=key, + value=value, + message=f"Can't parse provided publication date. Value: {publication_date_str}", + ) + raise IgnoreKey("publication_date") diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py new file mode 100644 index 00000000..683d7eb0 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/technical_support.py @@ -0,0 +1,29 @@ +import math + +from dateutil.parser import ParserError, parse +from dojson.errors import IgnoreKey +from dojson.utils import force_list +from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.transform.xml_processing.quality.decorators import for_each_value, \ + require +from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue +from .base import subjects as base_subjects +from ...config import IGNORED_THESIS_COLLECTIONS +from ...models.technical_support import technical_support_model as model +from .base import corporate_author as base_corporate_author +from .base import normalize + + +@model.over("administrative_unit", "^710__", override=True) +@for_each_value +def custom_fields(self, key, value): + """Translates administrative_unit.""" + unit = value.get("b") + if unit: + _custom_fields = self.get("custom_fields", {}) + _custom_fields["cern:administrative_unit"] = unit + self["custom_fields"] = _custom_fields + else: + base_corporate_author(self, key, value) + + raise IgnoreKey("administrative_unit") diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index b9d5b0cb..f10b9d13 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -266,6 +266,7 @@ records: - "2ca82423-148a-42b5-86ff-03336b961c8e" antares: data_dir: cds_migrator_kit/rdm/data/former_exp/antares + plots: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/antares transform: @@ -277,6 +278,7 @@ records: antares_restr: data_dir: cds_migrator_kit/rdm/data/former_exp/antares_restr restricted: "True" + plots: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/antares_restr transform: @@ -286,6 +288,7 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "e2d2ed70-8a3c-453b-aa5b-1fa985f0906b" dirac: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/dirac extract: dirpath: cds_migrator_kit/rdm/data/former_exp/dirac @@ -296,6 +299,7 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "6a289642-5378-4daf-87b5-bb58af00487a" harp_cdp: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/harp_cdp extract: dirpath: cds_migrator_kit/rdm/data/former_exp/harp_cdp @@ -306,6 +310,7 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2541c76f-6c9b-415b-9a47-e814ec01f576" imxgam: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/imXgam restricted: "True" extract: @@ -317,8 +322,10 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "88a5cdf4-974a-44d4-b145-d19ee6346bb8" lcd_restr: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr restricted: "True" + create_inclusion_request: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/lcd_restr transform: @@ -328,6 +335,8 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "adc02716-780b-4bba-8e89-6316c9a11cf0" lcd: + plots: true + create_inclusion_request: true data_dir: cds_migrator_kit/rdm/data/former_exp/lcd extract: dirpath: cds_migrator_kit/rdm/data/former_exp/lcd @@ -339,6 +348,7 @@ records: - "adc02716-780b-4bba-8e89-6316c9a11cf0" re29: data_dir: cds_migrator_kit/rdm/data/former_exp/re29 + plots: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/re29 transform: @@ -349,7 +359,9 @@ records: - "aff67e22-9027-4401-a16f-1aaac2cd2c27" re29_restr: data_dir: cds_migrator_kit/rdm/data/former_exp/re29_restr + plots: true restricted: "True" + create_inclusion_request: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/re29_restr transform: @@ -359,6 +371,7 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "aff67e22-9027-4401-a16f-1aaac2cd2c27" ua2: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/ua2 extract: dirpath: cds_migrator_kit/rdm/data/former_exp/ua2 @@ -370,6 +383,7 @@ records: - "4040d2c7-7648-4f1a-81fa-7112fec8d130" ua4: data_dir: cds_migrator_kit/rdm/data/former_exp/ua4 + plots: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/ua4 transform: @@ -380,6 +394,7 @@ records: - "1b9239a7-5938-44e3-8ea8-2c530627f659" ua5: data_dir: cds_migrator_kit/rdm/data/former_exp/ua5 + plots: true extract: dirpath: cds_migrator_kit/rdm/data/former_exp/ua5 transform: @@ -389,6 +404,7 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "7996b881-2420-4791-ac17-00cf3c1e7196" ua8: + plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/ua8 extract: dirpath: cds_migrator_kit/rdm/data/former_exp/ua8 @@ -410,9 +426,7 @@ records: - "" faser-drafts: data_dir: cds_migrator_kit/rdm/data/faser-drafts - tmp_dir: cds_migrator_kit/rdm/tmp/faser-drafts - log_dir: cds_migrator_kit/rdm/log/faser-drafts - restricted: "True" + restricted: "True" plots: true create_inclusion_request: true access_grants_view: @@ -426,8 +440,6 @@ records: - "33af9368-5bad-45cb-9360-8c9e5dfca09f" faser: data_dir: cds_migrator_kit/rdm/data/faser - tmp_dir: cds_migrator_kit/rdm/tmp/faser - log_dir: cds_migrator_kit/rdm/log/faser create_inclusion_request: true plots: true extract: @@ -438,11 +450,11 @@ records: communities_ids: - "33af9368-5bad-45cb-9360-8c9e5dfca09f" faser-ep: + plots: true data_dir: cds_migrator_kit/rdm/data/faser tmp_dir: cds_migrator_kit/rdm/tmp/faser log_dir: cds_migrator_kit/rdm/log/faser create_inclusion_request: true - plots: true extract: dirpath: cds_migrator_kit/rdm/data/faser-ep/dump/ transform: @@ -450,36 +462,6 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "e5de962b-5e2c-408f-be69-2823fda1f732" - lcd_restr: - data_dir: cds_migrator_kit/rdm/data/lep_exp/lcd - tmp_dir: cds_migrator_kit/rdm/tmp/lep_exp/lcd - log_dir: cds_migrator_kit/rdm/log/lep_exp/lcd - restricted: "True" - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lep_exp/lcd/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lep_exp/lcd/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/lep_exp/lcd/duplicated_pids.json - re29_restr: - data_dir: cds_migrator_kit/rdm/data/lep_exp/re29 - tmp_dir: cds_migrator_kit/rdm/tmp/lep_exp/re29 - log_dir: cds_migrator_kit/rdm/log/lep_exp/re29 - restricted: "True" - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lep_exp/re29/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lep_exp/re29/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/lep_exp/re29/duplicated_pids.json staff_association: data_dir: cds_migrator_kit/rdm/data/staff_association extract: @@ -499,3 +481,43 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "e849139c-dff5-45a7-90b1-90cdddc9c00e" + ts_dep: + plots: true + data_dir: cds_migrator_kit/rdm/data/dep_ts + extract: + dirpath: cds_migrator_kit/rdm/data/dep_ts/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/dep_ts/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "058c4659-2ad5-47c7-98ad-5e42333b09b3" + mt_dep: + plots: true + data_dir: cds_migrator_kit/rdm/data/dep_mt + extract: + dirpath: cds_migrator_kit/rdm/data/dep_mt/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/dep_mt/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "f508d592-580f-4d3e-8083-d8119cac1913" + sb_dep: + plots: true + data_dir: cds_migrator_kit/rdm/data/dep_sb + extract: + dirpath: cds_migrator_kit/rdm/data/dep_sb/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/dep_sb/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "87d73294-8f1f-4a63-a75e-21a87aa25980" + est_dep: + plots: true + data_dir: cds_migrator_kit/rdm/data/dep_est + extract: + dirpath: cds_migrator_kit/rdm/data/dep_est/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/dep_est/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "9a9c7474-18d2-41ee-b3ec-0708139dd0ee" diff --git a/setup.cfg b/setup.cfg index 42f92ef1..4e8d5cad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,8 @@ cds_migrator_kit.migrator.submitter.model = cds_migrator_kit.migrator.models = research = cds_migrator_kit.rdm.records.transform.models.research:research_model lep_research = cds_migrator_kit.rdm.records.transform.models.lep:lep_research_model + antares = cds_migrator_kit.rdm.records.transform.models.antares:antares_research_model + lcd = cds_migrator_kit.rdm.records.transform.models.lcd:lcd_research_model research_comm_model = cds_migrator_kit.rdm.records.transform.models.research_committee:research_comm_model fap = cds_migrator_kit.rdm.records.transform.models.fap:fap_model ssn = cds_migrator_kit.rdm.records.transform.models.summer_student_report:sspn_model @@ -90,6 +92,8 @@ cds_migrator_kit.migrator.models = bulletin_drafts = cds_migrator_kit.rdm.records.transform.models.bulletin_drafts:bulletin_drafts_model staff_association = cds_migrator_kit.rdm.records.transform.models.staff_association:staff_association_model faser_publication = cds_migrator_kit.rdm.records.transform.models.faser_publication:faser_publication_model + technical_support = cds_migrator_kit.rdm.records.transform.models.technical_support:technical_support_model + cds_migrator_kit.migrator.rules.base = base = cds_migrator_kit.transform.xml_processing.rules.base cds_migrator_kit.migrator.rules.submitter = @@ -110,6 +114,11 @@ cds_migrator_kit.migrator.rdm.rules.research = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base research = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research +cds_migrator_kit.migrator.rdm.rules.small_exp = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + research = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research + small_exp = cds_migrator_kit.rdm.records.transform.xml_processing.rules.small_experiments cds_migrator_kit.migrator.rules.thesis = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base @@ -137,11 +146,11 @@ cds_migrator_kit.migrator.rules.courier = ; base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base ; publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research ; beams = cds_migrator_kit.rdm.records.transform.xml_processing.rules.beams -cds_migrator_kit.migrator.rules.books = - base = cds_migrator_kit.transform.xml_processing.rules.base - base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base - publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research - books = cds_migrator_kit.rdm.records.transform.xml_processing.rules.books +;cds_migrator_kit.migrator.rules.books = +; base = cds_migrator_kit.transform.xml_processing.rules.base +; base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base +; publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research +; books = cds_migrator_kit.rdm.records.transform.xml_processing.rules.books cds_migrator_kit.migrator.rules.it = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base @@ -191,6 +200,11 @@ cds_migrator_kit.migrator.rules.faser_publication = base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research faser_publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.faser_publication +cds_migrator_kit.migrator.rules.technical_support = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research + technical_support = cds_migrator_kit.rdm.records.transform.xml_processing.rules.technical_support cds_migrator_kit.migrator.rules.people = people = cds_migrator_kit.rdm.users.transform.xml_processing.rules.people invenio_pidstore.minters = diff --git a/tests/cds-rdm/conftest.py b/tests/cds-rdm/conftest.py index 4ecca54f..501806fa 100644 --- a/tests/cds-rdm/conftest.py +++ b/tests/cds-rdm/conftest.py @@ -167,6 +167,11 @@ def app_config(app_config): app_config["CDS_MIGRATOR_KIT_ENV"] = "test" app_config["CDS_MIGRATOR_KIT_SITE_UI_URL"] = "https://localhost:5000" app_config["CDS_MIGRATOR_KIT_SITE_API_URL"] = "https://localhost:5000/api" + # Overrides just experiments.yaml (kept in sync with the `experiments_v` + # fixture below); other vocabulary files still fall back to cds-rdm's. + app_config["CDS_MIGRATOR_KIT_VOCABULARIES_DIR"] = os.path.join( + base_path, "data", "vocabularies" + ) return app_config @@ -857,6 +862,50 @@ def exp_type(app): @pytest.fixture(scope="module") def experiments_v(app, exp_type): """Experiment vocabulary record.""" + vocab = vocabulary_service.create( + system_identity, + { + "id": "RP", + "title": { + "en": "RP", + }, + "props": {"link": "http://bla.web.cern.ch/lhcb/"}, + "type": "experiments", + }, + ) + vocab = vocabulary_service.create( + system_identity, + { + "id": "COMPASS NA58", + "title": { + "en": "COMPASS NA58", + }, + "props": {"link": "http://bla.web.cern.ch/lhcb/"}, + "type": "experiments", + }, + ) + vocab = vocabulary_service.create( + system_identity, + { + "id": "I216", + "title": { + "en": "I216", + }, + "props": {"link": "http://bla.web.cern.ch/lhcb/"}, + "type": "experiments", + }, + ) + vocab = vocabulary_service.create( + system_identity, + { + "id": "ASACUSA AD-3", + "title": { + "en": "ASACUSA AD-3", + }, + "props": {"link": "http://bla.web.cern.ch/lhcb/"}, + "type": "experiments", + }, + ) vocab = vocabulary_service.create( system_identity, { diff --git a/tests/cds-rdm/data/vocabularies/experiments.yaml b/tests/cds-rdm/data/vocabularies/experiments.yaml new file mode 100644 index 00000000..9843e598 --- /dev/null +++ b/tests/cds-rdm/data/vocabularies/experiments.yaml @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +# Test-only "experiments" vocabulary consumed by +# cds_migrator_kit.rdm.records.transform.transform.VocabularyCache. +# +# Mirrors the `experiments_v` conftest.py fixture (which populates the +# vocabularies service/DB for other, service-backed test paths) so that the +# transform's file-based vocabulary cache and the DB fixture agree - both +# need updating when a record references a new experiment. +- id: RP + title: + en: RP +- id: COMPASS NA58 + title: + en: COMPASS NA58 +- id: I216 + title: + en: I216 +- id: ASACUSA AD-3 + title: + en: ASACUSA AD-3 +- id: LHCB + title: + en: LHCB +- id: CMS + title: + en: CMS +- id: ALICE + title: + en: ALICE +- id: ATLAS + title: + en: ATLAS diff --git a/tests/cds-rdm/test_contributors_quality.py b/tests/cds-rdm/test_contributors_quality.py new file mode 100644 index 00000000..01305cea --- /dev/null +++ b/tests/cds-rdm/test_contributors_quality.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for contributors.py quality module.""" + +import pytest + +from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.transform.xml_processing.quality.contributors import ( + extract_json_contributor_ids, +) + + +class TestExtractJsonContributorIdsOrcid: + """Test the ORCID handling of extract_json_contributor_ids.""" + + def test_valid_orcid_added(self): + """Test that a valid ORCID is added as an identifier.""" + ids = extract_json_contributor_ids({"k": "0000-0002-1825-0097"}) + assert ids == [{"identifier": "0000-0002-1825-0097", "scheme": "orcid"}] + + def test_orcid_prefix_stripped(self): + """Test that an 'ORCID:' prefix is stripped before validation.""" + ids = extract_json_contributor_ids({"k": "ORCID:0000-0002-1825-0097"}) + assert ids == [{"identifier": "0000-0002-1825-0097", "scheme": "orcid"}] + + def test_jacow_prefixed_value_ignored(self): + """Test that a 'jacow-' prefixed value is ignored, not treated as an ORCID.""" + ids = extract_json_contributor_ids({"k": "JACOW-1234567"}) + assert ids == [] + + def test_jacow_prefixed_value_ignored_lowercase(self): + """Test that the jacow check is case-insensitive.""" + ids = extract_json_contributor_ids({"k": "jacow-1234567"}) + assert ids == [] + + def test_jacow_prefixed_value_ignored_mixed_case(self): + """Test that the jacow check is case-insensitive for mixed case.""" + ids = extract_json_contributor_ids({"k": "Jacow-AbC123"}) + assert ids == [] + + def test_invalid_orcid_raises_error(self): + """Test that an invalid, non-jacow ORCID raises UnexpectedValue.""" + with pytest.raises(UnexpectedValue): + extract_json_contributor_ids({"k": "not-an-orcid"}) + + def test_no_orcid_subfield_no_identifier_added(self): + """Test that a missing ORCID subfield does not add an identifier or raise.""" + ids = extract_json_contributor_ids({}) + assert ids == [] + + def test_empty_orcid_subfield_no_identifier_added(self): + """Test that an empty ORCID subfield does not add an identifier or raise.""" + ids = extract_json_contributor_ids({"k": ""}) + assert ids == [] + + def test_custom_orcid_subfield(self): + """Test that a custom orcid_subfield is used instead of the default 'k'.""" + ids = extract_json_contributor_ids( + {"j": "0000-0002-1825-0097"}, orcid_subfield="j" + ) + assert ids == [{"identifier": "0000-0002-1825-0097", "scheme": "orcid"}] + + def test_custom_orcid_subfield_jacow_ignored(self): + """Test that the jacow check also applies with a custom orcid_subfield.""" + ids = extract_json_contributor_ids({"j": "JACOW-1234567"}, orcid_subfield="j") + assert ids == [] + + +class TestExtractJsonContributorIdsOtherSources: + """Test the non-ORCID identifier sources of extract_json_contributor_ids.""" + + def test_inspire_author_id_from_0_subfield(self): + """Test extraction of an INSPIRE author id from subfield '0'.""" + ids = extract_json_contributor_ids({"0": "AUTHOR|(INSPIRE)12345"}) + assert ids == [{"identifier": "12345", "scheme": "inspire_author"}] + + def test_cds_author_id_from_0_subfield(self): + """Test extraction of a CDS author id from subfield '0'.""" + ids = extract_json_contributor_ids({"0": "AUTHOR|(CDS)98765"}) + assert ids == [{"identifier": "98765", "scheme": "cds"}] + + def test_cern_author_id_from_0_subfield(self): + """Test extraction of a CERN (SzGeCERN) author id from subfield '0'.""" + ids = extract_json_contributor_ids({"0": "AUTHOR|(SzGeCERN)11111"}) + assert ids == [{"identifier": "11111", "scheme": "cern"}] + + def test_unmatched_0_subfield_ignored(self): + """Test that an unrecognized subfield '0' value is silently ignored.""" + ids = extract_json_contributor_ids({"0": "SOMETHING-ELSE"}) + assert ids == [] + + def test_inspire_id_from_i_subfield(self): + """Test extraction of an INSPIRE id from subfield 'i'.""" + ids = extract_json_contributor_ids({"i": "INSPIRE-98765"}) + assert ids == [{"identifier": "INSPIRE-98765", "scheme": "inspire_author"}] + + def test_i_subfield_without_inspire_prefix_ignored(self): + """Test that subfield 'i' without the INSPIRE- prefix is ignored.""" + ids = extract_json_contributor_ids({"i": "98765"}) + assert ids == [] + + def test_all_sources_combined(self): + """Test that identifiers from all sources are combined without duplicates.""" + ids = extract_json_contributor_ids( + { + "0": "AUTHOR|(INSPIRE)12345", + "k": "0000-0002-1825-0097", + "i": "INSPIRE-98765", + } + ) + assert {"identifier": "12345", "scheme": "inspire_author"} in ids + assert {"identifier": "0000-0002-1825-0097", "scheme": "orcid"} in ids + assert {"identifier": "INSPIRE-98765", "scheme": "inspire_author"} in ids + assert len(ids) == 3 + + def test_duplicate_0_subfield_ids_not_repeated(self): + """Test that repeated identical ids in subfield '0' are not duplicated.""" + ids = extract_json_contributor_ids( + {"0": ["AUTHOR|(CDS)98765", "AUTHOR|(CDS)98765"]} + ) + assert ids == [{"identifier": "98765", "scheme": "cds"}] diff --git a/tests/cds-rdm/test_full_migration.py b/tests/cds-rdm/test_full_migration.py index a444fcc9..014739f2 100644 --- a/tests/cds-rdm/test_full_migration.py +++ b/tests/cds-rdm/test_full_migration.py @@ -69,6 +69,8 @@ def suite_multi_field(record): assert dict_rec["files"]["total_bytes"] == 13264 assert "TN_T6SamplesGammaSpecBenchmark.pdf" in dict_rec["files"]["entries"] assert dict_rec["custom_fields"] == { + "cern:departments": [{"id": "HSE", "title": {"en": "HSE"}}], + "cern:experiments": [{"id": "RP", "title": {"en": "RP"}}], "cern:studies": ["Physics Beyond Colliders"], "cern:projects": ["Beam Dump Facility"], "imprint:imprint": { @@ -107,7 +109,6 @@ def suite_multi_field(record): {"subject": "FLUKA benchmark"}, {"subject": "ActiWiz benchmark"}, {"subject": "BDF"}, - {"subject": "RP"}, ] assert dict_rec["metadata"]["contributors"] == [ { @@ -269,6 +270,22 @@ def multiple_custom_fields(record): "cern:accelerators": [ {"id": "CERN AD", "title": {"en": "CERN AD"}}, ], + "cern:departments": [ + { + "id": "EP", + "title": { + "en": "EP", + }, + }, + ], + "cern:experiments": [ + { + "id": "ASACUSA AD-3", + "title": { + "en": "ASACUSA AD-3", + }, + }, + ], "imprint:imprint": {"place": "Geneva"}, } @@ -333,6 +350,22 @@ def irregular_exp_field(record): "cern:accelerators": [ {"id": "CERN SPS", "title": {"en": "CERN SPS"}}, ], + "cern:departments": [ + { + "id": "PH", + "title": { + "en": "PH", + }, + }, + ], + "cern:experiments": [ + { + "id": "COMPASS NA58", + "title": { + "en": "COMPASS NA58", + }, + }, + ], "imprint:imprint": {"place": "Geneva"}, } assert "subjects" in dict_rec["metadata"] @@ -346,7 +379,6 @@ def irregular_exp_field(record): {"subject": "DAQ"}, {"subject": "FPGA"}, {"subject": "GUI"}, - {"subject": "COMPASS NA58"}, ] diff --git a/tests/cds-rdm/test_it_override_delegation.py b/tests/cds-rdm/test_it_override_delegation.py new file mode 100644 index 00000000..1ea663c9 --- /dev/null +++ b/tests/cds-rdm/test_it_override_delegation.py @@ -0,0 +1,511 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for IT migration rules that override and delegate to base/publications rules.""" + +import pytest +from dojson.errors import IgnoreKey + +from cds_migrator_kit.errors import UnexpectedValue +from cds_migrator_kit.rdm.records.transform.xml_processing.rules.it import ( + additional_descriptions, + conference_title, + imprint_dates, + imprint_info, + meeting, + related_identifiers_and_imprint, + series, + subjects, +) + + +class TestSubjectsDelegation: + """Test subjects function delegation to base_subjects.""" + + def test_subjects_delegates_to_base_for_standard_subject(self): + """Test that subjects delegates to base_subjects for standard subjects.""" + record = {"subjects": []} + # This should delegate to base_subjects since it's not a special IT case + with pytest.raises(IgnoreKey): + subjects(record, "65017", {"a": "Physics", "2": "SzGeCERN"}) + # Should have the subject added by base_subjects + assert len(record["subjects"]) == 1 + assert record["subjects"][0]["subject"] == "Physics" + + def test_subjects_it_specific_talk_keyword(self): + """Test that IT-specific Talk keyword is handled before base delegation.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "6931_", {"a": "Talk"}) + assert {"subject": "Talk"} in record["subjects"] + + def test_subjects_it_specific_lecture_keyword(self): + """Test that IT-specific Lecture keyword is handled before base delegation.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "6931_", {"a": "Lecture"}) + assert {"subject": "Lecture"} in record["subjects"] + + def test_subjects_desy_ignored_no_delegation(self): + """Test that DESY subjects are ignored without delegation.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "694__", {"a": "Some subject", "9": "DESY"}) + # Should not delegate to base, subjects should remain empty + assert record["subjects"] == [] + + def test_subjects_jacow_handled_specially(self): + """Test that JACoW subjects are handled specially.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "695__", {"a": "Accelerators", "9": "JACoW"}) + assert {"subject": "Accelerators"} in record["subjects"] + assert {"subject": "JACoW"} in record["subjects"] + + def test_subjects_xx_ignored(self): + """Test that XX subject is ignored.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "6931_", {"a": "XX"}) + assert record["subjects"] == [] + + def test_subjects_freetext_keyword(self): + """Test freetext keywords are delegated to base.""" + record = {"subjects": []} + with pytest.raises(IgnoreKey): + subjects(record, "653__", {"a": "custom keyword"}) + assert {"subject": "custom keyword"} in record["subjects"] + + +class TestImprintDatesDelegation: + """Test imprint_dates function delegation to base_custom_fields_693.""" + + def test_imprint_dates_with_693_field_delegates_to_base(self): + """Test that 693__ field triggers delegation to base_custom_fields_693.""" + record = {} + # Field 693__ should trigger base_custom_fields_693 + with pytest.raises(IgnoreKey): + imprint_dates(record, "693__", {"e": "LHC", "c": "2021"}) + # Check that custom fields were populated by base function + assert "custom_fields" in record + assert "cern:experiments" in record["custom_fields"] + assert "LHC" in record["custom_fields"]["cern:experiments"] + + def test_imprint_dates_with_269_does_not_delegate_to_693(self): + """Test that 269__ field does not trigger 693 delegation.""" + record = {} + with pytest.raises(IgnoreKey): + imprint_dates(record, "269__", {"c": "2021"}) + # Should have publication_date but no experiments from 693 + assert record["publication_date"] == "2021" + assert "cern:experiments" not in record.get("custom_fields", {}) + + def test_imprint_dates_269_with_place(self): + """Test that 269__ field properly sets imprint place.""" + record = {} + with pytest.raises(IgnoreKey): + imprint_dates(record, "269__", {"a": "Geneva.", "c": "2021"}) + assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" + assert record["publication_date"] == "2021" + + def test_imprint_dates_269_with_publisher(self): + """Test that 269__ field sets publisher when not already set.""" + record = {} + with pytest.raises(IgnoreKey): + imprint_dates(record, "269__", {"b": "CERN", "c": "2021"}) + assert record["publisher"] == "CERN" + + def test_imprint_dates_933_field(self): + """Test that 933__ field is handled.""" + record = {} + with pytest.raises(IgnoreKey): + imprint_dates(record, "933__", {"c": "2022"}) + assert record["publication_date"] == "2022" + + +class TestConferenceTitleDelegation: + """Test conference_title function delegation to base_internal_notes.""" + + def test_conference_title_delegates_to_base_notes(self): + """Test that conference_title delegates to base_internal_notes for notes.""" + record = {} + # The 'a' subfield should be delegated to base_internal_notes + with pytest.raises(IgnoreKey): + conference_title( + record, "595__", {"d": "Conference Title", "a": "Some note"} + ) + # Check conference title was set + assert record["custom_fields"]["meeting:meeting"]["title"] == "Conference Title" + # Check that internal notes were set by base function + assert "internal_notes" in record + assert len(record["internal_notes"]) > 0 + + def test_conference_title_only_title_no_notes(self): + """Test conference title without notes field.""" + record = {} + with pytest.raises(IgnoreKey): + conference_title(record, "595__", {"d": "Physics Conference 2021"}) + assert ( + record["custom_fields"]["meeting:meeting"]["title"] + == "Physics Conference 2021" + ) + + def test_conference_title_empty_ignored(self): + """Test that empty conference title is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + conference_title(record, "595__", {}) + + +class TestImprintInfoDelegation: + """Test imprint_info function delegation to base_publication_imprint_info.""" + + def test_imprint_info_with_260_delegates_to_base(self): + """Test that 260__ field delegates to base_publication_imprint_info.""" + # Initialize custom_fields as base function expects it + record = {"custom_fields": {}} + # Note: IT function calls base but doesn't return its value + # This might be a bug, but we test the actual behavior + result = imprint_info( + record, "260__", {"c": "2021", "a": "Geneva", "b": "CERN"} + ) + # Check that imprint fields were set by base function + assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" + assert record["publisher"] == "CERN" + + def test_imprint_info_with_362_uses_regex(self): + """Test that 362__ field uses IT-specific regex parsing.""" + record = {} + result = imprint_info(record, "362__", {"a": "Published in 2021-05-15"}) + assert result == "2021-05-15" + + def test_imprint_info_362_no_date_ignored(self): + """Test that 362__ without date raises IgnoreKey.""" + record = {} + with pytest.raises(IgnoreKey): + imprint_info(record, "362__", {"a": "No date here"}) + + def test_imprint_info_362_invalid_date_raises_error(self): + """Test that 362__ with invalid date raises UnexpectedValue.""" + record = {} + # Date matches regex but fails to parse: 2021-99-99 + with pytest.raises(UnexpectedValue): + imprint_info(record, "362__", {"a": "2021-99-99"}) + + +class TestMeetingDelegation: + """Test meeting function delegation to base_journal.""" + + def test_meeting_delegates_to_base_journal(self): + """Test that meeting function delegates journal fields to base_journal.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "773__", + {"p": "Journal Title", "v": "10", "n": "2", "c": "123-145"}, + ) + # Check that journal fields were populated by base function + assert "custom_fields" in record + assert "journal:journal" in record["custom_fields"] + journal = record["custom_fields"]["journal:journal"] + assert journal["title"] == "Journal Title" + assert journal["volume"] == "10" + assert journal["issue"] == "2" + assert journal["pages"] == "123-145" + + def test_meeting_with_published_in_relation(self): + """Test that 'e' field creates ispublishedin relation.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "773__", {"e": "12345", "p": "Journal"}) + # Should have related identifier + assert "related_identifiers" in record + assert len(record["related_identifiers"]) == 1 + rel_id = record["related_identifiers"][0] + assert rel_id["identifier"] == "12345" + assert rel_id["scheme"] == "cds" + assert rel_id["relation_type"]["id"] == "ispublishedin" + + def test_meeting_combines_journal_and_relation(self): + """Test that both journal info and relation are handled.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "773__", + { + "e": "12345", + "p": "Physics Journal", + "v": "5", + "n": "3", + "c": "50-60", + }, + ) + # Check both journal and related identifiers + assert record["custom_fields"]["journal:journal"]["title"] == "Physics Journal" + assert len(record["related_identifiers"]) == 1 + + +class TestRelatedIdentifiersAndImprintDelegation: + """Test related_identifiers_and_imprint delegation to base_publications_related_identifiers.""" + + def test_related_identifiers_and_imprint_delegates_to_base(self): + """Test delegation to base_publications_related_identifiers.""" + record = {} + with pytest.raises(IgnoreKey): + related_identifiers_and_imprint(record, "962__", {"b": "123456"}) + # Check that related identifier was added by base function + assert "related_identifiers" in record + assert len(record["related_identifiers"]) > 0 + + def test_related_identifiers_and_imprint_sets_pages(self): + """Test that imprint pages are set from 'k' field.""" + record = {} + with pytest.raises(IgnoreKey): + related_identifiers_and_imprint( + record, "962__", {"k": "100-150", "b": "123"} + ) + # Check that pages were set + assert record["custom_fields"]["imprint:imprint"]["pages"] == "100-150" + + def test_related_identifiers_and_imprint_no_duplicates(self): + """Test that duplicate related identifiers are not added.""" + record = { + "related_identifiers": [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-conferenceproceeding"}, + } + ] + } + with pytest.raises(IgnoreKey): + related_identifiers_and_imprint(record, "962__", {"b": "123456"}) + # Should still have only one entry + assert len(record["related_identifiers"]) == 1 + + def test_related_identifiers_and_imprint_only_pages(self): + """Test with only pages field, no related identifiers.""" + record = {} + with pytest.raises(IgnoreKey): + related_identifiers_and_imprint(record, "962__", {"k": "200-250"}) + assert record["custom_fields"]["imprint:imprint"]["pages"] == "200-250" + + +class TestAdditionalDescriptionsDelegation: + """Test additional_descriptions delegation to base_additional_titles.""" + + def test_additional_descriptions_210_delegates_to_base(self): + """Test that 210__ field delegates to base_additional_titles.""" + record = {} + with pytest.raises(IgnoreKey): + additional_descriptions(record, "210__", {"a": "Abbreviation Text"}) + # The base_additional_titles should have been called + # Note: base_additional_titles actually adds to additional_titles, not descriptions + # This is based on the actual implementation in it.py:147 + + def test_additional_descriptions_500_other_type(self): + """Test that 500__ creates 'other' type description.""" + record = {} + result = additional_descriptions(record, "500__", {"a": "General description"}) + assert result[0]["description"] == "General description" + assert result[0]["type"]["id"] == "other" + + def test_additional_descriptions_935_technical_info(self): + """Test that 935__ creates 'technical-info' type description.""" + record = {} + result = additional_descriptions( + record, "935__", {"a": "Technical information"} + ) + assert result[0]["description"] == "Technical information" + assert result[0]["type"]["id"] == "technical-info" + + +class TestSeriesDelegation: + """Test series function delegation to urls.""" + + def test_series_delegates_to_urls(self): + """Test that series delegates URL handling to base urls function.""" + record = {"recid": "12345"} + with pytest.raises(IgnoreKey): + series( + record, + "85641_", + {"u": "https://example.com/resource", "3": "Series info"}, + ) + # Check that URL was added to related identifiers by urls function + assert "related_identifiers" in record + url_added = any( + rel["scheme"] == "url" and "example.com" in rel["identifier"] + for rel in record["related_identifiers"] + ) + assert url_added + + def test_series_with_description(self): + """Test that series description is added.""" + record = {"recid": "12345"} + with pytest.raises(IgnoreKey): + series( + record, + "85641_", + {"u": "https://example.com", "3": "Series description text"}, + ) + # Check description was added + assert "additional_descriptions" in record + assert len(record["additional_descriptions"]) == 1 + assert ( + record["additional_descriptions"][0]["description"] + == "Series description text" + ) + assert ( + record["additional_descriptions"][0]["type"]["id"] == "series-information" + ) + + def test_series_ignores_icon_urls(self): + """Test that icon URLs are ignored.""" + record = {"recid": "12345"} + with pytest.raises(IgnoreKey): + series(record, "85641_", {"x": "icon", "u": "https://example.com/icon.png"}) + # Should not have added anything + assert "related_identifiers" not in record + + def test_series_url_without_description(self): + """Test URL without description.""" + record = {"recid": "12345"} + with pytest.raises(IgnoreKey): + series(record, "85641_", {"u": "https://example.com/resource"}) + # URL should be added but no description + assert "related_identifiers" in record + assert "additional_descriptions" not in record + + def test_series_no_duplicate_urls(self): + """Test that duplicate URLs are not added.""" + record = { + "recid": "12345", + "related_identifiers": [ + { + "identifier": "http://example.com/resource", + "scheme": "url", + "relation_type": {"id": "references"}, + "resource_type": {"id": "other"}, + } + ], + } + with pytest.raises(IgnoreKey): + series(record, "85641_", {"u": "https://example.com/resource"}) + # Should still have only one entry + assert len(record["related_identifiers"]) == 1 + + +class TestOverrideBehavior: + """Test that override=True functions properly override base behavior.""" + + def test_resource_type_override_marker(self): + """Verify resource_type has override=True to replace base implementation.""" + # This is a sanity test to ensure the override decorator is present + # The actual behavior is tested in test_it_migration.py + from cds_migrator_kit.rdm.records.transform.xml_processing.rules.it import ( + resource_type, + ) + + # Just verify the function exists and is callable + assert callable(resource_type) + + def test_subjects_override_marker(self): + """Verify subjects has override=True.""" + # subjects should override base but still delegate for some cases + assert callable(subjects) + + def test_imprint_dates_override_marker(self): + """Verify imprint_dates has override=True.""" + assert callable(imprint_dates) + + def test_conference_title_override_marker(self): + """Verify conference_title has override=True.""" + assert callable(conference_title) + + def test_meeting_override_marker(self): + """Verify meeting has override=True.""" + assert callable(meeting) + + def test_related_identifiers_and_imprint_override_marker(self): + """Verify related_identifiers_and_imprint has override=True.""" + assert callable(related_identifiers_and_imprint) + + def test_imprint_info_override_marker(self): + """Verify imprint_info has override=True.""" + assert callable(imprint_info) + + +class TestCombinedBehavior: + """Test scenarios where IT and base logic work together.""" + + def test_subjects_it_and_base_both_contribute(self): + """Test that both IT-specific and base subjects can coexist.""" + record = {"subjects": []} + + # Add IT-specific subject + with pytest.raises(IgnoreKey): + subjects(record, "6931_", {"a": "Talk"}) + + # Add base subject (controlled subject with SzGeCERN scheme) + with pytest.raises(IgnoreKey): + subjects(record, "65017", {"a": "Computing", "2": "SzGeCERN"}) + + # Both should be present + # IT-specific subject has only "subject" field + assert {"subject": "Talk"} in record["subjects"] + # Base controlled subject has both "id" and "subject" fields + assert {"id": "Computing", "subject": "Computing"} in record["subjects"] + + def test_imprint_dates_both_693_and_269(self): + """Test handling both 693 custom fields and 269 imprint in same record.""" + record = {} + + # First add custom fields via 693 + with pytest.raises(IgnoreKey): + imprint_dates(record, "693__", {"e": "ATLAS", "c": "2020"}) + + # Then add imprint via 269 + with pytest.raises(IgnoreKey): + imprint_dates(record, "269__", {"a": "Geneva", "c": "2020"}) + + # Both should be present + assert "ATLAS" in record["custom_fields"]["cern:experiments"] + assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" + assert record["publication_date"] == "2020" + + def test_conference_title_and_notes_together(self): + """Test conference title and notes are both processed.""" + record = {} + with pytest.raises(IgnoreKey): + conference_title( + record, "595__", {"d": "CHEP 2024", "a": "Conference proceedings"} + ) + + assert record["custom_fields"]["meeting:meeting"]["title"] == "CHEP 2024" + # Note handling is delegated to base_internal_notes + assert "internal_notes" in record + + def test_series_url_and_description_together(self): + """Test that URL and description are both handled in series.""" + record = {"recid": "12345"} + with pytest.raises(IgnoreKey): + series( + record, + "85641_", + { + "u": "https://cds.cern.ch/collection/CERN-TALK", + "3": "CERN Talk Series", + }, + ) + + # Both URL and description should be added + assert len(record["related_identifiers"]) == 1 + assert len(record["additional_descriptions"]) == 1 diff --git a/tests/cds-rdm/test_load_reviewers.py b/tests/cds-rdm/test_load_reviewers.py new file mode 100644 index 00000000..7858bd5f --- /dev/null +++ b/tests/cds-rdm/test_load_reviewers.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for reviewer resolution in load.py::_after_publish_add_inclusion_request.""" + +import pytest +from invenio_accounts.testutils import create_test_user + +from cds_migrator_kit.errors import RecordFlaggedCuration +from cds_migrator_kit.rdm.records.load.load import ( + _is_email, + _parse_reviewer_name, + find_reviewer, +) + + +class TestIsEmail: + """Test the _is_email helper.""" + + def test_email_detected(self): + """Test that a string with '@' is detected as an email.""" + assert _is_email("john.doe@cern.ch") is True + + def test_name_not_detected_as_email(self): + """Test that a plain name is not detected as an email.""" + assert _is_email("Doe, John") is False + assert _is_email("John Doe") is False + + +class TestParseReviewerName: + """Test the _parse_reviewer_name helper.""" + + def test_family_comma_given(self): + """Test 'Family, Given' format.""" + assert _parse_reviewer_name("Doe, John") == ("Doe", "John") + + def test_given_family_no_comma(self): + """Test 'Given Family' format (no comma).""" + assert _parse_reviewer_name("John Doe") == ("Doe", "John") + + def test_multi_word_given_name(self): + """Test a multi-word given name without a comma.""" + assert _parse_reviewer_name("John Michael Doe") == ("Doe", "John Michael") + + def test_family_name_only(self): + """Test a single-word name with no given name available.""" + assert _parse_reviewer_name("Doe") == ("Doe", "") + + def test_strips_whitespace(self): + """Test that surrounding and inner whitespace is stripped.""" + assert _parse_reviewer_name(" Doe , John ") == ("Doe", "John") + + +class TestFindReviewer: + """Test find_reviewer() DB resolution (email or profile name match).""" + + def test_find_reviewer_by_email(self, app, db): + """Test that a reviewer given as an email is resolved by email.""" + user = create_test_user(email="jane.smith@cern.ch") + db.session.commit() + + found = find_reviewer("jane.smith@cern.ch") + assert found.id == user.id + + def test_find_reviewer_by_profile_name_family_given(self, app, db): + """Test that a 'Family, Given' reviewer is resolved via profile JSON.""" + user = create_test_user( + email="john.doe@cern.ch", + user_profile={"family_name": "Doe", "given_name": "John"}, + ) + db.session.commit() + + found = find_reviewer("Doe, John") + assert found.id == user.id + + def test_find_reviewer_by_profile_name_given_family(self, app, db): + """Test that a 'Given Family' reviewer (no comma) is resolved via profile JSON.""" + user = create_test_user( + email="mary.jones@cern.ch", + user_profile={"family_name": "Jones", "given_name": "Mary"}, + ) + db.session.commit() + + found = find_reviewer("Mary Jones") + assert found.id == user.id + + def test_find_reviewer_name_match_is_case_insensitive(self, app, db): + """Test that profile name matching ignores case.""" + user = create_test_user( + email="anna.lee@cern.ch", + user_profile={"family_name": "Lee", "given_name": "Anna"}, + ) + db.session.commit() + + found = find_reviewer("lee, ANNA") + assert found.id == user.id + + def test_find_reviewer_family_name_only(self, app, db): + """Test that a family-name-only reviewer resolves when unambiguous.""" + user = create_test_user( + email="solo@cern.ch", + user_profile={"family_name": "Solo", "given_name": "Han"}, + ) + db.session.commit() + + found = find_reviewer("Solo") + assert found.id == user.id + + def test_find_reviewer_by_email_not_found_raises(self, app, db): + """Test that an unmatched email raises RecordFlaggedCuration.""" + with pytest.raises(RecordFlaggedCuration): + find_reviewer("nobody@cern.ch") + + def test_find_reviewer_by_name_not_found_raises(self, app, db): + """Test that an unmatched name raises RecordFlaggedCuration.""" + with pytest.raises(RecordFlaggedCuration): + find_reviewer("Nobody, Here") + + def test_find_reviewer_name_wrong_given_name_raises(self, app, db): + """Test that a family-name match with a mismatched given name raises.""" + create_test_user( + email="doe2@cern.ch", + user_profile={"family_name": "Doe", "given_name": "John"}, + ) + db.session.commit() + + with pytest.raises(RecordFlaggedCuration): + find_reviewer("Doe, Someone Else") diff --git a/tests/cds-rdm/test_transform_metadata_title.py b/tests/cds-rdm/test_transform_metadata_title.py new file mode 100644 index 00000000..c8672f50 --- /dev/null +++ b/tests/cds-rdm/test_transform_metadata_title.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""Tests for the metadata.title conference-proceedings fallback in _metadata().""" + +import pytest + +from cds_migrator_kit.rdm.records.transform.transform import CDSToRDMRecordEntry + + +@pytest.fixture +def entry(): + """Transform entry instance (no DB/app context required).""" + return CDSToRDMRecordEntry() + + +def _dump(): + """Minimal record dump for _publication_date().""" + return {"files": []} + + +def _json_entry(**overrides): + """Minimal json_entry with a resource_type and a creation date set.""" + base = { + "recid": 123, + "resource_type": {"id": "publication-conferenceproceeding"}, + "status_week_date": "2020-01-01", + } + base.update(overrides) + return base + + +class TestMetadataTitleFromMeeting: + """Test the 111__a -> metadata.title fallback for conference proceedings.""" + + def test_title_falls_back_to_meeting_title_when_missing(self, entry): + """Test that a missing title is filled from the meeting:meeting title + when resource_type is publication-conferenceproceeding.""" + json_entry = _json_entry( + custom_fields={"meeting:meeting": [{"title": "Some Conference"}]} + ) + metadata = entry._metadata(json_entry, _dump()) + assert metadata["title"] == "Some Conference" + + def test_title_not_overridden_when_already_present(self, entry): + """Test that an existing title is not replaced by the meeting title.""" + json_entry = _json_entry( + title="Real Title", + custom_fields={"meeting:meeting": [{"title": "Some Conference"}]}, + ) + metadata = entry._metadata(json_entry, _dump()) + assert metadata["title"] == "Real Title" + + def test_title_not_filled_for_other_resource_types(self, entry): + """Test that the meeting-title fallback only applies to + publication-conferenceproceeding, not other resource types.""" + json_entry = _json_entry( + resource_type={"id": "publication-article"}, + custom_fields={"meeting:meeting": [{"title": "Some Conference"}]}, + ) + metadata = entry._metadata(json_entry, _dump()) + assert "title" not in metadata + + def test_title_missing_without_meeting_custom_field(self, entry): + """Test that title stays unset when there is no meeting:meeting entry + to fall back to, even for conference proceedings.""" + json_entry = _json_entry() + metadata = entry._metadata(json_entry, _dump()) + assert "title" not in metadata + + def test_title_uses_first_meeting_entry_with_a_title(self, entry): + """Test that the fallback skips meeting entries without a title and + uses the first one that has one.""" + json_entry = _json_entry( + custom_fields={ + "meeting:meeting": [ + {"place": "Geneva"}, + {"title": "Second Meeting Title"}, + ] + } + ) + metadata = entry._metadata(json_entry, _dump()) + assert metadata["title"] == "Second Meeting Title"