diff --git a/.gitignore b/.gitignore index 197cee84..9f9f49a9 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ target/ **/migration/data/** **/migration/log/** **/weblecture_migration/data/** +snapshots/ **.DS_Store -**/cds_migrator_kit/rdm/data/ \ No newline at end of file +**/cds_migrator_kit/rdm/data/ diff --git a/cds_migrator_kit/rdm/records/transform/entities/record.py b/cds_migrator_kit/rdm/records/transform/entities/record.py index 599fd94f..4a574c91 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/record.py +++ b/cds_migrator_kit/rdm/records/transform/entities/record.py @@ -229,7 +229,9 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry): creation date) and no creation date, raise an exception. """ if not raw_dump_entry.get("files") and not ( - dojson_entry.get("status_week_date") or dojson_entry.get("publication_date") + dojson_entry.get("status_week_date") + or dojson_entry.get("publication_date") + or dojson_entry.get("preprint_date") ): raise ManualImportRequired( message="Record missing publication date", diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index 81f8c383..17189b0e 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -7,6 +7,8 @@ """``metadata`` field mappers for CDS to RDM record transformation.""" +import re + from dateutil.parser import parse from cds_migrator_kit.errors import MissingRequiredField, UnexpectedValue @@ -61,8 +63,50 @@ def map_value(self, ctx): return title +#: An EDTF interval ("2020/2021", "2020-01/2020-05"), as produced by +#: `xml_processing/rules/base.py:normalize` - it leaves intervals +#: untouched, so the "/" here is a range separator, not a date one. +_DATE_INTERVAL = re.compile( + r"^(?P\d{4}(?:[-/]\d{1,2}){0,2})/(?P\d{4}(?:[-/]\d{1,2}){0,2})$" +) + + +def _date_precision(date_str): + """Return how granular a normalized date string is (year=1, month=2, day=3). + + `normalize` keeps whichever separator it found, so "2021-05" and + "2021/05" are both possible - split on both. For an interval only the + start date is measured. + """ + if not date_str: + return 0 + date_str = date_str.strip() + interval = _DATE_INTERVAL.match(date_str) + if interval: + date_str = interval.group("start") + return min(len(re.split(r"[-/]", date_str)), 3) + + +def _is_more_accurate(candidate, current): + """Return True if `candidate` has finer granularity than `current`.""" + return _date_precision(candidate) > _date_precision(current) + + class PublicationDateMapper(FieldMapper): - """Maps publication_date, falling back to status week or file creation date.""" + """Maps publication_date, preferring 260 (article) or 269 (preprint). + + - resource_type == "publication-article": publication_date (260) + always wins; preprint_date (269), if present, becomes a secondary + "submitted"/"preprint" entry in `dates`. + - any other resource_type: preprint_date (269) wins when present, + unless publication_date (260) is also present and at least as + accurate (day > month > year) - in that case publication_date wins + instead. Whichever one loses, if present, becomes a secondary entry + in `dates` ("available"/"published" for publication_date, + "submitted"/"preprint" for preprint_date). + + Falls back to status week or file creation date when neither applies. + """ id = "publication_date" @@ -70,6 +114,43 @@ def map_value(self, ctx): """Return publication_date, requiring at least one date source.""" dojson_entry = ctx.dojson_entry pub_date = dojson_entry.get("publication_date") + # `preprint_date` is bookkeeping produced by the 269 rules (see + # xml_processing/rules/base.py) - drop it before it reaches the + # final record. + preprint_date = dojson_entry.pop("preprint_date", None) + resource_type = (dojson_entry.get("resource_type") or {}).get("id") + + if resource_type == "publication-article": + if preprint_date: + dojson_entry.setdefault("dates", []).append( + { + "date": preprint_date, + "type": {"id": "submitted"}, + "description": "preprint", + } + ) + elif preprint_date: + if pub_date and not _is_more_accurate(preprint_date, pub_date): + # publication_date is present and at least as accurate - + # keep it, preprint_date becomes the secondary entry. + dojson_entry.setdefault("dates", []).append( + { + "date": preprint_date, + "type": {"id": "submitted"}, + "description": "preprint", + } + ) + else: + if pub_date: + dojson_entry.setdefault("dates", []).append( + { + "date": pub_date, + "type": {"id": "available"}, + "description": "published", + } + ) + pub_date = preprint_date + created = dojson_entry.get("status_week_date") files = ctx.raw_dump_entry["files"] if not (pub_date or created or files): diff --git a/cds_migrator_kit/rdm/records/transform/models/research_committee.py b/cds_migrator_kit/rdm/records/transform/models/research_committee.py index 817e21f2..3135e64b 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/models/research_committee.py @@ -94,8 +94,24 @@ class ResearchCommitteeModel(CdsOverdo): "999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en } + # `resource_type` is seeded so a former-committee record whose document + # type none of the available signals can resolve (no + # -- report number, no document type spelled + # out in 250__/490__/245__, no usable 980__/697C_ tag - the committee + # 980__ tag itself only yields `cern:committees`) still migrates, + # instead of being rejected by ResourceTypeMapper's MissingRequiredField. + # This is scoped to this model on purpose: it covers the former + # committees only (see `__query__` - current committees like LHCC, and + # SPSC from 1990 onwards, are handled by other models, where a missing + # resource_type stays an error). + # Only a default - every rule that resolves a real resource_type + # overrides it, because they gate on `_resource_type_rank`, which a + # seeded default deliberately doesn't set (see + # research_committee.py:_set_resource_type_if_higher_priority and + # research.py:resource_type). _default_fields = { "custom_fields": {}, + "resource_type": {"id": "publication-other"}, } diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index fe10eb0a..20b85620 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -1019,13 +1019,12 @@ def sync(self, key, value): @model.over("publication_date", "(^260__)", override=True) -def imprint_info(self, key, value): +def publication_date(self, key, value): """Translates publication_date field.""" publication_date_str = value.get("c") if publication_date_str: try: publication_date = normalize(publication_date_str) - return publication_date except (ParserError, TypeError) as e: raise UnexpectedValue( @@ -1074,7 +1073,8 @@ def imprint_info(self, key, value): # TODO: should we still set as the main publication date if it's uncertain? publication_date = normalize(publication_date_str) - self["publication_date"] = publication_date + + self["preprint_date"] = publication_date except (ParserError, TypeError) as e: raise UnexpectedValue( field=key, diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py index 88080e49..096cbe60 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py @@ -356,7 +356,7 @@ def imprint_dates(self, key, value): "type": {"id": "created"}, } ) - self["publication_date"] = normalize(pub) + self["preprint_date"] = normalize(pub) except (ParserError, TypeError): raise UnexpectedValue( field=key, @@ -413,7 +413,7 @@ def translated_description(self, key, value): def imprint_info(self, key, value): """Translates publication_date field.""" if key.startswith("260"): - base_publication_imprint_info(self, key, value) + return base_publication_imprint_info(self, key, value) else: publication_date_str = value.get("a") if publication_date_str: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index a9616fe5..e1ac8b33 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -305,7 +305,7 @@ def journal(self, key, value): raise UnexpectedValue("Journal fields already set", field=key, value=value) journal_fields["pages"] = StringValue(value.get("c", "")).parse() - pub_date = self.get("publication_date") + pub_date = self.get("publication_date") or self.get("preprint_date") # if we only have 773 in the record and no other journal fields, # it is not journal date if not is_journal_year and "y" in value: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py index 8a555a56..8f9ebf0b 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py @@ -90,13 +90,37 @@ "M": {"id": "publication-memorandum"}, "R": {"id": "publication-report"}, }, + "PSCC": { + "M": {"id": "publication-memorandum"}, + }, } -# Extra subject tagged onto the record for specific (committee, type) pairs. -# "*" matches any committee. + +def _controlled_subject(term): + """Build a subjects entry referencing a controlled-vocabulary term. + + Matches the shape base.py's `is_controlled_subject` branch writes for a + 65017 MARC subject with a recognized scheme ($2/$9 in + CONTROLLED_SUBJECTS_SCHEMES): just `id` (no `scheme` - invenio-vocabularies + resolves the term, including its scheme, from the id) plus `subject` for + display. `term` must match an entry's `id` in + cds-rdm/site/cds_rdm/app_data/vocabularies/subjects_scicommittees.yaml. + """ + return {"id": term, "subject": term} + + +# Extra subject tagged onto the record for specific (committee, type) pairs - +# see `_committee_report_type`. "*" matches any committee. Most of these +# reference controlled terms from subjects_scicommittees.yaml (via +# `_controlled_subject`); "recommendation" isn't in that vocabulary, so it +# stays free text. _TYPE_SUBJECTS = { - ("SPSC", "R"): "recommendation", - ("*", "UG"): "collection:upgrade cost group", + ("SPSC", "R"): {"subject": "recommendation"}, + ("*", "UG"): _controlled_subject("Upgrade Cost Group"), + ("*", "TDR"): _controlled_subject("Technical Design Report"), + ("*", "STATUS-REPORT"): _controlled_subject("Status Report"), + ("*", "SR"): _controlled_subject("Status Report"), + ("*", "RD"): _controlled_subject("Status Report"), } @@ -119,7 +143,6 @@ "status report": {"id": "publication-report"}, "progress report": {"id": "publication-report"}, "addendum": {"id": "publication-other"}, - "decisions": {"id": "publication-meetingminutes"}, "commentaires": {"id": "publication-peerreview"}, "comments": {"id": "publication-peerreview"}, "proposition": {"id": "publication-proposal"}, @@ -130,6 +153,29 @@ } +# Free-text phrases that carry a variable segment in the middle, which the +# fixed phrases in `_SERIES_RESOURCE_TYPES` can't express. Matched by +# `_free_text_resource_type` (245__ title and 250__ edition), and tried +# before the fixed phrases: a pattern is more specific than the bare words +# ("report", "note", "minutes", ...) its match may happen to contain. +_FREE_TEXT_PATTERN_RESOURCE_TYPES = ( + # "Decisions of the 117th meeting of the Nuclear Physics Research + # Committee ...", "Decisions of the 22nd meeting of the ...". The + # meeting designation between "the" and "meeting" is optional and can + # be a numeral ("117th", "22nd") or spelled out ("third"), so allow a + # few arbitrary words there. Also covers the "Decision taken at the + # meeting" wording already listed in `_SERIES_RESOURCE_TYPES` (kept + # there for 490__ series, which is matched exactly - see + # `_apply_series_resource_type`). + ( + re.compile( + r"\bdecisions?\s+(?:of|taken\s+at)\s+the\s+(?:\S+\s+){0,3}meeting\b" + ), + {"id": "publication-meetingminutes"}, + ), +) + + # Priority tiers for the different ways a research-committee record's # resource_type can be derived, most to least reliable/specific: # -- report number (structured, unambiguous) > a @@ -224,9 +270,8 @@ def _apply_committee_report_number(self, identifier): _set_resource_type_if_higher_priority(self, resource_type, _RANK_REPORT_NUMBER) if subject: subjects = self.get("subjects", []) - new_subject = {"subject": subject} - if new_subject not in subjects: - subjects.append(new_subject) + if subject not in subjects: + subjects.append(subject) self["subjects"] = subjects @@ -298,14 +343,20 @@ def _apply_series_resource_type(self, value_a): def _free_text_resource_type(text): """Return a resource_type matched from free-text phrases, or None. - Return the resource_type for text containing one of - `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it (e.g. "Draft minutes - of the third meeting of the EEC ...", + Return the resource_type for text matching one of + `_FREE_TEXT_PATTERN_RESOURCE_TYPES`'s patterns (e.g. "Decisions of the + 117th meeting of the Nuclear Physics Research Committee ..."), or + containing one of `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it + (e.g. "Draft minutes of the third meeting of the EEC ...", https://cds.cern.ch/record/1015008, or "Addendum 1"), matched at a word boundary so e.g. "Reported" doesn't match "report" - or None if it doesn't. """ text_lower = text.strip().lower() + # Patterns first - see `_FREE_TEXT_PATTERN_RESOURCE_TYPES`. + for pattern, resource_type in _FREE_TEXT_PATTERN_RESOURCE_TYPES: + if pattern.search(text_lower): + return resource_type # Longest phrase first, so "letter of intent" is tried before a # hypothetical single-word phrase it contains. for phrase in sorted(_SERIES_RESOURCE_TYPES, key=len, reverse=True): diff --git a/tests/cds-rdm/conftest.py b/tests/cds-rdm/conftest.py index dd169fc3..8cf329e7 100644 --- a/tests/cds-rdm/conftest.py +++ b/tests/cds-rdm/conftest.py @@ -1015,6 +1015,26 @@ def date_type_v(app, date_type): }, ) + vocabulary_service.create( + system_identity, + { + "id": "submitted", + "props": {"datacite": "Submitted"}, + "title": {"en": "Submitted"}, + "type": "datetypes", + }, + ) + + vocabulary_service.create( + system_identity, + { + "id": "available", + "props": {"datacite": "Available"}, + "title": {"en": "Available"}, + "type": "datetypes", + }, + ) + return vocab diff --git a/tests/cds-rdm/test_base_rules.py b/tests/cds-rdm/test_base_rules.py index 91e14492..230dd154 100644 --- a/tests/cds-rdm/test_base_rules.py +++ b/tests/cds-rdm/test_base_rules.py @@ -13,6 +13,7 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( custom_fields_693, + imprint_info, normalize, note, recid, @@ -397,3 +398,41 @@ def test_note_whitespace_only_ignored(self): record = {} with pytest.raises(IgnoreKey): note(record, "595__", {"a": " "}) + + +class TestImprintInfo269: + """Test the 269 (preprint) imprint_info function from base.py. + + It never sets `publication_date` itself - it stashes the parsed date + under `record["preprint_date"]`, to be reconciled with a possible 260 + (article) date once resource_type is known, in PublicationDateMapper. + """ + + def test_imprint_info_269_full(self): + """Test full imprint info with place, publisher, and date.""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"a": "Geneva.", "b": "CERN", "c": "2021"}) + assert record["preprint_date"] == "2021" + assert record["publisher"] == "CERN" + assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" + + def test_imprint_info_269_publisher_not_overwritten(self): + """Test that existing publisher is not overwritten.""" + record = {"custom_fields": {}, "publisher": "Existing Publisher"} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"b": "CERN", "c": "2021"}) + assert record["publisher"] == "Existing Publisher" + + def test_imprint_info_269_no_date_ignored(self): + """Test that missing date raises IgnoreKey and sets no date.""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"a": "Geneva", "b": "CERN"}) + assert "preprint_date" not in record + + def test_imprint_info_269_invalid_date_raises_error(self): + """Test that invalid date raises error.""" + record = {"custom_fields": {}} + with pytest.raises(UnexpectedValue): + imprint_info(record, "269__", {"c": "not-a-valid-date"}) diff --git a/tests/cds-rdm/test_full_migration.py b/tests/cds-rdm/test_full_migration.py index 3df27f70..ee45fa23 100644 --- a/tests/cds-rdm/test_full_migration.py +++ b/tests/cds-rdm/test_full_migration.py @@ -95,7 +95,18 @@ def suite_multi_field(record): }, }, ] + # resource_type is not an article), and 269 (preprint_date) is more + # precise than 260 (publication_date), so it wins - 260 becomes the + # secondary dates entry, and preprint_date never leaks into the record. assert dict_rec["metadata"]["publication_date"] == "2018-08-02" + assert dict_rec["metadata"]["dates"] == [ + { + "date": "2018", + "type": {"id": "available", "title": {"en": "Available"}}, + "description": "published", + } + ] + assert "preprint_date" not in dict_rec["metadata"] assert ( dict_rec["metadata"]["title"] == "FLUKA and ActiWiz benchmark on BDF materials" ) diff --git a/tests/cds-rdm/test_it_migration.py b/tests/cds-rdm/test_it_migration.py index ec1b8509..54562b25 100644 --- a/tests/cds-rdm/test_it_migration.py +++ b/tests/cds-rdm/test_it_migration.py @@ -521,7 +521,7 @@ def test_imprint_dates_basic(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021"}) - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" def test_imprint_dates_with_place(self): """Test imprint place is added.""" @@ -549,7 +549,7 @@ def test_imprint_dates_with_question_mark(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021?"}) - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" assert len(record["dates"]) == 1 assert record["dates"][0]["type"]["id"] == "created" assert "indeterminate" in record["dates"][0]["description"] diff --git a/tests/cds-rdm/test_it_override_delegation.py b/tests/cds-rdm/test_it_override_delegation.py index 1ea663c9..a6dab172 100644 --- a/tests/cds-rdm/test_it_override_delegation.py +++ b/tests/cds-rdm/test_it_override_delegation.py @@ -100,8 +100,8 @@ def test_imprint_dates_with_269_does_not_delegate_to_693(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021"}) - # Should have publication_date but no experiments from 693 - assert record["publication_date"] == "2021" + # Should have the preprint date but no experiments from 693 + assert record["preprint_date"] == "2021" assert "cern:experiments" not in record.get("custom_fields", {}) def test_imprint_dates_269_with_place(self): @@ -110,7 +110,7 @@ def test_imprint_dates_269_with_place(self): with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"a": "Geneva.", "c": "2021"}) assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" def test_imprint_dates_269_with_publisher(self): """Test that 269__ field sets publisher when not already set.""" @@ -124,7 +124,7 @@ def test_imprint_dates_933_field(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "933__", {"c": "2022"}) - assert record["publication_date"] == "2022" + assert record["preprint_date"] == "2022" class TestConferenceTitleDelegation: @@ -168,11 +168,12 @@ def test_imprint_info_with_260_delegates_to_base(self): """Test that 260__ field delegates to base_publication_imprint_info.""" # Initialize custom_fields as base function expects it record = {"custom_fields": {}} - # Note: IT function calls base but doesn't return its value - # This might be a bug, but we test the actual behavior + # base_publication_imprint_info's return value must be propagated + # by the IT wrapper (previously it was silently discarded). result = imprint_info( record, "260__", {"c": "2021", "a": "Geneva", "b": "CERN"} ) + assert result == "2021" # Check that imprint fields were set by base function assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" assert record["publisher"] == "CERN" @@ -479,7 +480,7 @@ def test_imprint_dates_both_693_and_269(self): # Both should be present assert "ATLAS" in record["custom_fields"]["cern:experiments"] assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" - assert record["publication_date"] == "2020" + assert record["preprint_date"] == "2020" def test_conference_title_and_notes_together(self): """Test conference title and notes are both processed.""" diff --git a/tests/cds-rdm/test_json_translation_rules.py b/tests/cds-rdm/test_json_translation_rules.py index 38ce09f2..6f70691a 100644 --- a/tests/cds-rdm/test_json_translation_rules.py +++ b/tests/cds-rdm/test_json_translation_rules.py @@ -52,7 +52,8 @@ def test_migrate_sspn_record(datadir, base_app): ], "title": "Deep Learning Methods for Particle Reconstruction in the HGCal", "publisher": "CERN", - "publication_date": "2017-06-24", + "publication_date": "2017", + "preprint_date": "2017-06-24", "description": "The High Granularity end-cap Calorimeter is part of the phase-2 CMS upgrade (see Figure \\ref{fig:cms})\\cite{Contardo:2020886}. It's goal it to provide measurements of high resolution in time, space and energy. Given such measurements, the purpose of this work is to discuss the use of Deep Neural Networks for the task of particle and trajectory reconstruction, identification and energy estimation, during my participation in the CERN Summer Students Program.", "internal_notes": [], "subjects": [ @@ -154,7 +155,8 @@ def test_migrate_record_all_fields(datadir, base_app): } ], "publisher": "CERN", - "publication_date": "2018-08-02", + "publication_date": "2018", + "preprint_date": "2018-08-02", "description": "This note describes the FLUKA and Actiwiz benchmark with gamma spectroscopy results of various material samples, which were irradiated during the Beam Dump Facility (BDF) prototype target test in the North Area of the Super Proton Synchrotron (SPS) at CERN. The samples represent most of the materials that will be used in the construction of the BDF facility.", "internal_notes": [{"note": "Comments submitted after 31-08-2021 10:41"}], "subjects": [ diff --git a/tests/cds-rdm/test_research_committee_rules.py b/tests/cds-rdm/test_research_committee_rules.py index 14fae731..a06c4d56 100644 --- a/tests/cds-rdm/test_research_committee_rules.py +++ b/tests/cds-rdm/test_research_committee_rules.py @@ -13,6 +13,9 @@ from cds_migrator_kit.errors import MissingRequiredField from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) from cds_migrator_kit.rdm.records.transform.models.research_committee import ( research_comm_model, ) @@ -69,6 +72,9 @@ def test_real_record_291072_status_report_matches_report(self): record = {} self._call(record, "088__", {"a": "DRDC-Status-report-RD-30"}) assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [ + {"id": "Status Report", "subject": "Status Report"} + ] def test_spsc_m_is_memorandum(self): """M is SPSC-specific: publication-memorandum, not meeting minutes.""" @@ -102,21 +108,37 @@ def test_type_t_is_technical_note(self): self._call(record, "088__", {"a": "TCC-T-3"}) assert record["resource_type"] == {"id": "publication-technicalnote"} - def test_type_tdr_is_report(self): + def test_type_tdr_is_report_with_technical_design_report_subject(self): record = {} self._call(record, "088__", {"a": "TCC-TDR-3"}) assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [ + {"id": "Technical Design Report", "subject": "Technical Design Report"} + ] - def test_type_sr_is_report(self): + def test_type_sr_is_report_with_status_report_subject(self): record = {} self._call(record, "088__", {"a": "TCC-SR-3"}) assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [ + {"id": "Status Report", "subject": "Status Report"} + ] + + def test_type_rd_is_report_with_status_report_subject(self): + record = {} + self._call(record, "088__", {"a": "TCC-RD-3"}) + assert record["resource_type"] == {"id": "publication-report"} + assert record["subjects"] == [ + {"id": "Status Report", "subject": "Status Report"} + ] def test_type_ug_is_report_with_upgrade_cost_group_subject(self): record = {} self._call(record, "088__", {"a": "TCC-UG-3"}) assert record["resource_type"] == {"id": "publication-report"} - assert record["subjects"] == [{"subject": "collection:upgrade cost group"}] + assert record["subjects"] == [ + {"id": "Upgrade Cost Group", "subject": "Upgrade Cost Group"} + ] def test_spsc_r_is_report_with_recommendation_subject(self): """R is only defined for SPSC: publication-report + subject.""" @@ -429,6 +451,41 @@ def test_title_wins_over_conflicting_subtitle(self): ) assert record["resource_type"] == {"id": "publication-proposal"} + def test_decisions_of_the_nth_meeting_matches_meetingminutes(self): + """ "Decisions of the meeting ..." is meeting minutes, with the + meeting number written either as a numeral or spelled out.""" + for title_value in ( + "Decisions of the 117th meeting of the Nuclear Physics Research Committee", + "Decisions of the 22nd meeting of the Nuclear Physics Research Committee", + "Decisions of the third meeting of the NPRC", + "Decisions of the meeting of the NPRC", + "Decision taken at the meeting of the NPRC", + ): + record = {} + title(record, "245__", {"a": title_value}) + assert record["resource_type"] == {"id": "publication-meetingminutes"} + assert record["_resource_type_rank"] == _RANK_TITLE + + def test_decisions_pattern_wins_over_a_bare_phrase_in_the_title(self): + """The pattern is tried before the single-word phrases, so a + "report"/"note" mentioned further along the title can't win.""" + record = {} + title( + record, + "245__", + { + "a": "Decisions of the 117th meeting of the NPRC and status " + "report of the experiments" + }, + ) + assert record["resource_type"] == {"id": "publication-meetingminutes"} + + def test_decisions_without_a_meeting_not_matched(self): + """ "Decisions" on its own isn't a meeting-minutes marker.""" + record = {} + title(record, "245__", {"a": "Decisions of the Director-General"}) + assert "resource_type" not in record + def test_base_title_behaviour_preserved(self): """title is still populated exactly like the generic 245__ rule (base.title).""" @@ -524,6 +581,55 @@ def test_committee_report_type_not_clobbered_by_generic_980_type(self): assert out["resource_type"] == {"id": "publication-letter"} + def test_undeterminable_resource_type_defaults_to_other(self): + """A former-committee record carrying no resource_type signal at all + (the committee 980__ tag only yields `cern:committees`) falls back to + publication-other instead of failing on a missing resource_type.""" + blob = GroupableOrderedDict( + ( + ("088__", {"a": "CERN-SPSLC-94-025"}), + ("245__", {"a": "A study of fluoride crystals for LHC"}), + ("980__", {"a": "SCICOMMPUBLSPSLC"}), + ) + ) + out = research_comm_model.do(blob) + + assert out["resource_type"] == {"id": "publication-other"} + assert out["custom_fields"]["cern:committees"] == [{"id": "SPSLC"}] + # The default is not a decision - it must leave the rank unset, so + # any rule can still override it. + assert "_resource_type_rank" not in out + + def test_default_does_not_shadow_a_title_derived_type(self): + """The weakest real signal (245__ title) must still win over the + seeded default.""" + blob = GroupableOrderedDict( + ( + ( + "245__", + { + "a": "Decisions of the 117th meeting of the Nuclear " + "Physics Research Committee" + }, + ), + ("980__", {"a": "SCICOMMPUBLNPRC"}), + ) + ) + out = research_comm_model.do(blob) + + assert out["resource_type"] == {"id": "publication-meetingminutes"} + + def test_default_does_not_shadow_a_generic_980_type(self): + """A generic 980__ document-type tag must also win over the seeded + default, even though `resource_type` is already present when + research.py:resource_type runs.""" + blob = GroupableOrderedDict( + (("980__", ({"a": "ARTICLE"}, {"a": "SCICOMMPUBLSPSLC"})),) + ) + out = research_comm_model.do(blob) + + assert out["resource_type"] == {"id": "publication-article"} + class TestResourceTypeFinalizer: """`_resource_type` (inside RecordEntry._metadata) must strip @@ -577,15 +683,22 @@ def test_raises_when_no_resource_type_resolved_at_all(self, entry): entry._metadata(dojson_entry, self._raw_dump_entry()) -class TestResearchCommitteeModelDoesNotDefaultResourceType: - """ResearchCommitteeModel must not seed resource_type with a default - - a record where no 980__/697C_ occurrence or committee report number - resolves a real type should end up with no resource_type key at all, - so it's caught as a missing required field downstream.""" +class TestResearchCommitteeModelDefaultsResourceType: + """ResearchCommitteeModel seeds resource_type with publication-other, so + a former-committee record where no 980__/697C_ occurrence, committee + report number or free-text document type resolves a real type still + migrates rather than being rejected downstream as missing a required + field. The default is scoped to this model - see its `_default_fields`.""" - def test_committee_only_record_has_no_resource_type(self): + def test_committee_only_record_defaults_to_other(self): """A record with only a committee tag (no resolvable document type) - must not end up with resource_type=publication-other.""" + ends up with resource_type=publication-other.""" blob = GroupableOrderedDict((("980__", {"a": "SCICOMMPUBLSPSLC"}),)) out = research_comm_model.do(blob) - assert "resource_type" not in out + assert out["resource_type"] == {"id": "publication-other"} + + def test_other_models_still_have_no_default(self): + """The fallback must not leak into the base publication model the + other collections use - there, an unresolved resource_type stays + missing and is caught downstream.""" + assert "resource_type" not in (rdm_base_publication_model._default_fields or {})