Former comms restypes

.gitignore

@@ -70,6 +70,7 @@ target/
 **/migration/data/**
 **/migration/log/**
 **/weblecture_migration/data/**
+snapshots/
 
 **.DS_Store
-**/cds_migrator_kit/rdm/data/
+**/cds_migrator_kit/rdm/data/

cds_migrator_kit/rdm/records/transform/entities/record.py

@@ -229,7 +229,9 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry):
         creation date) and no creation date, raise an exception.
         """
         if not raw_dump_entry.get("files") and not (
-            dojson_entry.get("status_week_date") or dojson_entry.get("publication_date")
+            dojson_entry.get("status_week_date")
+            or dojson_entry.get("publication_date")
+            or dojson_entry.get("preprint_date")
         ):
             raise ManualImportRequired(
                 message="Record missing publication date",

cds_migrator_kit/rdm/records/transform/mappers/metadata.py

@@ -7,6 +7,8 @@
 
 """``metadata`` field mappers for CDS to RDM record transformation."""
 
+import re
+
 from dateutil.parser import parse
 
 from cds_migrator_kit.errors import MissingRequiredField, UnexpectedValue
@@ -61,15 +63,94 @@ def map_value(self, ctx):
         return title
 
 
+#: An EDTF interval ("2020/2021", "2020-01/2020-05"), as produced by
+#: `xml_processing/rules/base.py:normalize` - it leaves intervals
+#: untouched, so the "/" here is a range separator, not a date one.
+_DATE_INTERVAL = re.compile(
+    r"^(?P<start>\d{4}(?:[-/]\d{1,2}){0,2})/(?P<end>\d{4}(?:[-/]\d{1,2}){0,2})$"
+)
+
+
+def _date_precision(date_str):
+    """Return how granular a normalized date string is (year=1, month=2, day=3).
+
+    `normalize` keeps whichever separator it found, so "2021-05" and
+    "2021/05" are both possible - split on both. For an interval only the
+    start date is measured.
+    """
+    if not date_str:
+        return 0
+    date_str = date_str.strip()
+    interval = _DATE_INTERVAL.match(date_str)
+    if interval:
+        date_str = interval.group("start")
+    return min(len(re.split(r"[-/]", date_str)), 3)
+
+
+def _is_more_accurate(candidate, current):
+    """Return True if `candidate` has finer granularity than `current`."""
+    return _date_precision(candidate) > _date_precision(current)
+
+
 class PublicationDateMapper(FieldMapper):
-    """Maps publication_date, falling back to status week or file creation date."""
+    """Maps publication_date, preferring 260 (article) or 269 (preprint).
+
+    - resource_type == "publication-article": publication_date (260)
+      always wins; preprint_date (269), if present, becomes a secondary
+      "submitted"/"preprint" entry in `dates`.
+    - any other resource_type: preprint_date (269) wins when present,
+      unless publication_date (260) is also present and at least as
+      accurate (day > month > year) - in that case publication_date wins
+      instead. Whichever one loses, if present, becomes a secondary entry
+      in `dates` ("available"/"published" for publication_date,
+      "submitted"/"preprint" for preprint_date).
+
+    Falls back to status week or file creation date when neither applies.
+    """
 
     id = "publication_date"
 
     def map_value(self, ctx):
         """Return publication_date, requiring at least one date source."""
         dojson_entry = ctx.dojson_entry
         pub_date = dojson_entry.get("publication_date")
+        # `preprint_date` is bookkeeping produced by the 269 rules (see
+        # xml_processing/rules/base.py) - drop it before it reaches the
+        # final record.
+        preprint_date = dojson_entry.pop("preprint_date", None)
+        resource_type = (dojson_entry.get("resource_type") or {}).get("id")
+
+        if resource_type == "publication-article":
+            if preprint_date:
+                dojson_entry.setdefault("dates", []).append(
+                    {
+                        "date": preprint_date,
+                        "type": {"id": "submitted"},
+                        "description": "preprint",
+                    }
+                )
+        elif preprint_date:
+            if pub_date and not _is_more_accurate(preprint_date, pub_date):
+                # publication_date is present and at least as accurate -
+                # keep it, preprint_date becomes the secondary entry.
+                dojson_entry.setdefault("dates", []).append(
+                    {
+                        "date": preprint_date,
+                        "type": {"id": "submitted"},
+                        "description": "preprint",
+                    }
+                )
+            else:
+                if pub_date:
+                    dojson_entry.setdefault("dates", []).append(
+                        {
+                            "date": pub_date,
+                            "type": {"id": "available"},
+                            "description": "published",
+                        }
+                    )
+                pub_date = preprint_date
+
         created = dojson_entry.get("status_week_date")
         files = ctx.raw_dump_entry["files"]
         if not (pub_date or created or files):

cds_migrator_kit/rdm/records/transform/models/research_committee.py

@@ -94,8 +94,24 @@ class ResearchCommitteeModel(CdsOverdo):
         "999C6v",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
     }
 
+    # `resource_type` is seeded so a former-committee record whose document
+    # type none of the available signals can resolve (no
+    # <COMMITTEE>-<TYPE>-<NUMBER> report number, no document type spelled
+    # out in 250__/490__/245__, no usable 980__/697C_ tag - the committee
+    # 980__ tag itself only yields `cern:committees`) still migrates,
+    # instead of being rejected by ResourceTypeMapper's MissingRequiredField.
+    # This is scoped to this model on purpose: it covers the former
+    # committees only (see `__query__` - current committees like LHCC, and
+    # SPSC from 1990 onwards, are handled by other models, where a missing
+    # resource_type stays an error).
+    # Only a default - every rule that resolves a real resource_type
+    # overrides it, because they gate on `_resource_type_rank`, which a
+    # seeded default deliberately doesn't set (see
+    # research_committee.py:_set_resource_type_if_higher_priority and
+    # research.py:resource_type).
     _default_fields = {
         "custom_fields": {},
+        "resource_type": {"id": "publication-other"},
     }
 
 

cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py

@@ -1019,13 +1019,12 @@ def sync(self, key, value):
 
 
 @model.over("publication_date", "(^260__)", override=True)
-def imprint_info(self, key, value):
+def publication_date(self, key, value):
     """Translates publication_date field."""
     publication_date_str = value.get("c")
     if publication_date_str:
         try:
             publication_date = normalize(publication_date_str)
-
             return publication_date
         except (ParserError, TypeError) as e:
             raise UnexpectedValue(
@@ -1074,7 +1073,8 @@ def imprint_info(self, key, value):
 
             # TODO: should we still set as the main publication date if it's uncertain?
             publication_date = normalize(publication_date_str)
-            self["publication_date"] = publication_date
+
+            self["preprint_date"] = publication_date
         except (ParserError, TypeError) as e:
             raise UnexpectedValue(
                 field=key,

cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py

@@ -356,7 +356,7 @@ def imprint_dates(self, key, value):
                     "type": {"id": "created"},
                 }
             )
-        self["publication_date"] = normalize(pub)
+        self["preprint_date"] = normalize(pub)
     except (ParserError, TypeError):
         raise UnexpectedValue(
             field=key,
@@ -413,7 +413,7 @@ def translated_description(self, key, value):
 def imprint_info(self, key, value):
     """Translates publication_date field."""
     if key.startswith("260"):
-        base_publication_imprint_info(self, key, value)
+        return base_publication_imprint_info(self, key, value)
     else:
         publication_date_str = value.get("a")
         if publication_date_str:

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py

@@ -305,7 +305,7 @@ def journal(self, key, value):
             raise UnexpectedValue("Journal fields already set", field=key, value=value)
         journal_fields["pages"] = StringValue(value.get("c", "")).parse()
 
-    pub_date = self.get("publication_date")
+    pub_date = self.get("publication_date") or self.get("preprint_date")
     # if we only have 773 in the record and no other journal fields,
     # it is not journal date
     if not is_journal_year and "y" in value:

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research_committee.py

@@ -90,13 +90,37 @@
         "M": {"id": "publication-memorandum"},
         "R": {"id": "publication-report"},
     },
+    "PSCC": {
+        "M": {"id": "publication-memorandum"},
+    },
 }
 
-# Extra subject tagged onto the record for specific (committee, type) pairs.
-# "*" matches any committee.
+
+def _controlled_subject(term):
+    """Build a subjects entry referencing a controlled-vocabulary term.
+
+    Matches the shape base.py's `is_controlled_subject` branch writes for a
+    65017 MARC subject with a recognized scheme ($2/$9 in
+    CONTROLLED_SUBJECTS_SCHEMES): just `id` (no `scheme` - invenio-vocabularies
+    resolves the term, including its scheme, from the id) plus `subject` for
+    display. `term` must match an entry's `id` in
+    cds-rdm/site/cds_rdm/app_data/vocabularies/subjects_scicommittees.yaml.
+    """
+    return {"id": term, "subject": term}
+
+
+# Extra subject tagged onto the record for specific (committee, type) pairs -
+# see `_committee_report_type`. "*" matches any committee. Most of these
+# reference controlled terms from subjects_scicommittees.yaml (via
+# `_controlled_subject`); "recommendation" isn't in that vocabulary, so it
+# stays free text.
 _TYPE_SUBJECTS = {
-    ("SPSC", "R"): "recommendation",
-    ("*", "UG"): "collection:upgrade cost group",
+    ("SPSC", "R"): {"subject": "recommendation"},
+    ("*", "UG"): _controlled_subject("Upgrade Cost Group"),
+    ("*", "TDR"): _controlled_subject("Technical Design Report"),
+    ("*", "STATUS-REPORT"): _controlled_subject("Status Report"),
+    ("*", "SR"): _controlled_subject("Status Report"),
+    ("*", "RD"): _controlled_subject("Status Report"),
 }
 
 
@@ -119,7 +143,6 @@
     "status report": {"id": "publication-report"},
     "progress report": {"id": "publication-report"},
     "addendum": {"id": "publication-other"},
-    "decisions": {"id": "publication-meetingminutes"},
     "commentaires": {"id": "publication-peerreview"},
     "comments": {"id": "publication-peerreview"},
     "proposition": {"id": "publication-proposal"},
@@ -130,6 +153,29 @@
 }
 
 
+# Free-text phrases that carry a variable segment in the middle, which the
+# fixed phrases in `_SERIES_RESOURCE_TYPES` can't express. Matched by
+# `_free_text_resource_type` (245__ title and 250__ edition), and tried
+# before the fixed phrases: a pattern is more specific than the bare words
+# ("report", "note", "minutes", ...) its match may happen to contain.
+_FREE_TEXT_PATTERN_RESOURCE_TYPES = (
+    # "Decisions of the 117th meeting of the Nuclear Physics Research
+    # Committee ...", "Decisions of the 22nd meeting of the ...". The
+    # meeting designation between "the" and "meeting" is optional and can
+    # be a numeral ("117th", "22nd") or spelled out ("third"), so allow a
+    # few arbitrary words there. Also covers the "Decision taken at the
+    # meeting" wording already listed in `_SERIES_RESOURCE_TYPES` (kept
+    # there for 490__ series, which is matched exactly - see
+    # `_apply_series_resource_type`).
+    (
+        re.compile(
+            r"\bdecisions?\s+(?:of|taken\s+at)\s+the\s+(?:\S+\s+){0,3}meeting\b"
+        ),
+        {"id": "publication-meetingminutes"},
+    ),
+)
+
+
 # Priority tiers for the different ways a research-committee record's
 # resource_type can be derived, most to least reliable/specific:
 # <COMMITTEE>-<TYPE>-<NUMBER> report number (structured, unambiguous) > a
@@ -224,9 +270,8 @@ def _apply_committee_report_number(self, identifier):
     _set_resource_type_if_higher_priority(self, resource_type, _RANK_REPORT_NUMBER)
     if subject:
         subjects = self.get("subjects", [])
-        new_subject = {"subject": subject}
-        if new_subject not in subjects:
-            subjects.append(new_subject)
+        if subject not in subjects:
+            subjects.append(subject)
         self["subjects"] = subjects
 
 
@@ -298,14 +343,20 @@ def _apply_series_resource_type(self, value_a):
 def _free_text_resource_type(text):
     """Return a resource_type matched from free-text phrases, or None.
 
-    Return the resource_type for text containing one of
-    `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it (e.g. "Draft minutes
-    of the third meeting of the EEC ...",
+    Return the resource_type for text matching one of
+    `_FREE_TEXT_PATTERN_RESOURCE_TYPES`'s patterns (e.g. "Decisions of the
+    117th meeting of the Nuclear Physics Research Committee ..."), or
+    containing one of `_SERIES_RESOURCE_TYPES`'s phrases anywhere in it
+    (e.g. "Draft minutes of the third meeting of the EEC ...",
     https://cds.cern.ch/record/1015008, or "Addendum 1"), matched at a word
     boundary so e.g. "Reported" doesn't match "report" - or None if it
     doesn't.
     """
     text_lower = text.strip().lower()
+    # Patterns first - see `_FREE_TEXT_PATTERN_RESOURCE_TYPES`.
+    for pattern, resource_type in _FREE_TEXT_PATTERN_RESOURCE_TYPES:
+        if pattern.search(text_lower):
+            return resource_type
     # Longest phrase first, so "letter of intent" is tried before a
     # hypothetical single-word phrase it contains.
     for phrase in sorted(_SERIES_RESOURCE_TYPES, key=len, reverse=True):

tests/cds-rdm/conftest.py

@@ -1015,6 +1015,26 @@ def date_type_v(app, date_type):
         },
     )
 
+    vocabulary_service.create(
+        system_identity,
+        {
+            "id": "submitted",
+            "props": {"datacite": "Submitted"},
+            "title": {"en": "Submitted"},
+            "type": "datetypes",
+        },
+    )
+
+    vocabulary_service.create(
+        system_identity,
+        {
+            "id": "available",
+            "props": {"datacite": "Available"},
+            "title": {"en": "Available"},
+            "type": "datetypes",
+        },
+    )
+
     return vocab
 
 

tests/cds-rdm/test_base_rules.py

@@ -13,6 +13,7 @@
 from cds_migrator_kit.errors import UnexpectedValue
 from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import (
     custom_fields_693,
+    imprint_info,
     normalize,
     note,
     recid,
@@ -397,3 +398,41 @@ def test_note_whitespace_only_ignored(self):
         record = {}
         with pytest.raises(IgnoreKey):
             note(record, "595__", {"a": "   "})
+
+
+class TestImprintInfo269:
+    """Test the 269 (preprint) imprint_info function from base.py.
+
+    It never sets `publication_date` itself - it stashes the parsed date
+    under `record["preprint_date"]`, to be reconciled with a possible 260
+    (article) date once resource_type is known, in PublicationDateMapper.
+    """
+
+    def test_imprint_info_269_full(self):
+        """Test full imprint info with place, publisher, and date."""
+        record = {"custom_fields": {}}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"a": "Geneva.", "b": "CERN", "c": "2021"})
+        assert record["preprint_date"] == "2021"
+        assert record["publisher"] == "CERN"
+        assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva"
+
+    def test_imprint_info_269_publisher_not_overwritten(self):
+        """Test that existing publisher is not overwritten."""
+        record = {"custom_fields": {}, "publisher": "Existing Publisher"}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"b": "CERN", "c": "2021"})
+        assert record["publisher"] == "Existing Publisher"
+
+    def test_imprint_info_269_no_date_ignored(self):
+        """Test that missing date raises IgnoreKey and sets no date."""
+        record = {"custom_fields": {}}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"a": "Geneva", "b": "CERN"})
+        assert "preprint_date" not in record
+
+    def test_imprint_info_269_invalid_date_raises_error(self):
+        """Test that invalid date raises error."""
+        record = {"custom_fields": {}}
+        with pytest.raises(UnexpectedValue):
+            imprint_info(record, "269__", {"c": "not-a-valid-date"})