diff --git a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py index 9a28a298..9e346a36 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/custom_fields.py @@ -63,8 +63,26 @@ def apply(self, ctx): "cern:departments", [] ) for department in departments: + if department.lower() == "xx": + continue if "-" in department: dep = department.split("-")[0] + if not ctx.custom_fields.get("cern:administrative_unit"): + ctx.custom_fields["cern:administrative_unit"] = department + elif ( + ctx.custom_fields.get("cern:administrative_unit").lower() + == department.lower() + ): + continue + else: + raise UnexpectedValue( + subfield="5", + value=department, + field="710", + message=f"conflict on administrative unit " + f"{ctx.custom_fields['cern:administrative_unit']} VS {department}", + stage="vocabulary match", + ) else: dep = department result = search_vocabulary(dep, "departments") diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 7d2f2516..4603be42 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -8,8 +8,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "9eff4bab-9e5e-4cdb-9d16-be6588f7c6c1" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" eec_comm: data_dir: cds_migrator_kit/rdm/data/committees/eec extract: @@ -18,8 +18,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "71a01272-00c7-48d2-9140-487d01ae8367" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" emc_comm: data_dir: cds_migrator_kit/rdm/data/committees/emc extract: @@ -28,8 +28,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "7b33dda1-a8b0-4d6b-808f-707fa3e63828" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" isc_comm: data_dir: cds_migrator_kit/rdm/data/committees/isc extract: @@ -38,8 +38,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "12019bcd-28f2-40ce-85f3-3bfff399e847" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" istc_comm: data_dir: cds_migrator_kit/rdm/data/committees/istc extract: @@ -48,8 +48,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "84c28df7-fbee-4332-883b-7b9b77480e1f" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" isrc_comm: data_dir: cds_migrator_kit/rdm/data/committees/isrc extract: @@ -58,8 +58,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2285af28-1808-4f11-aefb-64a5699940bb" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" lepc_comm: data_dir: cds_migrator_kit/rdm/data/committees/lepc extract: @@ -68,8 +68,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "a276bb34-e6ba-47c8-aa18-0ade1eec8b1d" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" nprc_comm: data_dir: cds_migrator_kit/rdm/data/committees/nprc extract: @@ -78,8 +78,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "5c7c772d-8513-49d3-bd43-ac9a8beec02d" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" nsc_comm: data_dir: cds_migrator_kit/rdm/data/committees/nsc extract: @@ -88,8 +88,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "847936da-b488-4426-b9aa-3659fa473779" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" phi_comm: data_dir: cds_migrator_kit/rdm/data/committees/phi extract: @@ -98,8 +98,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2d9b3d0c-930a-4231-8c3e-55feaf9150c0" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" phi3_comm: data_dir: cds_migrator_kit/rdm/data/committees/phi3 extract: @@ -108,8 +108,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "c35d2da3-8a47-420b-879e-be87b8b602c7" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" psc_comm: data_dir: cds_migrator_kit/rdm/data/committees/psc extract: @@ -118,8 +118,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "9d91ff75-573c-48db-a1c1-4f78b0a9dced" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" pscc_comm: data_dir: cds_migrator_kit/rdm/data/committees/pscc extract: @@ -128,8 +128,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "dd2c1204-4508-43d6-a7ac-7ec5e1394523" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" scc_comm: data_dir: cds_migrator_kit/rdm/data/committees/scc extract: @@ -138,8 +138,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "28bf99b9-1e72-405a-b95a-828019831def" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" scp_adv: data_dir: cds_migrator_kit/rdm/data/committees/sc_sdv extract: @@ -148,8 +148,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "dbcc1269-a481-4d40-8ef9-66305c831840" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" spsc_comm: data_dir: cds_migrator_kit/rdm/data/committees/spsc extract: @@ -158,8 +158,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "0ce97700-d80e-4bbb-89c4-690be8f37817" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" spslc_comm: data_dir: cds_migrator_kit/rdm/data/committees/spslc extract: @@ -168,8 +168,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "4db1a084-4280-401d-a260-54229944af1e" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" tcc_comm: data_dir: cds_migrator_kit/rdm/data/committees/tcc extract: @@ -178,8 +178,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/committees/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2ca82423-148a-42b5-86ff-03336b961c8e" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" antares: data_dir: cds_migrator_kit/rdm/data/former_exp/antares plots: true @@ -189,8 +189,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "e2d2ed70-8a3c-453b-aa5b-1fa985f0906b" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" antares_restr: data_dir: cds_migrator_kit/rdm/data/former_exp/antares_restr restricted: "True" @@ -201,8 +201,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "e2d2ed70-8a3c-453b-aa5b-1fa985f0906b" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" dirac: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/dirac @@ -212,8 +212,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "6a289642-5378-4daf-87b5-bb58af00487a" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" harp_cdp: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/harp_cdp @@ -223,8 +223,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2541c76f-6c9b-415b-9a47-e814ec01f576" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" imxgam: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/imXgam @@ -235,8 +235,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "88a5cdf4-974a-44d4-b145-d19ee6346bb8" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" lcd_restr: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/lcd_restr @@ -248,7 +248,6 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "adc02716-780b-4bba-8e89-6316c9a11cf0" lcd: plots: true @@ -260,8 +259,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "adc02716-780b-4bba-8e89-6316c9a11cf0" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" re29: data_dir: cds_migrator_kit/rdm/data/former_exp/re29 plots: true @@ -271,8 +270,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "aff67e22-9027-4401-a16f-1aaac2cd2c27" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" re29_restr: data_dir: cds_migrator_kit/rdm/data/former_exp/re29_restr plots: true @@ -284,8 +283,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "aff67e22-9027-4401-a16f-1aaac2cd2c27" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" ua2: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/ua2 @@ -295,8 +294,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "4040d2c7-7648-4f1a-81fa-7112fec8d130" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" ua4: data_dir: cds_migrator_kit/rdm/data/former_exp/ua4 plots: true @@ -306,8 +305,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "1b9239a7-5938-44e3-8ea8-2c530627f659" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" ua5: data_dir: cds_migrator_kit/rdm/data/former_exp/ua5 plots: true @@ -317,8 +316,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "7996b881-2420-4791-ac17-00cf3c1e7196" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" ua8: plots: true data_dir: cds_migrator_kit/rdm/data/former_exp/ua8 @@ -328,8 +327,8 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "2dbeb56c-b4ab-4cad-8d82-5a1f90499bd7" + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" fap: data_dir: cds_migrator_kit/rdm/data/fap restricted: "True" @@ -351,39 +350,6 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "2ebd631f-66c3-4891-b2dd-275ec3656b7b" - faser-drafts: - data_dir: cds_migrator_kit/rdm/data/faser-drafts - restricted: "True" - create_inclusion_request: true - access_grants_view: - - faser-all - extract: - dirpath: cds_migrator_kit/rdm/data/faser-drafts/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser-drafts/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "33af9368-5bad-45cb-9360-8c9e5dfca09f" - faser: - data_dir: cds_migrator_kit/rdm/data/faser - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/faser/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "33af9368-5bad-45cb-9360-8c9e5dfca09f" - faser-ep: - data_dir: cds_migrator_kit/rdm/data/faser - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/faser-ep/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "e5de962b-5e2c-408f-be69-2823fda1f732" isolde: data_dir: cds_migrator_kit/rdm/data/isolde plots: true @@ -398,25 +364,6 @@ records: - "1c46ceab-af68-4502-a436-cdf5a4cfbbe0" load: legacy_pids_to_redirect: cds_migrator_kit/rdm/data/isolde/duplicated_pids.json - staff_association: - data_dir: cds_migrator_kit/rdm/data/staff_association - extract: - dirpath: cds_migrator_kit/rdm/data/staff_association/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/staff_association/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "3d0a8ba2-7ec6-4016-b171-325d68e50e51" - bulletin_drafts: - data_dir: cds_migrator_kit/rdm/data/bulletin_drafts - restricted: "True" - extract: - dirpath: cds_migrator_kit/rdm/data/bulletin_drafts/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/bulletin_drafts/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "3d0a8ba2-7ec6-4016-b171-325d68e50e51" ts_dep: plots: true data_dir: cds_migrator_kit/rdm/data/dep_ts diff --git a/cds_migrator_kit/rdm/streams_done.yaml b/cds_migrator_kit/rdm/streams_done.yaml index ceac9fc1..8b445758 100644 --- a/cds_migrator_kit/rdm/streams_done.yaml +++ b/cds_migrator_kit/rdm/streams_done.yaml @@ -298,3 +298,36 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "e849139c-dff5-45a7-90b1-90cdddc9c00e" + faser-drafts: + data_dir: cds_migrator_kit/rdm/data/faser-drafts + restricted: "True" + create_inclusion_request: true + access_grants_view: + - faser-all + extract: + dirpath: cds_migrator_kit/rdm/data/faser-drafts/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/faser-drafts/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "33af9368-5bad-45cb-9360-8c9e5dfca09f" + faser: + data_dir: cds_migrator_kit/rdm/data/faser + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/faser/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "33af9368-5bad-45cb-9360-8c9e5dfca09f" + faser-ep: + data_dir: cds_migrator_kit/rdm/data/faser + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/faser-ep/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "e5de962b-5e2c-408f-be69-2823fda1f732"