Fail ocp_testing when stage_unstable_flag is set after all stages

Test stages mark failures as UNSTABLE via stage_unstable_flag during
rescue blocks but previously allowed the playbook to exit 0, so Zuul
and ci-framework reported SUCCESS despite failing tests.

Add fail_if_stage_unstable.yml and a final play in ocp_testing.yaml
that reads accumulated messages and fails once all stages complete.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tusharjadhav3302

collection/stages/roles/verification/defaults/main.yml

@@ -5,7 +5,7 @@ cinder_csi_project_name: cinder-test # project for testing cinder-csi sc provide
 manila_project_name: manila-test # project for testing manila sc provider
 demo_project_name: demo # project for testing demo app
 verify_existing_namespace: false
-soft_checks: true # run checks that won't stop the playbook but will generate the jenkins message to mark the run as unstable.
+soft_checks: true # run checks that won't stop the playbook but will mark the run as UNSTABLE via stage_unstable_flag.
 alerts_to_ignore:
   # - 'Watchdog' # Should not be considered as severity is none
   - 'NodeClockNotSynchronising'

collection/tools/roles/tools_stage_results/tasks/fail_if_stage_unstable.yml

@@ -0,0 +1,28 @@
+---
+- name: Check whether any stage was marked UNSTABLE
+  ansible.builtin.stat:
+    path: "{{ stage_unstable_flag }}"
+  register: _stage_unstable_flag
+
+- name: Read accumulated UNSTABLE stage messages
+  ansible.builtin.set_fact:
+    _stage_unstable_messages: >-
+      {{ lookup('ansible.builtin.file', stage_unstable_flag)
+         | split('\n')
+         | map('trim')
+         | reject('equalto', '')
+         | list
+         | join('; ') }}
+  when: _stage_unstable_flag.stat.exists
+
+- name: Print accumulated UNSTABLE stage messages
+  ansible.builtin.debug:
+    msg: "{{ _stage_unstable_messages }}"
+  when: _stage_unstable_flag.stat.exists
+
+- name: Fail the playbook when any stage was marked UNSTABLE
+  ansible.builtin.fail:
+    msg: >-
+      One or more stages were marked UNSTABLE:
+      {{ _stage_unstable_messages }}
+  when: _stage_unstable_flag.stat.exists

configs/global.yml

@@ -32,7 +32,9 @@ report_dir: "{{ artifacts_dir }}/test_results"
 
 # Sets the Ansible Controller Node machine's HOME environment variable
 controller_home_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}"
-# The resources_file and stage_unstable_flag are in the Ansible control node's HOME environment variable
+# The resources_file and stage_unstable_flag are on the Ansible control node's HOME.
+# stage_unstable_flag accumulates UNSTABLE stage messages during the run; ocp_testing.yaml
+# fails at the end if the file exists so Zuul/ci-framework see a non-zero exit.
 resources_file: "{{ controller_home_dir }}/artifacts/resources.yml"
 stage_unstable_flag: "{{ controller_home_dir }}/artifacts/stage_unstable_flag"
 

playbooks/ocp_testing.yaml

@@ -119,3 +119,14 @@
 - name: Run EgressIP tests on OpenShift
   ansible.builtin.import_playbook: plays/egressip_tests.yaml
   when: "'egressip_tests' in stages"
+
+- name: Fail if any stage was marked UNSTABLE
+  hosts: localhost
+  gather_facts: false
+  vars_files:
+    - "../configs/global.yml"
+  tasks:
+    - name: Check accumulated UNSTABLE stages and fail the playbook
+      ansible.builtin.include_role:
+        name: tools_stage_results
+        tasks_from: fail_if_stage_unstable.yml