Shopify · kieran-osgood-shopify · Jul 30, 2026
@@ -6,8 +6,9 @@ Kit sample apps. Two complementary setups live here:
 - A **local** React Native suite, run with `dev rn e2e`, that exercises guest and
   hardcoded buyer identity checkouts from seeded carts through Shopify checkout
   and back to the app. Tags select which tests run.
-- A **CI matrix** that expands applications, OS version tags, and suites into
-  BrowserStack Maestro run rows, starting with a shared launch smoke.
+- A **CI matrix** that expands applications and OS version tags into BrowserStack
+  Maestro run rows. Every row runs the whole `tests/` folder and tags select what
+  runs inside it.
 
 ## Run locally
 
@@ -90,24 +91,30 @@ the React Native sample configuration, not local in-repo native SDK overrides.
 
 ## Matrix
 
-CI runs are described by `config/matrix.yml`. The matrix expands applications, OS
-version tags, and suites into a BrowserStack run plan. Because Bitrise has no
-built-in matrix support, `e2e/lib/e2e_matrix_to_browserstack_run_plan.rb`
-transforms the matrix into a BrowserStack run plan and the pipeline
-parallelizes over the resulting rows.
+CI runs are described by `config/matrix.yml`. The matrix expands applications and
+OS version tags into a BrowserStack run plan. Because Bitrise has no built-in
+matrix support, `e2e/lib/e2e_matrix_to_browserstack_run_plan.rb` transforms the
+matrix into a BrowserStack run plan and the pipeline parallelizes over the
+resulting rows.
 
 Current applications:
 
 - React Native iOS sample app
 - React Native Android sample app
+- Kotlin Android sample app
+- Swift iOS sample app
 
 Current OS version tags:
 
 - `latest`
 
-Current suites:
+Every run executes the whole `tests/` folder. The top-level `tags:` block sets the
+default include and exclude lists, and an application may override either one to
+adopt a test before the others carry it. Adding a test adds no rows here.
 
-- `tests/shared/launch-smoke.yaml`
+The run plan derives `E2E_CONTROL_LINK` as `<app_id>://e2e`, because the deep link
+scheme equals the app id on all four targets. Each run also supplies `E2E_APP_ID`
+and `E2E_READY_MARKER`, and the local runners supply the same three names.
 
 Validate the matrix:
 
@@ -144,7 +151,7 @@ ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count
   from a bootstrapped cart with hardcoded buyer identity.
 - `config/matrix.yml`, `lib/e2e_matrix_to_browserstack_run_plan.rb`, and
   `scripts/` drive the BrowserStack run plan.
-- `tests/shared/launch-smoke.yaml` is the shared launch smoke suite.
+- `tests/shared/launch-smoke.yaml` is the shared launch smoke test.
 
 ## Shared app contract
 

@@ -1,5 +1,14 @@
 version: 1
 changed_file_filters: .ci/changed-file-filters.yml
+tests_path: tests
+# Every run executes the whole tests folder and lets tags decide what runs there.
+# An application may override either list to adopt a test ahead of the others.
+tags:
+  include:
+    - launch
+  exclude:
+    - flaky
+    - wip
 applications:
   - id: react-native-ios
     target: react-native
@@ -52,6 +61,3 @@ applications:
       - ciFilters
 os_version_tags:
   - latest
-suites:
-  - id: launch-smoke
-    execute: tests/shared/launch-smoke.yaml
@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - launchApp:
     clearState: true
@@ -8,12 +8,12 @@ appId: ${APP_ID}
       AppleLanguages: "(en)"
 - extendedWaitUntil:
     visible:
-      id: checkout-kit-sample-ready
+      id: ${E2E_READY_MARKER}
     timeout: 10000
 # Reset app data before testing the deep-link cold-start path. The app is
 # stopped so openLink is the launch that seeds the cart.
 - stopApp
-- openLink: "${CART_BOOTSTRAP_BASE_LINK}&buyerIdentityMode=${BUYER_IDENTITY_MODE}"
+- openLink: "${E2E_CONTROL_LINK}/cart?productIndex=0&quantity=1&buyerIdentityMode=${BUYER_IDENTITY_MODE}"
 - waitForAnimationToEnd
 - extendedWaitUntil:
     visible:

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - extendedWaitUntil:
     visible: "${POST_SUBMIT_RESULT_PATTERN}"

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 # RN E2E runs against released native SDK artifacts. Android 4.0.0-alpha.2
 # still exposes checkoutKitCloseBtn; #333 normalized the source id to

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - runFlow:
     when:

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - tapOn:
     text: "^Email( or mobile phone number)?$"

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - scrollUntilVisible:
     element:

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - scrollUntilVisible:
     element:

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - tapOn:
     id: checkout-button

@@ -1,4 +1,4 @@
-appId: ${APP_ID}
+appId: ${E2E_APP_ID}
 ---
 - scrollUntilVisible:
     element:

@@ -12,11 +12,14 @@
 # parallelizes over it via `run_at`/`count`.
 #
 # Today the numbers make the config and the expansion look equivalent -
-# 2 applications + 1 os_version_tag + 1 suite, and 2 * 1 * 1 = 2 runs - so it can look
+# 4 applications + 1 os_version_tag, and 4 * 1 = 4 runs - so it can look
 # like a plain YAML-to-JSON copy. The point is the multiplication, not the copy:
 # adding a single os_version_tag (e.g. a minimum-supported OS) transparently duplicates
-# every application x suite across that OS, turning an additive config change into
+# every application across that OS, turning an additive config change into
 # a multiplicative set of runs without hand-writing each one.
+#
+# Every run executes the whole tests folder. Tags decide what runs inside it, so a new
+# test file adds no rows here - it only needs a tag the matrix already includes.
 class E2EMatrixToBrowserStackRunPlan
   # Drift between the matrix and the pipeline graph happens in both directions: a stale
   # branch predating a newly added target, or a build workflow deleted while the matrix
@@ -39,10 +42,8 @@ def initialize(config_path, config, changed_files: nil)
   def expand
     ensure_valid!
     selected_applications.flat_map do |application|
-      os_version_tags.flat_map do |os_version_tag|
-        suites.map do |suite|
-          build_run(application, os_version_tag, suite)
-        end
+      os_version_tags.map do |os_version_tag|
+        build_run(application, os_version_tag)
       end
     end
   end
@@ -96,11 +97,11 @@ def validation_errors
     errors << "version must be 1" unless @config.fetch("version", nil) == 1
     validate_collection(errors, "applications", applications)
     validate_collection(errors, "os_version_tags", os_version_tags)
-    validate_collection(errors, "suites", suites)
     validate_changed_file_filters(errors)
+    validate_tests_path(errors)
     validate_applications(errors)
     validate_os_version_tags(errors)
-    validate_suites(errors)
+    validate_default_tags(errors)
     errors
   end
 
@@ -111,27 +112,43 @@ def ensure_valid!
 
   private
 
-  def build_run(application, os_version_tag, suite)
+  def build_run(application, os_version_tag)
     platform = application.fetch("platform")
     os_version_tag_id = os_version_tag_id(os_version_tag)
-    suite_id = suite.fetch("id")
     application_id = application.fetch("id")
+    app_id = application.fetch("app_id")
 
     {
-      "id" => "#{application_id}-#{os_version_tag_id}-#{suite_id}",
+      "id" => "#{application_id}-#{os_version_tag_id}",
       "application_id" => application_id,
       "target" => application.fetch("target"),
       "platform" => platform,
       "os_version_tag" => os_version_tag_id,
       "device_selector" => device_selector(platform, os_version_tag),
-      "app_id" => application.fetch("app_id"),
+      "app_id" => app_id,
+      "control_link" => control_link(app_id),
       "artifact_env" => application.fetch("artifact_env"),
-      "execute" => suite.fetch("execute"),
+      "execute" => tests_path,
+      "include_tags" => application_tags(application, "include"),
+      "exclude_tags" => application_tags(application, "exclude"),
       "ready_marker" => application.fetch("ready_marker"),
-      "status_context" => "checkout-kit/e2e/#{application_id}/#{os_version_tag_id}/#{suite_id}"
+      "status_context" => "checkout-kit/e2e/#{application_id}/#{os_version_tag_id}"
     }
   end
 
+  # The control link scheme is the app id on all four targets, so deriving it keeps
+  # one source of truth instead of a second copy that drifts.
+  def control_link(app_id)
+    "#{app_id}://e2e"
+  end
+
+  def application_tags(application, kind)
+    override = application.fetch("#{kind}_tags", nil)
+    return Array(override) if override
+
+    Array(default_tags.fetch(kind, nil))
+  end
+
   def application_matches_changed_files?(application)
     filter_names = application_changed_file_filter_names(application)
     return true if filter_names.empty?
@@ -185,8 +202,21 @@ def os_version_tags
     @config.fetch("os_version_tags", []) || []
   end
 
-  def suites
-    @config.fetch("suites", []) || []
+  def tests_path
+    @config.fetch("tests_path", "tests")
+  end
+
+  def default_tags
+    @config.fetch("tags", {}) || {}
+  end
+
+  def declared_tags
+    @declared_tags ||= Dir.glob("#{tests_path}/**/*.yaml", base: e2e_root).flat_map do |path|
+      header = File.read(File.join(e2e_root, path)).split("\n---\n").first
+      YAML.safe_load(header)["tags"] || []
+    rescue Psych::Exception
+      []
+    end.uniq
   end
 
   def validate_collection(errors, name, collection)
@@ -219,6 +249,7 @@ def validate_applications(errors)
         errors << "application #{id} missing #{key}" if application.fetch(key, "").to_s.empty?
       end
       validate_application_changed_files_filters(errors, application)
+      validate_application_tags(errors, application)
       platform = application.fetch("platform", nil)
       errors << "application #{id} platform must be ios or android" unless ["ios", "android"].include?(platform)
     end
@@ -272,21 +303,57 @@ def safe_os_version_tag_id(os_version_tag)
     nil
   end
 
-  def validate_suites(errors)
-    return unless suites.is_a?(Array)
+  def validate_tests_path(errors)
+    return if File.directory?(File.join(e2e_root, tests_path))
 
-    validate_unique_ids(errors, "suite", suites)
-    suites.each do |suite|
-      id = suite.fetch("id", "")
-      errors << "suite missing id" if id.to_s.empty?
-      execute = suite.fetch("execute", "")
-      errors << "suite #{id} missing execute" if execute.empty?
-      next if execute.empty?
+    errors << "tests_path is not a directory: #{tests_path}"
+  end
+
+  # An include tag no test carries produces a green run that tested nothing.
+  def validate_default_tags(errors)
+    ["include", "exclude"].each do |kind|
+      tags = default_tags.fetch(kind, nil)
+      next if tags.nil?
+
+      unless tags.is_a?(Array)
+        errors << "tags #{kind} must be an array"
+        next
+      end
 
-      errors << "suite #{id} execute path does not exist: #{execute}" unless File.exist?(File.join(e2e_root, execute))
+      next unless kind == "include"
+
+      errors.concat(unknown_include_tag_errors(tags) { |tag| "tags include '#{tag}' but no test in tests/ carries it" })
+    end
+  end
+
+  def validate_application_tags(errors, application)
+    id = application.fetch("id", "<missing>")
+
+    ["include", "exclude"].each do |kind|
+      tags = application.fetch("#{kind}_tags", nil)
+      next if tags.nil?
+
+      unless tags.is_a?(Array)
+        errors << "application #{id} #{kind}_tags must be an array"
+        next
+      end
+
+      next unless kind == "include"
+
+      errors.concat(
+        unknown_include_tag_errors(tags) do |tag|
+          "application #{id} include_tags '#{tag}' but no test in tests/ carries it"
+        end
+      )
     end
   end
 
+  def unknown_include_tag_errors(tags)
+    return [] if declared_tags.empty?
+
+    (tags - declared_tags).map { |tag| yield(tag) }
+  end
+
   def validate_unique_ids(errors, label, collection)
     ids = collection.map { |item| item.fetch("id", nil) }
     errors << "#{label} ids must be unique" unless ids.compact.uniq.length == ids.compact.length

@@ -147,9 +147,14 @@ class BrowserStackRunExecutor
       customBuildName: run.fetch("id"),
       devices: [device],
       execute: [run.fetch("execute")],
+      tags: {
+        includeTags: run.fetch("include_tags"),
+        excludeTags: run.fetch("exclude_tags")
+      },
       setEnvVariables: {
         E2E_APP_ID: run.fetch("app_id"),
-        E2E_READY_MARKER: run.fetch("ready_marker")
+        E2E_READY_MARKER: run.fetch("ready_marker"),
+        E2E_CONTROL_LINK: run.fetch("control_link")
       }
     }
     response = @client.start_build(run.fetch("platform"), body)
@@ -205,6 +210,8 @@ class BrowserStackRunExecutor
       "target" => run.fetch("target"),
       "os_version_tag" => run.fetch("os_version_tag"),
       "execute" => run.fetch("execute"),
+      "include_tags" => run.fetch("include_tags"),
+      "exclude_tags" => run.fetch("exclude_tags"),
       "device_selector" => device.fetch("device_selector"),
       "resolved_device" => device.fetch("resolved_device"),
       "resolved_os_version" => device.fetch("resolved_os_version"),
@@ -228,6 +235,8 @@ class BrowserStackRunExecutor
       "target" => run["target"],
       "os_version_tag" => run["os_version_tag"],
       "execute" => run["execute"],
+      "include_tags" => run["include_tags"],
+      "exclude_tags" => run["exclude_tags"],
       "device_selector" => run["device_selector"],
       "resolved_device" => device["resolved_device"] || run["device_selector"],
       "resolved_os_version" => device["resolved_os_version"],