Update from source

cmd/api.go

@@ -50,7 +50,7 @@ Examples:
 func init() {
 	apiCmd.Flags().StringP("method", "X", "", "HTTP method (default: GET, or POST if --json or --file is provided)")
 	apiCmd.Flags().StringArray("file", nil, "Send the request as multipart/form-data with a file part: --file @path, or --file field=@path to name the part (repeatable). --json then supplies the request's other form fields")
-	apiCmd.Flags().Bool("no-preflight", false, "Send the request even if the path is absent from the API spec")
+	apiCmd.Flags().Bool("no-preflight", false, "Send the request even if the path is absent from the API spec; it cannot make a missing endpoint exist")
 	rootCmd.AddCommand(apiCmd)
 }
 
@@ -315,13 +315,21 @@ func preflightPath(cmd *cobra.Command, c *client.Client, httpMethod, resolvedPat
 
 	suggestions := idx.Suggest(resolvedPath, 5)
 	hint := "run 'cio schema' to list resources"
-	if len(suggestions) > 0 && suggestions[0].Resource != "" {
-		hint = fmt.Sprintf("run 'cio schema %s' to list that resource's endpoints", suggestions[0].Resource)
+	// Suggest ranks by shared leading segments, so an unknown resource still
+	// gets a neighbour from under the same scope — a different resource.
+	if requested := routes.ResourceFor(resolvedPath); requested != "" {
+		for _, s := range suggestions {
+			if s.Resource == requested {
+				hint = fmt.Sprintf("run 'cio schema %s' to list that resource's endpoints", requested)
+				break
+			}
+		}
 	}
 	// The spec omits a few real endpoints, and for those `cio schema` cannot
 	// list what it does not describe — so the way past a wrong rejection has to
 	// be in the message, not only in the details below.
-	hint += ", or resend with --no-preflight if you know the endpoint exists"
+	hint += ". --no-preflight resends without this check, but it cannot make a missing endpoint exist: " +
+		"an unknown path returns the app's HTML page rather than JSON, so reserve it for an endpoint you know the spec omits"
 
 	closest := make([]string, 0, len(suggestions))
 	for _, s := range suggestions {

cmd/api_preflight_test.go

@@ -10,12 +10,16 @@ import (
 
 // preflightSpec is a Journeys OpenAPI fixture with enough of the campaigns
 // resource to exercise the pre-send path check: a collection with two verbs and
-// a single-campaign read.
+// a single-campaign read, plus an alphabetically earlier second resource the
+// suggestion ranking would otherwise reach for.
 func preflightSpec() string {
 	return `{
 		"openapi": "3.1.0",
 		"info": {"title": "Test", "version": "1.0.0"},
 		"paths": {
+			"/v1/environments/{environment_id}/all_metrics": {
+				"get": {"summary": "List metrics"}
+			},
 			"/v1/environments/{environment_id}/campaigns": {
 				"get": {"summary": "List campaigns"},
 				"post": {"summary": "Create campaign"}
@@ -352,3 +356,36 @@ func TestAPIPreflight_FailsOpenWhenSpecUnavailable(t *testing.T) {
 		t.Errorf("expected the request to be sent, got %v", got)
 	}
 }
+
+// The nearest route under the same scope is some other resource entirely.
+func TestAPIPreflight_DoesNotNameAnUnrelatedResource(t *testing.T) {
+	server := setupPreflightTest(t)
+
+	_, _, err := executeCommand("api", "/v1/environments/456/topics", "--api-url", server.URL)
+	if err == nil {
+		t.Fatal("expected an unknown resource to be rejected before sending")
+	}
+	if strings.Contains(err.Error(), "cio schema all_metrics") {
+		t.Errorf("error must not point at a resource the path never named, got: %s", err.Error())
+	}
+	if !strings.Contains(err.Error(), "run 'cio schema' to list resources") {
+		t.Errorf("error should fall back to listing resources, got: %s", err.Error())
+	}
+}
+
+// Without this, a rejected guess is just resent with the check off.
+func TestAPIPreflight_CautionsAgainstForcingAGuessedPath(t *testing.T) {
+	server := setupPreflightTest(t)
+
+	_, _, err := executeCommand("api", "/v1/environments/456/campaigns/48/actions",
+		"--api-url", server.URL)
+	if err == nil {
+		t.Fatal("expected an invented sub-path to be rejected before sending")
+	}
+	if !strings.Contains(err.Error(), "cannot make a missing endpoint exist") {
+		t.Errorf("error should say --no-preflight does not create the endpoint, got: %s", err.Error())
+	}
+	if !strings.Contains(err.Error(), "HTML page rather than JSON") {
+		t.Errorf("error should name what forcing it returns, got: %s", err.Error())
+	}
+}

cmd/prime_context.md

@@ -202,8 +202,11 @@ real endpoints plus the `cio schema` command that lists them — so treat it as
 rather than trying another spelling. A path that exists only for another method
 reports that method and the `-X` flag to use.
 
-A few real endpoints are absent from the spec. If you know the path exists,
-resend it with `--no-preflight`.
+`--no-preflight` skips the check; it does not make a missing endpoint exist. An
+unknown path is answered by the web app with its HTML page and a 200, so forcing
+a guessed path returns no data. Reserve the flag for the few real endpoints the
+spec omits, where you already know the path; otherwise run `cio schema
+<resource>` and use a documented endpoint.
 
 ## Retry Behavior
 

internal/client/pagination.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -26,7 +27,10 @@ type paginationMeta struct {
 	Total      *int
 	HasMore    *bool
 	EmptyData  bool
-	DataKey    string
+	// DataLen is the number of records on this page: the length of the
+	// collection array, taken as the largest top-level array when the key is
+	// not one of the well-known names.
+	DataLen int
 }
 
 const maxAutoPages = 10000
@@ -44,6 +48,11 @@ func (c *Client) PageAll(cfg PageAllConfig) error {
 	}
 
 	page := cfg.StartPage
+	// The page size for the total-based stop is what the server actually
+	// returned on the first page, not --limit: servers clamp oversized limits,
+	// and trusting the flag would end the walk early. A later, final page may
+	// be shorter, so only the first page is read.
+	pageSize := 0
 
 	for {
 		params := copyParams(cfg.Params)
@@ -62,13 +71,16 @@ func (c *Client) PageAll(cfg PageAllConfig) error {
 		}
 
 		meta := extractPaginationMeta(result, cfg.Limit)
+		if pageSize == 0 {
+			pageSize = meta.DataLen
+		}
 
 		if meta.TotalPages != nil && page >= *meta.TotalPages {
 			return nil
 		}
 
-		if meta.Total != nil && cfg.Limit > 0 {
-			totalPages := int(math.Ceil(float64(*meta.Total) / float64(cfg.Limit)))
+		if meta.Total != nil && pageSize > 0 {
+			totalPages := int(math.Ceil(float64(*meta.Total) / float64(pageSize)))
 			if totalPages == 0 {
 				totalPages = 1
 			}
@@ -101,17 +113,16 @@ func extractPaginationMeta(data json.RawMessage, limit int) paginationMeta {
 		return meta
 	}
 
-	if raw, ok := obj["total_pages"]; ok {
-		var v int
-		if json.Unmarshal(raw, &v) == nil {
-			meta.TotalPages = &v
+	// Totals live either at the top level or, on Journeys list endpoints,
+	// under meta.pagination.
+	meta.TotalPages = intField(obj, "total_pages")
+	meta.Total = intField(obj, "total")
+	if pagination := nestedObject(obj, "meta", "pagination"); pagination != nil {
+		if meta.TotalPages == nil {
+			meta.TotalPages = intField(pagination, "total_pages")
 		}
-	}
-
-	if raw, ok := obj["total"]; ok {
-		var v int
-		if json.Unmarshal(raw, &v) == nil {
-			meta.Total = &v
+		if meta.Total == nil {
+			meta.Total = intField(pagination, "total")
 		}
 	}
 
@@ -123,21 +134,87 @@ func extractPaginationMeta(data json.RawMessage, limit int) paginationMeta {
 	}
 
 	for _, key := range []string{"data", "items", "results", "records", "entries", "campaigns"} {
-		if raw, ok := obj[key]; ok {
-			var arr []json.RawMessage
-			if json.Unmarshal(raw, &arr) == nil {
-				meta.DataKey = key
-				if len(arr) == 0 {
-					meta.EmptyData = true
-				}
-				break
-			}
+		raw, ok := obj[key]
+		if !ok {
+			continue
+		}
+		// A known collection key that is null is an empty page: Go backends
+		// emit null for a nil slice.
+		if isNull(raw) {
+			meta.EmptyData = true
+			return meta
 		}
+		if arr, ok := arrayField(obj, key); ok {
+			meta.DataLen = len(arr)
+			meta.EmptyData = len(arr) == 0
+			return meta
+		}
+	}
+
+	// Endpoints name their collection after the resource (imports, segments,
+	// ...). Without a known key, take the largest top-level array as the
+	// collection, so a response like {"imports": [], "meta": {...}} still ends
+	// the walk instead of running to maxAutoPages.
+	arrays := 0
+	for key := range obj {
+		arr, ok := arrayField(obj, key)
+		if !ok {
+			continue
+		}
+		arrays++
+		meta.DataLen = max(meta.DataLen, len(arr))
 	}
+	meta.EmptyData = arrays > 0 && meta.DataLen == 0
 
 	return meta
 }
 
+func intField(obj map[string]json.RawMessage, key string) *int {
+	raw, ok := obj[key]
+	if !ok {
+		return nil
+	}
+	var v int
+	if json.Unmarshal(raw, &v) != nil {
+		return nil
+	}
+	return &v
+}
+
+func isNull(raw json.RawMessage) bool {
+	return bytes.Equal(bytes.TrimSpace(raw), []byte("null"))
+}
+
+// arrayField returns the field only when it is syntactically an array.
+// json.Unmarshal accepts null into a slice, so an unrelated null field
+// ("next": null) would otherwise read as an empty collection.
+func arrayField(obj map[string]json.RawMessage, key string) ([]json.RawMessage, bool) {
+	raw, ok := obj[key]
+	if !ok || !bytes.HasPrefix(bytes.TrimSpace(raw), []byte("[")) {
+		return nil, false
+	}
+	var arr []json.RawMessage
+	if json.Unmarshal(raw, &arr) != nil {
+		return nil, false
+	}
+	return arr, true
+}
+
+func nestedObject(obj map[string]json.RawMessage, keys ...string) map[string]json.RawMessage {
+	for _, key := range keys {
+		raw, ok := obj[key]
+		if !ok {
+			return nil
+		}
+		var next map[string]json.RawMessage
+		if json.Unmarshal(raw, &next) != nil {
+			return nil
+		}
+		obj = next
+	}
+	return obj
+}
+
 func copyParams(params map[string]string) map[string]string {
 	out := make(map[string]string, len(params))
 	for k, v := range params {