From f8d528f1192e96094188c9308e03504ecfb3d979 Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:42:19 +0700 Subject: [PATCH] feat(api): bound event list responses by size and time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/events and /api/events/preview returned every event stored for the requested type/project — FindOptions already had Limit and Offset, but the handlers never filled them in. On a busy instance one preview request weighed 25.8 MB, and the frontend then filtered that in the browser. Both endpoints now accept limit, offset/page, from, to and window (15m, 24h, "7d"; "all" opts out of the configured default), and report what was applied in the response meta so a client can tell a truncated response from an exhausted one. Defaults are deliberately conservative and non-breaking: 1000 events per response, no time window. An operator can set ui.default_window (or UI_DEFAULT_WINDOW) to also narrow lists by time. --- buggregator.yaml.example | 6 ++ internal/app/app.go | 29 ++++++- internal/app/config.go | 38 +++++++++ internal/event/store.go | 5 ++ internal/server/http/api.go | 142 ++++++++++++++++++++++++++++--- internal/server/http/api_test.go | 113 +++++++++++++++++++++++- internal/storage/sqlite.go | 11 +++ 7 files changed, 331 insertions(+), 13 deletions(-) diff --git a/buggregator.yaml.example b/buggregator.yaml.example index a2bbbc4..fe449f7 100644 --- a/buggregator.yaml.example +++ b/buggregator.yaml.example @@ -21,6 +21,12 @@ storage: mode: ${STORAGE_MODE:memory} # "memory" (default, lost on restart) or "filesystem" path: ${STORAGE_PATH:./storage} # Directory for filesystem mode +# UI event lists: how much /api/events and /api/events/preview return +ui: + default_limit: ${UI_DEFAULT_LIMIT:1000} # Events returned when the request has no limit param + max_limit: ${UI_MAX_LIMIT:5000} # Ceiling for an explicit ?limit= + default_window: ${UI_DEFAULT_WINDOW:} # Default time window, e.g. "1h", "24h", "7d". Empty = no window + # Prometheus metrics metrics: enabled: ${METRICS_ENABLED:false} # Set to true to expose Prometheus metrics diff --git a/internal/app/app.go b/internal/app/app.go index bb62682..13860dd 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "github.com/buggregator/go-buggregator/internal/auth" @@ -105,7 +106,7 @@ func (a *App) Run() { } // Register core API routes (settings endpoint is public, others go through auth middleware). - httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware) + httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware, a.listLimits()) // Register attachment API endpoints. httpserver.RegisterAttachmentAPI(mux, a.db, a.attachments) @@ -232,3 +233,29 @@ func (a *App) Run() { _ = srv.Shutdown(context.Background()) tcpManager.Wait() } + +// listLimits turns the ui config section into list limits. An unreadable +// default_window must not take the service down: the window is dropped and the +// reason is logged. +func (a *App) listLimits() httpserver.ListLimits { + lim := httpserver.ListLimits{ + DefaultLimit: a.cfg.UI.DefaultLimit, + MaxLimit: a.cfg.UI.MaxLimit, + } + + switch w := strings.TrimSpace(a.cfg.UI.DefaultWindow); w { + case "", "all", "0": + lim.DefaultWindow = 0 + default: + d, err := httpserver.ParseWindow(w) + if err != nil || d <= 0 { + slog.Warn("ui.default_window is not a duration, no time window applied", "value", w, "err", err) + d = 0 + } + lim.DefaultWindow = d + } + + slog.Info("event list limits", "default_limit", lim.DefaultLimit, + "max_limit", lim.MaxLimit, "default_window", lim.DefaultWindow) + return lim +} diff --git a/internal/app/config.go b/internal/app/config.go index b482656..e4b50b7 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "regexp" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -23,6 +24,17 @@ type AuthConfig struct { JWTSecret string `yaml:"jwt_secret"` // Secret for signing internal JWT tokens. Required when auth is enabled. } +// UIConfig bounds the event lists served to the UI. +// +// /api/events and /api/events/preview used to return everything stored for a +// project. The defaults below (1000 events, no time window) only cap the +// response size; set default_window to also narrow it by time, e.g. "24h". +type UIConfig struct { + DefaultLimit int `yaml:"default_limit"` // events returned without a limit param (default 1000) + MaxLimit int `yaml:"max_limit"` // ceiling for an explicit limit (default 5000) + DefaultWindow string `yaml:"default_window"` // time window without from/to: "24h", "7d"; empty or "all" = no window +} + // Config holds application configuration. type Config struct { Server ServerConfig `yaml:"server"` @@ -32,6 +44,7 @@ type Config struct { Metrics MetricsConfig `yaml:"metrics"` MCP MCPConfig `yaml:"mcp"` Auth AuthConfig `yaml:"auth"` + UI UIConfig `yaml:"ui"` Modules ModulesConfig `yaml:"modules"` Webhooks []WebhookDef `yaml:"webhooks"` Projects []ProjectDef `yaml:"projects"` @@ -207,6 +220,11 @@ func LoadConfig() Config { cfg.Auth.Scopes = coalesce(os.Getenv("AUTH_SCOPES"), fileCfg.Auth.Scopes, "openid,email,profile") cfg.Auth.JWTSecret = coalesce(os.Getenv("AUTH_JWT_SECRET"), fileCfg.Auth.JWTSecret) + // UI list limits. + cfg.UI.DefaultLimit = coalesceInt(atoiOrZero(os.Getenv("UI_DEFAULT_LIMIT")), fileCfg.UI.DefaultLimit, 1000) + cfg.UI.MaxLimit = coalesceInt(atoiOrZero(os.Getenv("UI_MAX_LIMIT")), fileCfg.UI.MaxLimit, 5000) + cfg.UI.DefaultWindow = coalesce(os.Getenv("UI_DEFAULT_WINDOW"), fileCfg.UI.DefaultWindow) + // CORS origins. cfg.Server.CORSOrigins = fileCfg.Server.CORSOrigins if env := os.Getenv("CORS_ORIGINS"); env != "" { @@ -318,6 +336,26 @@ func expandEnvVars(input string) string { }) } +// coalesceInt is coalesce for numbers: zero counts as "not set". +func coalesceInt(values ...int) int { + for _, v := range values { + if v > 0 { + return v + } + } + return 0 +} + +// atoiOrZero lets an env variable be passed to coalesceInt in one expression: a +// non-empty but non-numeric value counts as "not set". +func atoiOrZero(s string) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil || n < 0 { + return 0 + } + return n +} + func coalesce(values ...string) string { for _, v := range values { if v != "" { diff --git a/internal/event/store.go b/internal/event/store.go index 11d68cd..6573b97 100644 --- a/internal/event/store.go +++ b/internal/event/store.go @@ -8,6 +8,11 @@ type FindOptions struct { Project string Limit int Offset int + + // From and To bound the selection by event time (unix seconds with a + // fraction, same as Event.Timestamp). Zero means "no bound". + From float64 + To float64 } // DeleteOptions configures batch deletion. diff --git a/internal/server/http/api.go b/internal/server/http/api.go index 6f76954..163d2a4 100644 --- a/internal/server/http/api.go +++ b/internal/server/http/api.go @@ -4,6 +4,9 @@ import ( "database/sql" "encoding/json" "net/http" + "strconv" + "strings" + "time" "github.com/buggregator/go-buggregator/internal/event" ) @@ -14,9 +17,132 @@ type AuthSettings struct { LoginURL string } +// ListLimits bounds what /api/events and /api/events/preview return. +// +// Both endpoints used to return every event matching type/project: Limit and +// Offset existed in FindOptions but were never filled in. On a busy project +// that is a very large response — measured on a production instance, a single +// project preview weighed 25.8 MB — which the frontend then filters in the +// browser. +type ListLimits struct { + DefaultLimit int // how many events to return when limit is absent; 0 = unlimited + MaxLimit int // ceiling for an explicit limit; 0 = no ceiling + DefaultWindow time.Duration // time window applied when neither from/to nor window is given; 0 = no window +} + +// parseListOptions builds FindOptions from query parameters. +// +// type, project — unchanged; +// limit — how many events to return (capped by MaxLimit); +// offset / page — offset (page is counted from limit, 1-based); +// from, to — window bounds: unix seconds or RFC3339 ("2026-09-01T10:00:00Z"); +// window — window relative to now: "24h", "15m", "7d"; +// "all" or "0" opt out of DefaultWindow. +func parseListOptions(r *http.Request, lim ListLimits) event.FindOptions { + q := r.URL.Query() + opts := event.FindOptions{ + Type: q.Get("type"), + Project: q.Get("project"), + Limit: lim.DefaultLimit, + } + + if v := q.Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Limit = n + } + } + if lim.MaxLimit > 0 && opts.Limit > lim.MaxLimit { + opts.Limit = lim.MaxLimit + } + + if v := q.Get("offset"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Offset = n + } + } else if v := q.Get("page"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 1 { + opts.Offset = (n - 1) * opts.Limit + } + } + + opts.From = parseTimeParam(q.Get("from")) + opts.To = parseTimeParam(q.Get("to")) + + // The default window only applies when the caller set no bounds itself. + if opts.From == 0 && opts.To == 0 { + switch w := strings.TrimSpace(q.Get("window")); w { + case "": + if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + case "all", "0": + // explicit opt-out + default: + if d, err := ParseWindow(w); err == nil && d > 0 { + opts.From = epochSeconds(time.Now().Add(-d)) + } else if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + } + } + + return opts +} + +// parseTimeParam reads a window bound: unix seconds (fraction allowed) or +// RFC3339. Returns 0 for an empty or unreadable value, which leaves the bound +// unset instead of failing the request. +func parseTimeParam(v string) float64 { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 { + return f + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return epochSeconds(t) + } + } + return 0 +} + +// ParseWindow is time.ParseDuration plus the "d" suffix for days, which Go does +// not support but is the most common unit in a UI ("7d"). +func ParseWindow(v string) (time.Duration, error) { + if strings.HasSuffix(v, "d") { + if n, err := strconv.ParseFloat(strings.TrimSuffix(v, "d"), 64); err == nil { + return time.Duration(n * float64(24*time.Hour)), nil + } + } + return time.ParseDuration(v) +} + +func epochSeconds(t time.Time) float64 { + return float64(t.UnixMicro()) / 1e6 +} + +// listMeta reports the applied limits, so a client can tell a truncated +// response from an exhausted one. +func listMeta(opts event.FindOptions, returned int) map[string]any { + meta := map[string]any{ + "limit": opts.Limit, + "offset": opts.Offset, + "returned": returned, + } + if opts.From > 0 { + meta["from"] = opts.From + } + if opts.To > 0 { + meta["to"] = opts.To + } + return meta +} + // RegisterAPI registers core API routes on the given mux. // authMiddleware wraps protected routes; pass a no-op when auth is disabled. -func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler) { +func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler, listLimits ListLimits) { // Public routes (no auth required). mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"version": version}) @@ -40,10 +166,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR // List events. protect("GET /api/events", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -52,15 +175,12 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR if events == nil { events = []event.Event{} } - writeJSON(w, map[string]any{"data": events, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": events, "meta": listMeta(opts, len(events))}) }) // List event previews. protect("GET /api/events/preview", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -70,7 +190,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR for _, ev := range events { result = append(result, previews.BuildPreview(ev)) } - writeJSON(w, map[string]any{"data": result, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": result, "meta": listMeta(opts, len(result))}) }) // Get single event. diff --git a/internal/server/http/api_test.go b/internal/server/http/api_test.go index 4310625..7a4f58a 100644 --- a/internal/server/http/api_test.go +++ b/internal/server/http/api_test.go @@ -5,8 +5,10 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + "time" "github.com/buggregator/go-buggregator/internal/event" "github.com/buggregator/go-buggregator/internal/module" @@ -16,6 +18,12 @@ import ( ) func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { + t.Helper() + // Empty limits keep the historical behaviour: return everything. + return setupAPILimits(t, serverhttp.ListLimits{}) +} + +func setupAPILimits(t *testing.T, limits serverhttp.ListLimits) (*http.ServeMux, *storage.SQLiteStore) { t.Helper() db, err := storage.Open(":memory:") if err != nil { @@ -43,7 +51,7 @@ func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { mux := http.NewServeMux() noopMiddleware := func(next http.Handler) http.Handler { return next } - serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware) + serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware, limits) return mux, store } @@ -259,3 +267,106 @@ func TestAPI_Projects(t *testing.T) { t.Errorf("key = %v", proj["key"]) } } + +func TestAPI_Events_Limits(t *testing.T) { + store := func(t *testing.T, mux *http.ServeMux, s *storage.SQLiteStore, n int, ts float64) { + t.Helper() + ctx := context.Background() + for i := 0; i < n; i++ { + if err := s.Store(ctx, event.Event{ + UUID: "uuid-" + strconv.Itoa(i) + "-" + strconv.FormatFloat(ts, 'f', 0, 64), + Type: "sentry", + Payload: json.RawMessage(`{"message":"error"}`), + Timestamp: ts + float64(i), + Project: "default", + }); err != nil { + t.Fatal(err) + } + } + } + list := func(t *testing.T, mux *http.ServeMux, query string) (int, map[string]any) { + t.Helper() + r := httptest.NewRequest("GET", "/api/events"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + + var resp struct { + Data []event.Event `json:"data"` + Meta map[string]any `json:"meta"` + } + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + return len(resp.Data), resp.Meta + } + + now := float64(time.Now().Unix()) + + t.Run("default limit caps the response", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, meta := list(t, mux, ""); n != 5 || meta["limit"] != float64(5) { + t.Fatalf("got %d events, meta %v; want 5", n, meta) + } + }) + + t.Run("explicit limit is capped by max_limit", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, _ := list(t, mux, "?limit=1000"); n != 10 { + t.Fatalf("got %d events, want 10 (max_limit)", n) + } + }) + + t.Run("default window hides old events", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 3, now-24*3600) // yesterday + store(t, mux, s, 2, now-60) // a minute ago + + if n, _ := list(t, mux, ""); n != 2 { + t.Fatalf("got %d events in the last hour, want 2", n) + } + // window=all opts out of the default window. + if n, _ := list(t, mux, "?window=all"); n != 5 { + t.Fatalf("got %d events for window=all, want 5", n) + } + // An explicit window overrides the default one. + if n, _ := list(t, mux, "?window=7d"); n != 5 { + t.Fatalf("got %d events for window=7d, want 5", n) + } + }) + + t.Run("explicit from/to bounds", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 4, 1700000000) // long before the default window + + from := strconv.FormatFloat(1700000000, 'f', 0, 64) + to := strconv.FormatFloat(1700000002, 'f', 0, 64) + if n, _ := list(t, mux, "?from="+from+"&to="+to); n != 3 { + t.Fatalf("got %d events for [%s..%s], want 3", n, from, to) + } + if n, _ := list(t, mux, "?from=2023-01-01"); n != 4 { + t.Fatalf("got %d events for an ISO from, want 4", n) + } + }) +} + +func TestParseWindow(t *testing.T) { + cases := map[string]time.Duration{ + "15m": 15 * time.Minute, + "24h": 24 * time.Hour, + "7d": 7 * 24 * time.Hour, + "1.5d": 36 * time.Hour, + } + for in, want := range cases { + got, err := serverhttp.ParseWindow(in) + if err != nil || got != want { + t.Errorf("ParseWindow(%q) = %v, %v; want %v", in, got, err, want) + } + } + if _, err := serverhttp.ParseWindow("yesterday"); err == nil { + t.Error("ParseWindow(\"yesterday\") should fail") + } +} diff --git a/internal/storage/sqlite.go b/internal/storage/sqlite.go index 6c13dda..4a214a5 100644 --- a/internal/storage/sqlite.go +++ b/internal/storage/sqlite.go @@ -62,6 +62,17 @@ func (s *SQLiteStore) FindAll(ctx context.Context, opts event.FindOptions) ([]ev conditions = append(conditions, "project = ?") args = append(args, opts.Project) } + // Window bounds are compared numerically: timestamp is stored as text + // (fmt.Sprintf("%.6f")), and a string comparison would break as soon as the + // number of digits in unix seconds changes. + if opts.From > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) >= ?") + args = append(args, opts.From) + } + if opts.To > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) <= ?") + args = append(args, opts.To) + } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") }