From f8d528f1192e96094188c9308e03504ecfb3d979 Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:42:19 +0700 Subject: [PATCH 1/3] feat(api): bound event list responses by size and time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/events and /api/events/preview returned every event stored for the requested type/project — FindOptions already had Limit and Offset, but the handlers never filled them in. On a busy instance one preview request weighed 25.8 MB, and the frontend then filtered that in the browser. Both endpoints now accept limit, offset/page, from, to and window (15m, 24h, "7d"; "all" opts out of the configured default), and report what was applied in the response meta so a client can tell a truncated response from an exhausted one. Defaults are deliberately conservative and non-breaking: 1000 events per response, no time window. An operator can set ui.default_window (or UI_DEFAULT_WINDOW) to also narrow lists by time. --- buggregator.yaml.example | 6 ++ internal/app/app.go | 29 ++++++- internal/app/config.go | 38 +++++++++ internal/event/store.go | 5 ++ internal/server/http/api.go | 142 ++++++++++++++++++++++++++++--- internal/server/http/api_test.go | 113 +++++++++++++++++++++++- internal/storage/sqlite.go | 11 +++ 7 files changed, 331 insertions(+), 13 deletions(-) diff --git a/buggregator.yaml.example b/buggregator.yaml.example index a2bbbc4..fe449f7 100644 --- a/buggregator.yaml.example +++ b/buggregator.yaml.example @@ -21,6 +21,12 @@ storage: mode: ${STORAGE_MODE:memory} # "memory" (default, lost on restart) or "filesystem" path: ${STORAGE_PATH:./storage} # Directory for filesystem mode +# UI event lists: how much /api/events and /api/events/preview return +ui: + default_limit: ${UI_DEFAULT_LIMIT:1000} # Events returned when the request has no limit param + max_limit: ${UI_MAX_LIMIT:5000} # Ceiling for an explicit ?limit= + default_window: ${UI_DEFAULT_WINDOW:} # Default time window, e.g. "1h", "24h", "7d". Empty = no window + # Prometheus metrics metrics: enabled: ${METRICS_ENABLED:false} # Set to true to expose Prometheus metrics diff --git a/internal/app/app.go b/internal/app/app.go index bb62682..13860dd 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "github.com/buggregator/go-buggregator/internal/auth" @@ -105,7 +106,7 @@ func (a *App) Run() { } // Register core API routes (settings endpoint is public, others go through auth middleware). - httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware) + httpserver.RegisterAPI(mux, store, a.registry.Previews(), eventService, a.cfg.Version, a.db, a.cfg.Modules.EnabledTypes(), authSettings, authMiddleware, a.listLimits()) // Register attachment API endpoints. httpserver.RegisterAttachmentAPI(mux, a.db, a.attachments) @@ -232,3 +233,29 @@ func (a *App) Run() { _ = srv.Shutdown(context.Background()) tcpManager.Wait() } + +// listLimits turns the ui config section into list limits. An unreadable +// default_window must not take the service down: the window is dropped and the +// reason is logged. +func (a *App) listLimits() httpserver.ListLimits { + lim := httpserver.ListLimits{ + DefaultLimit: a.cfg.UI.DefaultLimit, + MaxLimit: a.cfg.UI.MaxLimit, + } + + switch w := strings.TrimSpace(a.cfg.UI.DefaultWindow); w { + case "", "all", "0": + lim.DefaultWindow = 0 + default: + d, err := httpserver.ParseWindow(w) + if err != nil || d <= 0 { + slog.Warn("ui.default_window is not a duration, no time window applied", "value", w, "err", err) + d = 0 + } + lim.DefaultWindow = d + } + + slog.Info("event list limits", "default_limit", lim.DefaultLimit, + "max_limit", lim.MaxLimit, "default_window", lim.DefaultWindow) + return lim +} diff --git a/internal/app/config.go b/internal/app/config.go index b482656..e4b50b7 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "regexp" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -23,6 +24,17 @@ type AuthConfig struct { JWTSecret string `yaml:"jwt_secret"` // Secret for signing internal JWT tokens. Required when auth is enabled. } +// UIConfig bounds the event lists served to the UI. +// +// /api/events and /api/events/preview used to return everything stored for a +// project. The defaults below (1000 events, no time window) only cap the +// response size; set default_window to also narrow it by time, e.g. "24h". +type UIConfig struct { + DefaultLimit int `yaml:"default_limit"` // events returned without a limit param (default 1000) + MaxLimit int `yaml:"max_limit"` // ceiling for an explicit limit (default 5000) + DefaultWindow string `yaml:"default_window"` // time window without from/to: "24h", "7d"; empty or "all" = no window +} + // Config holds application configuration. type Config struct { Server ServerConfig `yaml:"server"` @@ -32,6 +44,7 @@ type Config struct { Metrics MetricsConfig `yaml:"metrics"` MCP MCPConfig `yaml:"mcp"` Auth AuthConfig `yaml:"auth"` + UI UIConfig `yaml:"ui"` Modules ModulesConfig `yaml:"modules"` Webhooks []WebhookDef `yaml:"webhooks"` Projects []ProjectDef `yaml:"projects"` @@ -207,6 +220,11 @@ func LoadConfig() Config { cfg.Auth.Scopes = coalesce(os.Getenv("AUTH_SCOPES"), fileCfg.Auth.Scopes, "openid,email,profile") cfg.Auth.JWTSecret = coalesce(os.Getenv("AUTH_JWT_SECRET"), fileCfg.Auth.JWTSecret) + // UI list limits. + cfg.UI.DefaultLimit = coalesceInt(atoiOrZero(os.Getenv("UI_DEFAULT_LIMIT")), fileCfg.UI.DefaultLimit, 1000) + cfg.UI.MaxLimit = coalesceInt(atoiOrZero(os.Getenv("UI_MAX_LIMIT")), fileCfg.UI.MaxLimit, 5000) + cfg.UI.DefaultWindow = coalesce(os.Getenv("UI_DEFAULT_WINDOW"), fileCfg.UI.DefaultWindow) + // CORS origins. cfg.Server.CORSOrigins = fileCfg.Server.CORSOrigins if env := os.Getenv("CORS_ORIGINS"); env != "" { @@ -318,6 +336,26 @@ func expandEnvVars(input string) string { }) } +// coalesceInt is coalesce for numbers: zero counts as "not set". +func coalesceInt(values ...int) int { + for _, v := range values { + if v > 0 { + return v + } + } + return 0 +} + +// atoiOrZero lets an env variable be passed to coalesceInt in one expression: a +// non-empty but non-numeric value counts as "not set". +func atoiOrZero(s string) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil || n < 0 { + return 0 + } + return n +} + func coalesce(values ...string) string { for _, v := range values { if v != "" { diff --git a/internal/event/store.go b/internal/event/store.go index 11d68cd..6573b97 100644 --- a/internal/event/store.go +++ b/internal/event/store.go @@ -8,6 +8,11 @@ type FindOptions struct { Project string Limit int Offset int + + // From and To bound the selection by event time (unix seconds with a + // fraction, same as Event.Timestamp). Zero means "no bound". + From float64 + To float64 } // DeleteOptions configures batch deletion. diff --git a/internal/server/http/api.go b/internal/server/http/api.go index 6f76954..163d2a4 100644 --- a/internal/server/http/api.go +++ b/internal/server/http/api.go @@ -4,6 +4,9 @@ import ( "database/sql" "encoding/json" "net/http" + "strconv" + "strings" + "time" "github.com/buggregator/go-buggregator/internal/event" ) @@ -14,9 +17,132 @@ type AuthSettings struct { LoginURL string } +// ListLimits bounds what /api/events and /api/events/preview return. +// +// Both endpoints used to return every event matching type/project: Limit and +// Offset existed in FindOptions but were never filled in. On a busy project +// that is a very large response — measured on a production instance, a single +// project preview weighed 25.8 MB — which the frontend then filters in the +// browser. +type ListLimits struct { + DefaultLimit int // how many events to return when limit is absent; 0 = unlimited + MaxLimit int // ceiling for an explicit limit; 0 = no ceiling + DefaultWindow time.Duration // time window applied when neither from/to nor window is given; 0 = no window +} + +// parseListOptions builds FindOptions from query parameters. +// +// type, project — unchanged; +// limit — how many events to return (capped by MaxLimit); +// offset / page — offset (page is counted from limit, 1-based); +// from, to — window bounds: unix seconds or RFC3339 ("2026-09-01T10:00:00Z"); +// window — window relative to now: "24h", "15m", "7d"; +// "all" or "0" opt out of DefaultWindow. +func parseListOptions(r *http.Request, lim ListLimits) event.FindOptions { + q := r.URL.Query() + opts := event.FindOptions{ + Type: q.Get("type"), + Project: q.Get("project"), + Limit: lim.DefaultLimit, + } + + if v := q.Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Limit = n + } + } + if lim.MaxLimit > 0 && opts.Limit > lim.MaxLimit { + opts.Limit = lim.MaxLimit + } + + if v := q.Get("offset"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.Offset = n + } + } else if v := q.Get("page"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 1 { + opts.Offset = (n - 1) * opts.Limit + } + } + + opts.From = parseTimeParam(q.Get("from")) + opts.To = parseTimeParam(q.Get("to")) + + // The default window only applies when the caller set no bounds itself. + if opts.From == 0 && opts.To == 0 { + switch w := strings.TrimSpace(q.Get("window")); w { + case "": + if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + case "all", "0": + // explicit opt-out + default: + if d, err := ParseWindow(w); err == nil && d > 0 { + opts.From = epochSeconds(time.Now().Add(-d)) + } else if lim.DefaultWindow > 0 { + opts.From = epochSeconds(time.Now().Add(-lim.DefaultWindow)) + } + } + } + + return opts +} + +// parseTimeParam reads a window bound: unix seconds (fraction allowed) or +// RFC3339. Returns 0 for an empty or unreadable value, which leaves the bound +// unset instead of failing the request. +func parseTimeParam(v string) float64 { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 { + return f + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return epochSeconds(t) + } + } + return 0 +} + +// ParseWindow is time.ParseDuration plus the "d" suffix for days, which Go does +// not support but is the most common unit in a UI ("7d"). +func ParseWindow(v string) (time.Duration, error) { + if strings.HasSuffix(v, "d") { + if n, err := strconv.ParseFloat(strings.TrimSuffix(v, "d"), 64); err == nil { + return time.Duration(n * float64(24*time.Hour)), nil + } + } + return time.ParseDuration(v) +} + +func epochSeconds(t time.Time) float64 { + return float64(t.UnixMicro()) / 1e6 +} + +// listMeta reports the applied limits, so a client can tell a truncated +// response from an exhausted one. +func listMeta(opts event.FindOptions, returned int) map[string]any { + meta := map[string]any{ + "limit": opts.Limit, + "offset": opts.Offset, + "returned": returned, + } + if opts.From > 0 { + meta["from"] = opts.From + } + if opts.To > 0 { + meta["to"] = opts.To + } + return meta +} + // RegisterAPI registers core API routes on the given mux. // authMiddleware wraps protected routes; pass a no-op when auth is disabled. -func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler) { +func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewRegistry, es *EventService, version string, db *sql.DB, enabledEvents []string, authSettings AuthSettings, authMiddleware func(http.Handler) http.Handler, listLimits ListLimits) { // Public routes (no auth required). mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"version": version}) @@ -40,10 +166,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR // List events. protect("GET /api/events", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -52,15 +175,12 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR if events == nil { events = []event.Event{} } - writeJSON(w, map[string]any{"data": events, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": events, "meta": listMeta(opts, len(events))}) }) // List event previews. protect("GET /api/events/preview", func(w http.ResponseWriter, r *http.Request) { - opts := event.FindOptions{ - Type: r.URL.Query().Get("type"), - Project: r.URL.Query().Get("project"), - } + opts := parseListOptions(r, listLimits) events, err := store.FindAll(r.Context(), opts) if err != nil { writeError(w, err.Error(), http.StatusInternalServerError) @@ -70,7 +190,7 @@ func RegisterAPI(mux *http.ServeMux, store event.Store, previews *event.PreviewR for _, ev := range events { result = append(result, previews.BuildPreview(ev)) } - writeJSON(w, map[string]any{"data": result, "meta": map[string]any{}}) + writeJSON(w, map[string]any{"data": result, "meta": listMeta(opts, len(result))}) }) // Get single event. diff --git a/internal/server/http/api_test.go b/internal/server/http/api_test.go index 4310625..7a4f58a 100644 --- a/internal/server/http/api_test.go +++ b/internal/server/http/api_test.go @@ -5,8 +5,10 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + "time" "github.com/buggregator/go-buggregator/internal/event" "github.com/buggregator/go-buggregator/internal/module" @@ -16,6 +18,12 @@ import ( ) func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { + t.Helper() + // Empty limits keep the historical behaviour: return everything. + return setupAPILimits(t, serverhttp.ListLimits{}) +} + +func setupAPILimits(t *testing.T, limits serverhttp.ListLimits) (*http.ServeMux, *storage.SQLiteStore) { t.Helper() db, err := storage.Open(":memory:") if err != nil { @@ -43,7 +51,7 @@ func setupAPI(t *testing.T) (*http.ServeMux, *storage.SQLiteStore) { mux := http.NewServeMux() noopMiddleware := func(next http.Handler) http.Handler { return next } - serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware) + serverhttp.RegisterAPI(mux, store, event.NewPreviewRegistry(), es, "test-version", db, []string{"sentry", "ray"}, serverhttp.AuthSettings{}, noopMiddleware, limits) return mux, store } @@ -259,3 +267,106 @@ func TestAPI_Projects(t *testing.T) { t.Errorf("key = %v", proj["key"]) } } + +func TestAPI_Events_Limits(t *testing.T) { + store := func(t *testing.T, mux *http.ServeMux, s *storage.SQLiteStore, n int, ts float64) { + t.Helper() + ctx := context.Background() + for i := 0; i < n; i++ { + if err := s.Store(ctx, event.Event{ + UUID: "uuid-" + strconv.Itoa(i) + "-" + strconv.FormatFloat(ts, 'f', 0, 64), + Type: "sentry", + Payload: json.RawMessage(`{"message":"error"}`), + Timestamp: ts + float64(i), + Project: "default", + }); err != nil { + t.Fatal(err) + } + } + } + list := func(t *testing.T, mux *http.ServeMux, query string) (int, map[string]any) { + t.Helper() + r := httptest.NewRequest("GET", "/api/events"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + + var resp struct { + Data []event.Event `json:"data"` + Meta map[string]any `json:"meta"` + } + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + return len(resp.Data), resp.Meta + } + + now := float64(time.Now().Unix()) + + t.Run("default limit caps the response", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, meta := list(t, mux, ""); n != 5 || meta["limit"] != float64(5) { + t.Fatalf("got %d events, meta %v; want 5", n, meta) + } + }) + + t.Run("explicit limit is capped by max_limit", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 5, MaxLimit: 10}) + store(t, mux, s, 12, now) + + if n, _ := list(t, mux, "?limit=1000"); n != 10 { + t.Fatalf("got %d events, want 10 (max_limit)", n) + } + }) + + t.Run("default window hides old events", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 3, now-24*3600) // yesterday + store(t, mux, s, 2, now-60) // a minute ago + + if n, _ := list(t, mux, ""); n != 2 { + t.Fatalf("got %d events in the last hour, want 2", n) + } + // window=all opts out of the default window. + if n, _ := list(t, mux, "?window=all"); n != 5 { + t.Fatalf("got %d events for window=all, want 5", n) + } + // An explicit window overrides the default one. + if n, _ := list(t, mux, "?window=7d"); n != 5 { + t.Fatalf("got %d events for window=7d, want 5", n) + } + }) + + t.Run("explicit from/to bounds", func(t *testing.T) { + mux, s := setupAPILimits(t, serverhttp.ListLimits{DefaultLimit: 100, DefaultWindow: time.Hour}) + store(t, mux, s, 4, 1700000000) // long before the default window + + from := strconv.FormatFloat(1700000000, 'f', 0, 64) + to := strconv.FormatFloat(1700000002, 'f', 0, 64) + if n, _ := list(t, mux, "?from="+from+"&to="+to); n != 3 { + t.Fatalf("got %d events for [%s..%s], want 3", n, from, to) + } + if n, _ := list(t, mux, "?from=2023-01-01"); n != 4 { + t.Fatalf("got %d events for an ISO from, want 4", n) + } + }) +} + +func TestParseWindow(t *testing.T) { + cases := map[string]time.Duration{ + "15m": 15 * time.Minute, + "24h": 24 * time.Hour, + "7d": 7 * 24 * time.Hour, + "1.5d": 36 * time.Hour, + } + for in, want := range cases { + got, err := serverhttp.ParseWindow(in) + if err != nil || got != want { + t.Errorf("ParseWindow(%q) = %v, %v; want %v", in, got, err, want) + } + } + if _, err := serverhttp.ParseWindow("yesterday"); err == nil { + t.Error("ParseWindow(\"yesterday\") should fail") + } +} diff --git a/internal/storage/sqlite.go b/internal/storage/sqlite.go index 6c13dda..4a214a5 100644 --- a/internal/storage/sqlite.go +++ b/internal/storage/sqlite.go @@ -62,6 +62,17 @@ func (s *SQLiteStore) FindAll(ctx context.Context, opts event.FindOptions) ([]ev conditions = append(conditions, "project = ?") args = append(args, opts.Project) } + // Window bounds are compared numerically: timestamp is stored as text + // (fmt.Sprintf("%.6f")), and a string comparison would break as soon as the + // number of digits in unix seconds changes. + if opts.From > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) >= ?") + args = append(args, opts.From) + } + if opts.To > 0 { + conditions = append(conditions, "CAST(timestamp AS REAL) <= ?") + args = append(args, opts.To) + } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } From 95a9ec7c65d22605b7892d2ee33a9e3ae338f5cf Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:45:01 +0700 Subject: [PATCH 2/3] feat(sentry): filter exceptions, traces and logs by project, environment and period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sentry endpoints could only be narrowed by level, handled and trace_id. project_id and environment are written for every error event but had no reader, so picking a project in the UI still listed every project's errors — including in the grouped view, which is what makes it useful on a shared instance. There was no way to narrow anything by time at all, so reaching a particular hour meant paging through the entire list. Adds: - project and environment filters on /api/sentry/exceptions (both the chronological and the grouped view); - environment and release filters on /api/sentry/traces (columns that were also written but never read); - from / to / window ("15m", "24h", "7d") on exceptions, traces, logs and counts. /api/sentry/counts now respects the same project and period as the lists it labels, and counts transactions rather than traces — that is what the traces list shows and the table that carries a timestamp to filter on. Filters only apply when the parameters are present, so existing clients see no change. --- modules/sentry/api.go | 77 ++++++++++++++++++++++++++ modules/sentry/api_exceptions.go | 22 ++++++++ modules/sentry/api_logs.go | 2 + modules/sentry/api_service_map.go | 21 ++++++- modules/sentry/api_test.go | 92 +++++++++++++++++++++++++++++++ modules/sentry/api_traces.go | 28 ++++++++-- 6 files changed, 235 insertions(+), 7 deletions(-) diff --git a/modules/sentry/api.go b/modules/sentry/api.go index 296e005..cf05669 100644 --- a/modules/sentry/api.go +++ b/modules/sentry/api.go @@ -5,6 +5,10 @@ import ( "encoding/json" "net/http" "strconv" + "strings" + "time" + + httpserver "github.com/buggregator/go-buggregator/internal/server/http" ) func registerAPI(mux *http.ServeMux, db *sql.DB) { @@ -41,6 +45,79 @@ func handleClearAll(db *sql.DB) http.HandlerFunc { } } +// timeWindow reads the period bounds from the query parameters. +// +// from, to — unix seconds or RFC3339 ("2026-09-01T10:00:00Z", "2026-09-01"); +// window — a window relative to now: "24h", "15m", "7d". +// +// A zero result means "unbounded". None of the sentry endpoints could be +// narrowed by time before, so on a stream of tens of thousands of events a day +// the only way to reach a particular hour was to page through the whole list. +func timeWindow(r *http.Request) (from, to time.Time) { + q := r.URL.Query() + from = parseWhen(q.Get("from")) + to = parseWhen(q.Get("to")) + + if from.IsZero() && to.IsZero() { + if w := strings.TrimSpace(q.Get("window")); w != "" && w != "all" && w != "0" { + if d, err := httpserver.ParseWindow(w); err == nil && d > 0 { + from = time.Now().UTC().Add(-d) + } + } + } + return from, to +} + +// parseWhen reads a single bound: unix seconds or one of the ISO forms. +func parseWhen(v string) time.Time { + v = strings.TrimSpace(v) + if v == "" { + return time.Time{} + } + if f, err := strconv.ParseFloat(v, 64); err == nil && f > 0 { + return time.Unix(int64(f), 0).UTC() + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// appendTimeConditions adds the period bounds to a WHERE clause. +// +// The stored format differs per column, hence the iso flag: +// - received_at, first_seen, last_seen — 'YYYY-MM-DD HH:MM:SS' (datetime('now')); +// - start_ts, end_ts of transactions and spans — ISO with T and Z. +// +// The comparison is done on strings: for both formats lexicographic order +// matches chronological order, and the indexes on these columns are textual. +func appendTimeConditions(conditions []string, args []any, r *http.Request, column string, iso bool) ([]string, []any) { + from, to := timeWindow(r) + layout := "2006-01-02 15:04:05" + if iso { + layout = "2006-01-02T15:04:05" + } + if !from.IsZero() { + conditions = append(conditions, column+" >= ?") + args = append(args, from.UTC().Format(layout)) + } + if !to.IsZero() { + conditions = append(conditions, column+" <= ?") + args = append(args, to.UTC().Format(layout)) + } + return conditions, args +} + +// whereOf builds a WHERE clause from conditions (empty string when there are none). +func whereOf(conditions []string) string { + if len(conditions) == 0 { + return "" + } + return " WHERE " + strings.Join(conditions, " AND ") +} + // pagination extracts page/limit from query params with defaults. func pagination(r *http.Request, defaultLimit int) (limit, offset int) { limit = defaultLimit diff --git a/modules/sentry/api_exceptions.go b/modules/sentry/api_exceptions.go index ff1d915..61f28d2 100644 --- a/modules/sentry/api_exceptions.go +++ b/modules/sentry/api_exceptions.go @@ -55,6 +55,14 @@ func handleExceptionsGrouped(db *sql.DB, w http.ResponseWriter, r *http.Request) conditions = append(conditions, "e.level = ?") args = append(args, v) } + if v := q.Get("environment"); v != "" { + conditions = append(conditions, "e.environment = ?") + args = append(args, v) + } + if v := q.Get("project"); v != "" { + conditions = append(conditions, "e.project_id = ?") + args = append(args, v) + } if v := q.Get("handled"); v != "" { if v == "true" { conditions = append(conditions, "e.handled = 1") @@ -63,6 +71,9 @@ func handleExceptionsGrouped(db *sql.DB, w http.ResponseWriter, r *http.Request) } } + // Period: from/to/window. received_at is covered by idx_sentry_errors_received_at. + conditions, args = appendTimeConditions(conditions, args, r, "e.received_at", false) + where := "" if len(conditions) > 0 { where = " WHERE " + strings.Join(conditions, " AND ") @@ -142,6 +153,14 @@ func handleExceptionsChronological(db *sql.DB, w http.ResponseWriter, r *http.Re conditions = append(conditions, "e.level = ?") args = append(args, v) } + if v := q.Get("environment"); v != "" { + conditions = append(conditions, "e.environment = ?") + args = append(args, v) + } + if v := q.Get("project"); v != "" { + conditions = append(conditions, "e.project_id = ?") + args = append(args, v) + } if v := q.Get("handled"); v != "" { if v == "true" { conditions = append(conditions, "e.handled = 1") @@ -150,6 +169,9 @@ func handleExceptionsChronological(db *sql.DB, w http.ResponseWriter, r *http.Re } } + // Period: from/to/window. received_at is covered by idx_sentry_errors_received_at. + conditions, args = appendTimeConditions(conditions, args, r, "e.received_at", false) + where := "" if len(conditions) > 0 { where = " WHERE " + strings.Join(conditions, " AND ") diff --git a/modules/sentry/api_logs.go b/modules/sentry/api_logs.go index a24fb65..f0704c3 100644 --- a/modules/sentry/api_logs.go +++ b/modules/sentry/api_logs.go @@ -30,6 +30,8 @@ func handleLogsList(db *sql.DB) http.HandlerFunc { conditions = append(conditions, "l.trace_id = ?") args = append(args, v) } + // Period: log_ts is stored in ISO form and covered by idx_sentry_logs_ts. + conditions, args = appendTimeConditions(conditions, args, r, "l.log_ts", true) where := "" if len(conditions) > 0 { diff --git a/modules/sentry/api_service_map.go b/modules/sentry/api_service_map.go index e750287..cf52338 100644 --- a/modules/sentry/api_service_map.go +++ b/modules/sentry/api_service_map.go @@ -133,9 +133,24 @@ func labelForNode(address, opType string) string { func handleCounts(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var exceptions, traces, logs int - db.QueryRow(`SELECT COUNT(*) FROM sentry_error_events`).Scan(&exceptions) - db.QueryRow(`SELECT COUNT(*) FROM sentry_traces`).Scan(&traces) - db.QueryRow(`SELECT COUNT(*) FROM sentry_logs`).Scan(&logs) + // The counters follow the same project and period as the lists they label. + // They used to count the whole database, so the project selected in the UI + // had no effect on the numbers next to the tabs. + var errConds, txnConds, logConds []string + var errArgs, txnArgs, logArgs []any + errConds, errArgs = appendTimeConditions(errConds, errArgs, r, "received_at", false) + txnConds, txnArgs = appendTimeConditions(txnConds, txnArgs, r, "start_ts", true) + logConds, logArgs = appendTimeConditions(logConds, logArgs, r, "log_ts", true) + if v := r.URL.Query().Get("project"); v != "" { + errConds = append(errConds, "project_id = ?") + errArgs = append(errArgs, v) + } + + db.QueryRow(`SELECT COUNT(*) FROM sentry_error_events`+whereOf(errConds), errArgs...).Scan(&exceptions) + // Counted over sentry_transactions rather than sentry_traces: that is what + // the traces list shows, and it is the table carrying a timestamp to filter on. + db.QueryRow(`SELECT COUNT(*) FROM sentry_transactions`+whereOf(txnConds), txnArgs...).Scan(&traces) + db.QueryRow(`SELECT COUNT(*) FROM sentry_logs`+whereOf(logConds), logArgs...).Scan(&logs) apiJSON(w, map[string]any{ "exceptions": exceptions, diff --git a/modules/sentry/api_test.go b/modules/sentry/api_test.go index bec6cd7..45c08d1 100644 --- a/modules/sentry/api_test.go +++ b/modules/sentry/api_test.go @@ -463,3 +463,95 @@ func TestAPIEmptyDatabase(t *testing.T) { }) } } + +// The exceptions list could not be narrowed to a project: the column is filled +// in on write, but the handler had no reader for it, so selecting a project in +// the UI showed every project's errors. Same for environment, and for the +// period there was no filter at all. +func TestAPIExceptionsFilters(t *testing.T) { + db := setupTestDB(t) + mux := http.NewServeMux() + registerAPI(mux, db) + + seed := func(eventID, env, project string) { + t.Helper() + payload, _ := json.Marshal(map[string]any{ + "event_id": eventID, + "level": "error", + "environment": env, + "exception": map[string]any{"values": []map[string]any{{"type": "RuntimeException", "value": eventID}}}, + }) + var ev ErrorEvent + if err := json.Unmarshal(payload, &ev); err != nil { + t.Fatal(err) + } + if _, err := storeErrorEvent(db, &ev, payload, project); err != nil { + t.Fatalf("seed %s: %v", eventID, err) + } + } + seed("evt-prod-a", "production", "shop") + seed("evt-prod-b", "production", "shop") + seed("evt-stage", "staging", "shop") + seed("evt-other", "production", "billing") + + count := func(query string) int { + t.Helper() + req := httptest.NewRequest("GET", "/api/sentry/exceptions"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("status = %d for %q", w.Code, query) + } + var resp map[string]any + json.Unmarshal(w.Body.Bytes(), &resp) + return len(resp["data"].([]any)) + } + + cases := map[string]struct { + query string + want int + }{ + "no filters": {"", 4}, + "by project": {"?project=shop", 3}, + "by environment": {"?environment=staging", 1}, + "project and env": {"?project=shop&environment=production", 2}, + "grouped by project": {"?grouped=true&project=billing", 1}, + "within window": {"?window=24h", 4}, + "outside window": {"?from=2030-01-01", 0}, + "grouped in window": {"?grouped=true&from=2030-01-01", 0}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if got := count(tc.query); got != tc.want { + t.Errorf("%q returned %d events, want %d", tc.query, got, tc.want) + } + }) + } +} + +// The tab counters counted the whole database, so they disagreed with the lists +// they label as soon as a project or a period was selected. +func TestAPICountsFollowFilters(t *testing.T) { + mux := http.NewServeMux() + seedTestData(t, mux) + + counts := func(query string) map[string]any { + t.Helper() + req := httptest.NewRequest("GET", "/api/sentry/counts"+query, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + var resp map[string]any + json.Unmarshal(w.Body.Bytes(), &resp) + return resp + } + + if got := counts("?project=default")["exceptions"]; int(got.(float64)) != 3 { + t.Errorf("exceptions for the seeded project = %v, want 3", got) + } + if got := counts("?project=nonexistent")["exceptions"]; int(got.(float64)) != 0 { + t.Errorf("exceptions for an unknown project = %v, want 0", got) + } + if got := counts("?from=2030-01-01")["exceptions"]; int(got.(float64)) != 0 { + t.Errorf("exceptions in a future window = %v, want 0", got) + } +} diff --git a/modules/sentry/api_traces.go b/modules/sentry/api_traces.go index a0a4d0a..a4e2a35 100644 --- a/modules/sentry/api_traces.go +++ b/modules/sentry/api_traces.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "net/http" + "strings" "time" ) @@ -11,17 +12,36 @@ func handleTracesList(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { limit, offset := pagination(r, 50) - countQuery := `SELECT COUNT(*) FROM sentry_transactions` + // Period: start_ts of a transaction is in ISO form, index idx_sentry_txn_start. + // Plus environment and release filters: both columns are populated but had + // no reader. + var conditions []string + var args []any + conditions, args = appendTimeConditions(conditions, args, r, "t.start_ts", true) + if v := r.URL.Query().Get("environment"); v != "" { + conditions = append(conditions, "t.environment = ?") + args = append(args, v) + } + if v := r.URL.Query().Get("release"); v != "" { + conditions = append(conditions, "t.release = ?") + args = append(args, v) + } + where := "" + if len(conditions) > 0 { + where = " WHERE " + strings.Join(conditions, " AND ") + } + var total int - db.QueryRow(countQuery).Scan(&total) + db.QueryRow(`SELECT COUNT(*) FROM sentry_transactions t`+where, args...).Scan(&total) + queryArgs := append(append([]any{}, args...), limit, offset) rows, err := db.Query( `SELECT t.trace_id, t.id, t.transaction_name, t.op, t.status, t.duration_ms, tr.span_count, tr.error_count, t.start_ts FROM sentry_transactions t - JOIN sentry_traces tr ON tr.trace_id = t.trace_id + JOIN sentry_traces tr ON tr.trace_id = t.trace_id`+where+` ORDER BY t.start_ts DESC - LIMIT ? OFFSET ?`, limit, offset, + LIMIT ? OFFSET ?`, queryArgs..., ) if err != nil { apiError(w, err.Error(), http.StatusInternalServerError) From 79446b797e2c96da00e9e5be1f8ef0fd6fadf94d Mon Sep 17 00:00:00 2001 From: tsan88 Date: Fri, 4 Sep 2026 15:46:03 +0700 Subject: [PATCH 3/3] fix(sentry): return timestamps with an explicit UTC zone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit received_at, first_seen and last_seen are written with datetime('now') — UTC, but with no zone marker: "2026-09-04 05:33:12". The frontend parses that with moment(str), which treats a zone-less string as local time, so every timestamp in the UI was shifted by the viewer's UTC offset. In UTC+3 the grouped view showed a error that had just arrived as "3 hours ago", and "last seen" could even read as being in the future for negative offsets. The columns are now formatted as ...T...Z on the way out. Sorting and filtering keep using the raw column, so no index is bypassed. --- modules/sentry/api.go | 14 +++++++++++ modules/sentry/api_exceptions.go | 8 +++---- modules/sentry/api_test.go | 41 ++++++++++++++++++++++++++++++++ modules/sentry/api_traces.go | 2 +- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/modules/sentry/api.go b/modules/sentry/api.go index cf05669..77c6815 100644 --- a/modules/sentry/api.go +++ b/modules/sentry/api.go @@ -110,6 +110,20 @@ func appendTimeConditions(conditions []string, args []any, r *http.Request, colu return conditions, args } +// tsUTC wraps a time column so the value leaves the API as an ISO string with +// an explicit zone: 2026-09-04T05:33:12Z. +// +// received_at, first_seen and last_seen are written with datetime('now'), which +// is UTC but carries **no zone marker**: "2026-09-04 05:33:12". A browser reads +// such a string as local time, so "last seen" in the UI was off by the viewer's +// UTC offset — in UTC+3 a fresh error showed up as "3 hours ago". Sorting and +// filtering still use the raw column; only the representation changes. +func tsUTC(column string) string { + // COALESCE covers a value strftime cannot parse (empty string, garbage): + // the original value goes out instead of NULL. + return "COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ', " + column + "), " + column + ")" +} + // whereOf builds a WHERE clause from conditions (empty string when there are none). func whereOf(conditions []string) string { if len(conditions) == 0 { diff --git a/modules/sentry/api_exceptions.go b/modules/sentry/api_exceptions.go index 61f28d2..7b43538 100644 --- a/modules/sentry/api_exceptions.go +++ b/modules/sentry/api_exceptions.go @@ -30,8 +30,8 @@ func handleExceptionsGrouped(db *sql.DB, w http.ResponseWriter, r *http.Request) se.exception_type, se.exception_value, g.count, - g.first_seen, - g.last_seen, + ` + tsUTC("g.first_seen") + ` as first_seen, + ` + tsUTC("g.last_seen") + ` as last_seen, g.level, g.handled, g.sample_event_id @@ -141,7 +141,7 @@ func handleExceptionsChronological(db *sql.DB, w http.ResponseWriter, r *http.Re (SELECT COUNT(*) FROM sentry_error_events e2 WHERE e2.fingerprint = e.fingerprint) as occurrence_count, (SELECT se.exception_type FROM sentry_exceptions se WHERE se.error_event_id = e.id AND se.position = 0 LIMIT 1) as exception_type, (SELECT se.exception_value FROM sentry_exceptions se WHERE se.error_event_id = e.id AND se.position = 0 LIMIT 1) as exception_value, - e.level, e.handled, e."transaction", e.received_at, e.trace_id + e.level, e.handled, e."transaction", ` + tsUTC("e.received_at") + ` as received_at, e.trace_id FROM sentry_error_events e` countQuery := `SELECT COUNT(*) FROM sentry_error_events e` @@ -245,7 +245,7 @@ func handleExceptionDetail(db *sql.DB) http.HandlerFunc { ) err := db.QueryRow( `SELECT id, event_id, fingerprint, level, handled, platform, environment, server_name, - "transaction", release, trace_id, span_id, received_at, event_ts, payload + "transaction", release, trace_id, span_id, ` + tsUTC("received_at") + ` as received_at, event_ts, payload FROM sentry_error_events WHERE id = ? OR event_id = ? ORDER BY (id = ?) DESC LIMIT 1`, id, id, id, ).Scan(&internalID, &eventID, &fingerprint, &level, &handled, &platform, &environment, &serverName, &txn, &release, &traceID, &spanID, &receivedAt, &eventTS, &payloadStr) diff --git a/modules/sentry/api_test.go b/modules/sentry/api_test.go index 45c08d1..a153fdc 100644 --- a/modules/sentry/api_test.go +++ b/modules/sentry/api_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -555,3 +556,43 @@ func TestAPICountsFollowFilters(t *testing.T) { t.Errorf("exceptions in a future window = %v, want 0", got) } } + +// received_at / first_seen / last_seen are stored as datetime('now'), i.e. UTC +// without a zone marker. A browser parses such a string as local time, so +// "last seen" in the UI was off by the viewer's UTC offset. The API must hand +// out an explicit zone. +func TestAPIExceptionsTimestampsAreUTC(t *testing.T) { + mux := http.NewServeMux() + seedTestData(t, mux) + + get := func(path string) map[string]any { + t.Helper() + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("status = %d for %s", w.Code, path) + } + var resp map[string]any + json.Unmarshal(w.Body.Bytes(), &resp) + return resp + } + + assertZoned := func(what string, v any) { + t.Helper() + s, ok := v.(string) + if !ok || !strings.HasSuffix(s, "Z") || !strings.Contains(s, "T") { + t.Errorf("%s = %v, want an ISO 8601 string with a zone (…T…Z)", what, v) + } + } + + grouped := get("/api/sentry/exceptions?grouped=true")["data"].([]any)[0].(map[string]any) + assertZoned("grouped first_seen", grouped["first_seen"]) + assertZoned("grouped last_seen", grouped["last_seen"]) + + chronological := get("/api/sentry/exceptions")["data"].([]any)[0].(map[string]any) + assertZoned("received_at", chronological["received_at"]) + + detail := get("/api/sentry/exceptions/" + chronological["event_id"].(string)) + assertZoned("detail received_at", detail["received_at"]) +} diff --git a/modules/sentry/api_traces.go b/modules/sentry/api_traces.go index a4e2a35..bbd45f1 100644 --- a/modules/sentry/api_traces.go +++ b/modules/sentry/api_traces.go @@ -200,7 +200,7 @@ func loadAllSpans(db *sql.DB, traceID string, txnStartTS sql.NullString) []map[s func loadRelatedErrors(db *sql.DB, traceID string) []map[string]any { // Use a single query to avoid nested connection issues with SQLite. rows, err := db.Query( - `SELECT e.event_id, se.exception_type, e.received_at + `SELECT e.event_id, se.exception_type, ` + tsUTC("e.received_at") + ` FROM sentry_error_events e LEFT JOIN sentry_exceptions se ON se.error_event_id = e.id AND se.position = 0 WHERE e.trace_id = ? ORDER BY e.received_at DESC`, traceID,