From 0c703b61b18851f9aa95b2c933a1bf9a2360d224 Mon Sep 17 00:00:00 2001 From: Gene Date: Tue, 14 Apr 2026 11:59:05 +0300 Subject: [PATCH 1/4] Update README.md Mention article where Tinode is claimed to be the backend for the Russian National Messenger Max --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9f9db763b..e9dc28db5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Tinode is *not* XMPP/Jabber. It is *not* compatible with XMPP. It's meant as a r +[Accorting to creators](https://habr.com/ru/articles/1022914/), Tinode is used as a backend for the [Russian National Messenger Max](https://en.wikipedia.org/wiki/Max_(app)), claiming active users exceeding 40 million. + ## Why? The promise of [XMPP](http://xmpp.org/) was to deliver federated instant messaging: anyone would be able to spin up an IM server capable of exchanging messages with any other XMPP server in the world. Unfortunately, XMPP never delivered on this promise. Instant messengers are still a bunch of incompatible walled gardens, similar to what AoL of the late 1990s was to the open Internet. From 9e3d9083737904be6c9f0688ef4dbb6136740e47 Mon Sep 17 00:00:00 2001 From: Gene Date: Tue, 14 Apr 2026 12:05:07 +0300 Subject: [PATCH 2/4] Update README.md Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9dc28db5..1c02cc170 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Tinode is *not* XMPP/Jabber. It is *not* compatible with XMPP. It's meant as a r -[Accorting to creators](https://habr.com/ru/articles/1022914/), Tinode is used as a backend for the [Russian National Messenger Max](https://en.wikipedia.org/wiki/Max_(app)), claiming active users exceeding 40 million. +[According to its creators](https://habr.com/ru/articles/1022914/), Tinode is used as a backend for the [Russian National Messenger Max](https://en.wikipedia.org/wiki/Max_(app)), claiming active users exceeding 40 million. ## Why? From 985f11f78b88c0a6a3254cf372de92022a1fac75 Mon Sep 17 00:00:00 2001 From: Gene Date: Tue, 14 Apr 2026 12:43:32 +0300 Subject: [PATCH 3/4] Update README.md Remove text added by mistake --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 1c02cc170..9f9db763b 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ Tinode is *not* XMPP/Jabber. It is *not* compatible with XMPP. It's meant as a r -[According to its creators](https://habr.com/ru/articles/1022914/), Tinode is used as a backend for the [Russian National Messenger Max](https://en.wikipedia.org/wiki/Max_(app)), claiming active users exceeding 40 million. - ## Why? The promise of [XMPP](http://xmpp.org/) was to deliver federated instant messaging: anyone would be able to spin up an IM server capable of exchanging messages with any other XMPP server in the world. Unfortunately, XMPP never delivered on this promise. Instant messengers are still a bunch of incompatible walled gardens, similar to what AoL of the late 1990s was to the open Internet. From 1628223f2f673dbe2f1d75594f6e3c92407600ec Mon Sep 17 00:00:00 2001 From: Ericson Muros Date: Sun, 31 Aug 2025 12:24:09 +0800 Subject: [PATCH 4/4] feature/integrate gcs kms bucket for storing images Overview: - KMS-protected buckets gives full control over encryption keys (vs Google-managed keys), granular permissions: You can restrict access to specific keys per team or service, and rotation & auditability: You can rotate keys periodically and audit access using Cloud Logging. --- server/main.go | 1 + server/media/gcs/README.md | 138 ++++++++++++++ server/media/gcs/gcs.go | 360 +++++++++++++++++++++++++++++++++++++ server/tinode.conf | 22 +++ 4 files changed, 521 insertions(+) create mode 100644 server/media/gcs/README.md create mode 100644 server/media/gcs/gcs.go diff --git a/server/main.go b/server/main.go index c8d24f523..78ecc0a1c 100644 --- a/server/main.go +++ b/server/main.go @@ -57,6 +57,7 @@ import ( // File upload handlers _ "github.com/tinode/chat/server/media/fs" + _ "github.com/tinode/chat/server/media/gcs" _ "github.com/tinode/chat/server/media/s3" ) diff --git a/server/media/gcs/README.md b/server/media/gcs/README.md new file mode 100644 index 000000000..eb9a2d597 --- /dev/null +++ b/server/media/gcs/README.md @@ -0,0 +1,138 @@ +# GCS Media Handler for Tinode + +This package implements the Tinode media handler interface for Google Cloud Storage with KMS encryption. + +## Features + +- **KMS Encryption**: All uploaded files are automatically encrypted using Google Cloud KMS +- **Automatic File Management**: Handles upload, download, and deletion of files +- **CORS Support**: Configurable CORS headers for web applications +- **Cache Control**: Configurable cache headers for optimal performance +- **Base Path Support**: Optional base path prefix for organizing files in buckets + +## Configuration + +Add the following configuration to your Tinode server config: + +```json +{ + "media": { + "use_handler": "gcs", + "handlers":{ + "gcs": { + "bucket_name": "your-bucket-name", + "serve_url": "/v0/file/s/", + "cors_origins": ["*"], + "cache_control": "max-age=86400", + "project_id": "your-gcp-project-id", + "location_id": "your-kms-location", + "key_ring_id": "your-key-ring-id", + "key_id": "your-crypto-key-id", + "base_path": "tinode-files", + "client_timeout": "30s" + } + } + } +} +``` + +### Configuration Parameters + +- `bucket_name` (required): The GCS bucket name where files will be stored +- `serve_url` (optional): URL path prefix for serving files (default: `/v0/file/s/`) +- `cors_origins` (optional): List of allowed CORS origins (default: `["*"]`) +- `cache_control` (optional): Cache control header value (default: `max-age=86400`) +- `project_id` (required): Your Google Cloud Project ID +- `location_id` (required): The location of your KMS key ring +- `key_ring_id` (required): The ID of your KMS key ring +- `key_id` (required): The ID of your KMS crypto key +- `base_path` (optional): Base path prefix for organizing files in the bucket +- `client_timeout` (optional): Timeout for GCS operations (default: 30s) + +## Setup Requirements + +1. **Google Cloud Storage**: Create a bucket for storing files +2. **Google Cloud KMS**: Set up a key ring and crypto key for encryption +3. **Service Account**: Create a service account with the following roles: + - `Storage Object Admin` for bucket operations + - `Cloud KMS CryptoKey Encrypter/Decrypter` for KMS operations + +4. **Authentication**: Ensure your application has proper authentication: + - Use Application Default Credentials (ADC) + - Or set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable + +## Usage + +The handler automatically registers itself with Tinode's media handler registry. Once configured, Tinode will use this handler for all file operations. + +### File Upload + +Files are uploaded with the following structure: +- Object name: `{file-id}{extension}` (e.g., `abc123def456.jpg`) +- If base_path is configured: `{base-path}/{file-id}{extension}` (e.g., `tinode-files/abc123def456.jpg`) + +### File Download + +Files are served through Tinode's standard file serving endpoints with proper headers and caching. + +### File Deletion + +When files are no longer needed, they are automatically deleted from GCS. + +## Security + +- All files are encrypted at rest using Google Cloud KMS +- Files are served through Tinode's authentication system +- CORS headers are configurable for web security +- No direct public access to GCS bucket (all access through Tinode) + +## Performance Considerations + +- Files are served through Tinode's HTTP endpoints, not directly from GCS +- Consider using a CDN in front of Tinode for better performance +- The handler doesn't support seeking in files (GCS limitation) +- Large files may take time to upload/download + +## Troubleshooting + +1. **Authentication Errors**: Ensure your service account has proper permissions +2. **KMS Errors**: Verify your KMS key ring and crypto key exist and are accessible +3. **Bucket Errors**: Ensure the bucket exists and is accessible +4. **Timeout Errors**: Increase the `client_timeout` value for slow connections + +## Example Configuration + +```json +{ + "media": { + "use_handler": "gcs", + "handlers":{ + "gcs": { + "bucket_name": "my-tinode-files", + "serve_url": "/v0/file/s/", + "cors_origins": ["https://myapp.com", "https://www.myapp.com"], + "cache_control": "max-age=3600", + "project_id": "my-gcp-project", + "location_id": "us-central1", + "key_ring_id": "tinode-keyring", + "key_id": "tinode-crypto-key", + "base_path": "uploads", + "client_timeout": "60s" + } + } + } +} +``` + +## Implementation Details + +The GCS media handler implements the complete Tinode media interface: + +- **Init()**: Initializes the GCS client and validates KMS configuration +- **Headers()**: Handles CORS headers and cache management +- **Upload()**: Uploads files to GCS with KMS encryption +- **Download()**: Downloads files from GCS through Tinode's HTTP endpoints +- **Delete()**: Deletes files from GCS when no longer needed +- **GetIdFromUrl()**: Extracts file IDs from Tinode URLs + +The handler integrates directly with Google Cloud Storage and uses the official Go client library for all operations. diff --git a/server/media/gcs/gcs.go b/server/media/gcs/gcs.go new file mode 100644 index 000000000..4977540d3 --- /dev/null +++ b/server/media/gcs/gcs.go @@ -0,0 +1,360 @@ +// Package gcs implements github.com/tinode/chat/server/media interface by storing media objects in Google Cloud Storage. +package gcs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strings" + "time" + + "cloud.google.com/go/storage" + + "github.com/tinode/chat/server/logs" + "github.com/tinode/chat/server/media" + "github.com/tinode/chat/server/store" + "github.com/tinode/chat/server/store/types" +) + +const ( + defaultServeURL = "/v0/file/s/" + defaultCacheControl = "max-age=86400" + handlerName = "gcs" +) + +type configType struct { + BucketName string `json:"bucket_name"` + ServeURL string `json:"serve_url"` + CorsOrigins []string `json:"cors_origins"` + CacheControl string `json:"cache_control"` + ProjectID string `json:"project_id"` + LocationID string `json:"location_id"` + KeyRingID string `json:"key_ring_id"` + KeyID string `json:"key_id"` + BasePath string `json:"base_path"` + ClientTimeout *time.Duration `json:"client_timeout"` +} + +type gcshandler struct { + bucketName string + serveURL string + corsOrigins []string + cacheControl string + projectID string + locationID string + keyRingID string + keyID string + basePath string + client *storage.Client +} + +func (gh *gcshandler) Init(jsconf string) error { + var err error + var config configType + + if err = json.Unmarshal([]byte(jsconf), &config); err != nil { + return errors.New("failed to parse config: " + err.Error()) + } + + gh.bucketName = config.BucketName + if gh.bucketName == "" { + return errors.New("missing bucket name") + } + + gh.serveURL = config.ServeURL + if gh.serveURL == "" { + gh.serveURL = defaultServeURL + } + + gh.cacheControl = config.CacheControl + if gh.cacheControl == "" { + gh.cacheControl = defaultCacheControl + } + + gh.corsOrigins = config.CorsOrigins + gh.projectID = config.ProjectID + gh.locationID = config.LocationID + gh.keyRingID = config.KeyRingID + gh.keyID = config.KeyID + gh.basePath = config.BasePath + + // Validate KMS configuration + if gh.projectID == "" || gh.locationID == "" || gh.keyRingID == "" || gh.keyID == "" { + return errors.New("missing required KMS configuration (project_id, location_id, key_ring_id, key_id)") + } + + // Initialize GCS client + ctx := context.Background() + gh.client, err = storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create storage client: %w", err) + } + + return nil +} + +// Headers is used for cache management and serving CORS headers. +func (gh *gcshandler) Headers(method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) { + if method == http.MethodGet { + fid := gh.GetIdFromUrl(url.String()) + if fid.IsZero() { + return nil, 0, types.ErrNotFound + } + + fdef, err := gh.getFileRecord(fid) + if err != nil { + return nil, 0, err + } + + if etag := strings.Trim(headers.Get("If-None-Match"), "\""); etag != "" && etag == fdef.ETag { + // Create headers with CORS support + responseHeaders := http.Header{ + "Last-Modified": {fdef.UpdatedAt.Format(http.TimeFormat)}, + "ETag": {`"` + fdef.ETag + `"`}, + "Cache-Control": {gh.cacheControl}, + } + + // Add CORS headers for all responses + responseHeaders.Set("Access-Control-Allow-Origin", "*") + responseHeaders.Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS") + responseHeaders.Set("Access-Control-Allow-Headers", "*") + + return responseHeaders, http.StatusNotModified, nil + } + + // Create headers with CORS support + responseHeaders := http.Header{ + "Content-Type": {fdef.MimeType}, + "Cache-Control": {gh.cacheControl}, + "ETag": {`"` + fdef.ETag + `"`}, + } + + // Add CORS headers for all responses + responseHeaders.Set("Access-Control-Allow-Origin", "*") + responseHeaders.Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS") + responseHeaders.Set("Access-Control-Allow-Headers", "*") + + return responseHeaders, 0, nil + } + + if method != http.MethodOptions { + // Not an OPTIONS request. No special handling for all other requests. + return nil, 0, nil + } + + // Ensure CORS origins are set, default to "*" if not configured + corsOrigins := gh.corsOrigins + if len(corsOrigins) == 0 { + corsOrigins = []string{"*"} + } + + header, status := media.CORSHandler(method, headers, corsOrigins, serve) + + // Ensure Tinode-specific headers are allowed for preflight requests + if method == http.MethodOptions && status == http.StatusNoContent { + if header == nil { + header = http.Header{} + } + // The media.CORSHandler already sets Access-Control-Allow-Headers to "*" + // but let's be explicit about Tinode headers for clarity + if header.Get("Access-Control-Allow-Headers") == "*" { + // Keep the wildcard, it's correct + } else { + // Fallback to explicit headers if needed + header.Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-Tinode-APIKey, X-Tinode-Auth") + } + } + + return header, status, nil +} + +// Upload processes request for file upload. The file is given as io.Reader. +func (gh *gcshandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) { + // Generate object name using file ID + objectName := fdef.Uid().String32() + + // Add file extension if available + ext, _ := mime.ExtensionsByType(fdef.MimeType) + if len(ext) > 0 { + objectName += ext[0] + } + + // Add base path if configured + if gh.basePath != "" { + objectName = gh.basePath + "/" + objectName + } + + // Start upload record in database + if err := store.Files.StartUpload(fdef); err != nil { + logs.Warn.Println("failed to create file record", fdef.Id, err) + return "", 0, err + } + + // Upload to GCS with KMS encryption + ctx := context.Background() + err := gh.uploadFileWithKMS(ctx, gh.bucketName, objectName, file) + if err != nil { + logs.Warn.Println("Upload: failed to upload file to GCS", objectName, err) + return "", 0, err + } + + // Get file size + size, err := gh.getFileSize(ctx, gh.bucketName, objectName) + if err != nil { + logs.Warn.Println("Upload: failed to get file size", objectName, err) + return "", 0, err + } + + // Generate ETag (using object name as it's unique) + fdef.Location = objectName + fdef.ETag = etagFromObjectName(objectName) + + // Generate serve URL + serveURL := gh.serveURL + fdef.Id + if len(ext) > 0 { + serveURL += ext[0] + } + + return serveURL, size, nil +} + +// Download processes request for file download. +// The returned ReadSeekCloser must be closed after use. +func (gh *gcshandler) Download(url string) (*types.FileDef, media.ReadSeekCloser, error) { + fid := gh.GetIdFromUrl(url) + if fid.IsZero() { + return nil, nil, types.ErrNotFound + } + + fd, err := gh.getFileRecord(fid) + if err != nil { + logs.Warn.Println("Download: file not found", fid) + return nil, nil, err + } + + // Download from GCS + ctx := context.Background() + reader, err := gh.downloadFile(ctx, gh.bucketName, fd.Location) + if err != nil { + logs.Warn.Println("Download: failed to download from GCS", fd.Location, err) + return nil, nil, err + } + + // Create a ReadSeekCloser wrapper + readSeekCloser := &gcsReadSeekCloser{ + reader: reader, + close: func() error { return nil }, // GCS reader doesn't need explicit close + } + + return fd, readSeekCloser, nil +} + +// Delete deletes files from storage by provided slice of locations. +func (gh *gcshandler) Delete(locations []string) error { + ctx := context.Background() + for _, loc := range locations { + if err := gh.deleteFile(ctx, gh.bucketName, loc); err != nil { + logs.Warn.Println("gcs: error deleting file", loc, err) + } + } + return nil +} + +// GetIdFromUrl converts an attachment URL to a file UID. +func (gh *gcshandler) GetIdFromUrl(url string) types.Uid { + return media.GetIdFromUrl(url, gh.serveURL) +} + +// getFileRecord given file ID reads file record from the database. +func (gh *gcshandler) getFileRecord(fid types.Uid) (*types.FileDef, error) { + fd, err := store.Files.Get(fid.String()) + if err != nil { + return nil, err + } + if fd == nil { + return nil, types.ErrNotFound + } + return fd, nil +} + +// GCS helper methods +func (gh *gcshandler) uploadFileWithKMS(ctx context.Context, bucketName, objectName string, file io.Reader) error { + kmsKeyName := fmt.Sprintf( + "projects/%s/locations/%s/keyRings/%s/cryptoKeys/%s", + gh.projectID, + gh.locationID, + gh.keyRingID, + gh.keyID, + ) + + obj := gh.client.Bucket(bucketName).Object(objectName) + w := obj.NewWriter(ctx) + w.KMSKeyName = kmsKeyName + defer w.Close() + + if _, err := io.Copy(w, file); err != nil { + return fmt.Errorf("failed to upload file with KMS encryption: %w", err) + } + + return nil +} + +func (gh *gcshandler) downloadFile(ctx context.Context, bucketName, objectName string) (io.Reader, error) { + bkt := gh.client.Bucket(bucketName) + obj := bkt.Object(objectName) + r, err := obj.NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("failed to open reader: %w", err) + } + return r, nil +} + +func (gh *gcshandler) deleteFile(ctx context.Context, bucketName, objectName string) error { + bkt := gh.client.Bucket(bucketName) + obj := bkt.Object(objectName) + if err := obj.Delete(ctx); err != nil { + return fmt.Errorf("failed to delete object: %w", err) + } + return nil +} + +func (gh *gcshandler) getFileSize(ctx context.Context, bucketName, objectName string) (int64, error) { + objAttrs, err := gh.client.Bucket(bucketName).Object(objectName).Attrs(ctx) + if err != nil { + return 0, fmt.Errorf("failed to get object attrs: %w", err) + } + return objAttrs.Size, nil +} + +// gcsReadSeekCloser implements media.ReadSeekCloser interface +type gcsReadSeekCloser struct { + reader io.Reader + close func() error +} + +func (g *gcsReadSeekCloser) Read(p []byte) (n int, err error) { + return g.reader.Read(p) +} + +func (g *gcsReadSeekCloser) Seek(offset int64, whence int) (int64, error) { + // GCS doesn't support seeking, so we return an error + return 0, errors.New("seeking not supported for GCS objects") +} + +func (g *gcsReadSeekCloser) Close() error { + return g.close() +} + +func etagFromObjectName(objectName string) string { + // Use object name as ETag since it's unique + return strings.ToLower(objectName) +} + +func init() { + store.RegisterMediaHandler(handlerName, &gcshandler{}) +} diff --git a/server/tinode.conf b/server/tinode.conf index 47c4502e9..feaa1a11c 100644 --- a/server/tinode.conf +++ b/server/tinode.conf @@ -147,6 +147,28 @@ // Origin URLs allowed to download files, e.g. ["https://www.example.com", "http://example.com"]. // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin "cors_origins": ["*"] + }, + "gcs": { + // Google Cloud Storage bucket name where files will be stored + "bucket_name": "your-bucket-name", + // URL path prefix for serving files through your web server/reverse proxy + "serve_url": "/v0/file/s/", + // CORS origins allowed to access files (use specific domains in production, not "*") + "cors_origins": ["*"], + // HTTP cache control header for served files (86400 seconds = 24 hours) + "cache_control": "max-age=86400", + // GCP project ID where your Cloud Storage bucket is located + "project_id": "your-gcp-project-id", + // GCP region/location for your bucket (e.g., "us-central1", "europe-west1") + "location_id": "your-kms-location", + // Cloud KMS key ring ID for server-side encryption (optional security feature) + "key_ring_id": "your-key-ring-id", + // Cloud KMS crypto key ID for server-side encryption (optional security feature) + "key_id": "your-crypto-key-id", + // Base directory path within the bucket for organizing uploaded files + "base_path": "tinode-files", + // Maximum time to wait for GCS operations before timing out + "client_timeout": "30s" } } },