feat(blob): optional blob event capture to files (Event Grid schema)

README.md

@@ -208,6 +208,8 @@ Following extension configurations are supported:
 - `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost.
 - `azurite.extentMemoryLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of megabytes. This does not limit blob, queue, or table metadata. Defaults to 50% of total memory.
 - `azurite.disableTelemetry` Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product.
+- `azurite.blobEventCapture` Enable capturing blob mutation events as Azure Event Grid-shaped JSON files into a folder for later processing, by default false.
+- `azurite.blobEventCapturePath` Folder to write captured blob event JSON files to. Relative paths resolve against the workspace location. Defaults to `__blobevents__` under the workspace location when `azurite.blobEventCapture` is enabled.
 
 ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite)
 
@@ -458,6 +460,22 @@ Optional. By default, Azurite will collect telemetry data to help improve the pr
 --disableTelemetry
 ```
 
+### Blob Event Capture
+
+Optional. Capture mutating blob operations as an [Azure Event Grid](https://learn.microsoft.com/azure/storage/blobs/storage-blob-event-overview)-shaped JSON file (one file per event) written into a folder, so the events can be processed later. Disabled by default. Enable it by:
+
+```cmd
+--blobEventCapture
+```
+
+By default the files are written to a `__blobevents__` folder under the workspace location. Write them to a different folder (relative paths resolve against the workspace location) by:
+
+```cmd
+--blobEventCapturePath path/to/folder
+```
+
+Each event is written to its own file named `{timestamp}-{uuid}.json`, published atomically so a consumer watching the folder only ever reads a complete file. The captured event uses the Azure Event Grid schema: `eventType` is `Microsoft.Storage.BlobCreated` or `Microsoft.Storage.BlobDeleted` (plus the Azurite convention-named `Microsoft.Storage.ContainerCreated` / `Microsoft.Storage.ContainerDeleted`), with the precise operation carried in `data.api` (for example `PutBlob`, `PutBlockList`, `AppendBlock`, `PutPage`, `DeleteBlob`, `CreateContainer`, `DeleteContainer`). Capture is fire-and-forget and never affects the outcome of a storage operation. If `--blobEventCapturePath` is supplied without `--blobEventCapture`, capture stays off and the path is ignored.
+
 ### Use in-memory storage
 
 Optional. Disable persisting any data to disk and only store data in-memory. If the Azurite process is terminated, all

package.json

@@ -275,6 +275,16 @@
             "type": "boolean",
             "default": false,
             "description": "Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product."
+          },
+          "azurite.blobEventCapture": {
+            "type": "boolean",
+            "default": false,
+            "description": "Enable capturing blob mutation events as Azure Event Grid-shaped JSON files into a folder for later processing."
+          },
+          "azurite.blobEventCapturePath": {
+            "type": "string",
+            "default": "",
+            "description": "Folder to write captured blob event JSON files to. Relative paths resolve against the workspace location. Defaults to '__blobevents__' under the workspace location when blob event capture is enabled."
           }
         }
       }

src/blob/BlobConfiguration.ts

@@ -45,6 +45,8 @@ export default class BlobConfiguration extends ConfigurationBase {
     disableProductStyleUrl: boolean = false,
     public readonly isMemoryPersistence: boolean = false,
     public readonly memoryStore?: MemoryExtentChunkStore,
+    public readonly enableBlobEventCapture: boolean = false,
+    public readonly blobEventCapturePath: string = ""
   ) {
     super(
       host,

src/blob/BlobEnvironment.ts

@@ -70,6 +70,14 @@ if (!(args as any).config.name) {
     .option(
       ["", "disableTelemetry"],
       "Optional. Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product."
+    )
+    .option(
+      ["", "blobEventCapture"],
+      "Optional. Enable capturing blob mutation events as Azure Event Grid-shaped JSON files for later processing"
+    )
+    .option(
+      ["", "blobEventCapturePath"],
+      "Optional. Folder to write captured blob event JSON files to. Defaults to '__blobevents__' under the workspace location"
     );
 
   (args as any).config.name = "azurite-blob";
@@ -148,6 +156,18 @@ export default class BlobEnvironment implements IBlobEnvironment {
     return false;
   }
 
+  public blobEventCapture(): boolean {
+    if (this.flags.blobEventCapture !== undefined) {
+      return this.flags.blobEventCapture;
+    }
+    // default is false: blob event capture is opt-in
+    return false;
+  }
+
+  public blobEventCapturePath(): string | undefined {
+    return this.flags.blobEventCapturePath;
+  }
+
   public inMemoryPersistence(): boolean {
     if (this.flags.inMemoryPersistence !== undefined) {
       if (this.flags.location) {

src/blob/BlobRequestListenerFactory.ts

@@ -26,6 +26,7 @@ import StrictModelMiddlewareFactory, {
   UnsupportedHeadersBlocker,
   UnsupportedParametersBlocker
 } from "./middlewares/StrictModelMiddlewareFactory";
+import IBlobEventSink from "./events/IBlobEventSink";
 import IBlobMetadataStore from "./persistence/IBlobMetadataStore";
 import { DEFAULT_CONTEXT_PATH } from "./utils/constants";
 
@@ -56,7 +57,8 @@ export default class BlobRequestListenerFactory
     private readonly loose?: boolean,
     private readonly skipApiVersionCheck?: boolean,
     private readonly oauth?: OAuthLevel,
-    private readonly disableProductStyleUrl?: boolean
+    private readonly disableProductStyleUrl?: boolean,
+    private readonly eventSink?: IBlobEventSink
   ) { }
 
   public createRequestListener(): RequestListener {
@@ -77,20 +79,23 @@ export default class BlobRequestListenerFactory
         this.metadataStore,
         this.extentStore,
         logger,
-        loose
+        loose,
+        this.eventSink
       ),
       blobHandler: new BlobHandler(
         this.metadataStore,
         this.extentStore,
         logger,
         loose,
-        pageBlobRangesManager
+        pageBlobRangesManager,
+        this.eventSink
       ),
       blockBlobHandler: new BlockBlobHandler(
         this.metadataStore,
         this.extentStore,
         logger,
-        loose
+        loose,
+        this.eventSink
       ),
       containerHandler: new ContainerHandler(
         this.accountDataStore,
@@ -99,14 +104,16 @@ export default class BlobRequestListenerFactory
         this.extentStore,
         logger,
         loose,
-        this.disableProductStyleUrl
+        this.disableProductStyleUrl,
+        this.eventSink
       ),
       pageBlobHandler: new PageBlobHandler(
         this.metadataStore,
         this.extentStore,
         logger,
         loose,
-        pageBlobRangesManager
+        pageBlobRangesManager,
+        this.eventSink
       ),
       serviceHandler: new ServiceHandler(
         this.accountDataStore,

src/blob/BlobServer.ts

@@ -19,6 +19,8 @@ import { handleGCCriticalErrorClose } from "../common/GCCriticalErrorCloseHelper
 import ServerBase, { ServerStatus } from "../common/ServerBase";
 import BlobConfiguration from "./BlobConfiguration";
 import BlobRequestListenerFactory from "./BlobRequestListenerFactory";
+import FileBlobEventSink from "./events/FileBlobEventSink";
+import IBlobEventSink from "./events/IBlobEventSink";
 import BlobGCManager from "./gc/BlobGCManager";
 import IBlobMetadataStore from "./persistence/IBlobMetadataStore";
 import LokiBlobMetadataStore from "./persistence/LokiBlobMetadataStore";
@@ -47,6 +49,7 @@ export default class BlobServer extends ServerBase implements ICleaner {
   private readonly extentStore: IExtentStore;
   private readonly accountDataStore: IAccountDataStore;
   private readonly gcManager: IGCManager;
+  private readonly eventSink?: IBlobEventSink;
 
   /**
    * Creates an instance of Server.
@@ -107,6 +110,11 @@ export default class BlobServer extends ServerBase implements ICleaner {
     // We can also change the HTTP framework here by
     // creating a new XXXListenerFactory implementing IRequestListenerFactory interface
     // and replace the default Express based request listener
+    const eventSink: IBlobEventSink | undefined =
+      configuration.enableBlobEventCapture && configuration.blobEventCapturePath
+        ? new FileBlobEventSink(configuration.blobEventCapturePath, logger)
+        : undefined;
+
     const requestListenerFactory: IRequestListenerFactory =
       new BlobRequestListenerFactory(
         metadataStore,
@@ -117,7 +125,8 @@ export default class BlobServer extends ServerBase implements ICleaner {
         configuration.loose,
         configuration.skipApiVersionCheck,
         configuration.getOAuthLevel(),
-        configuration.disableProductStyleUrl
+        configuration.disableProductStyleUrl,
+        eventSink
       );
 
     super(host, port, httpServer, requestListenerFactory, configuration);
@@ -158,6 +167,7 @@ export default class BlobServer extends ServerBase implements ICleaner {
     this.extentStore = extentStore;
     this.accountDataStore = accountDataStore;
     this.gcManager = gcManager;
+    this.eventSink = eventSink;
   }
 
   /**
@@ -212,6 +222,10 @@ export default class BlobServer extends ServerBase implements ICleaner {
     if (this.gcManager !== undefined) {
       await this.gcManager.start();
     }
+
+    if (this.eventSink !== undefined) {
+      await this.eventSink.init();
+    }
   }
 
   protected async afterStart(): Promise<void> {
@@ -224,6 +238,10 @@ export default class BlobServer extends ServerBase implements ICleaner {
   }
 
   protected async afterClose(): Promise<void> {
+    if (this.eventSink !== undefined) {
+      await this.eventSink.close();
+    }
+
     if (this.gcManager !== undefined) {
       await this.gcManager.close();
     }

src/blob/BlobServerFactory.ts

@@ -1,17 +1,19 @@
 import { join } from "path";
 
 import { DEFAULT_SQL_OPTIONS } from "../common/utils/constants";
+import logger from "../common/Logger";
 import BlobConfiguration from "./BlobConfiguration";
 import BlobEnvironment from "./BlobEnvironment";
 import BlobServer from "./BlobServer";
+import { resolveBlobEventCapturePath } from "./events/resolveBlobEventCapturePath";
 import IBlobEnvironment from "./IBlobEnvironment";
 import SqlBlobConfiguration from "./SqlBlobConfiguration";
 import SqlBlobServer from "./SqlBlobServer";
-import { DEFAULT_BLOB_PERSISTENCE_PATH } from "./utils/constants";
 import {
   DEFAULT_BLOB_EXTENT_LOKI_DB_PATH,
   DEFAULT_BLOB_LOKI_DB_PATH,
-  DEFAULT_BLOB_PERSISTENCE_ARRAY
+  DEFAULT_BLOB_PERSISTENCE_ARRAY,
+  DEFAULT_BLOB_PERSISTENCE_PATH
 } from "./utils/constants";
 
 export class BlobServerFactory {
@@ -32,6 +34,26 @@ export class BlobServerFactory {
         );
       }
 
+      const enableBlobEventCapture = env.blobEventCapture();
+      const configuredCapturePath = env.blobEventCapturePath();
+      const blobEventCapturePath = resolveBlobEventCapturePath(
+        enableBlobEventCapture,
+        configuredCapturePath,
+        location
+      );
+      if (
+        !enableBlobEventCapture &&
+        configuredCapturePath &&
+        configuredCapturePath.length > 0
+      ) {
+        // Note the empty-string check: the VS Code setting declares a "" default,
+        // so get<string>() returns "" (not undefined) when unset. Warn only when
+        // a non-empty path was actually supplied without enabling capture.
+        logger.warn(
+          "blobEventCapturePath is configured but blobEventCapture is disabled; blob event capture is OFF and the path will be ignored."
+        );
+      }
+
       DEFAULT_BLOB_PERSISTENCE_ARRAY[0].locationPath = join(
         location,
         DEFAULT_BLOB_PERSISTENCE_PATH
@@ -66,7 +88,9 @@ export class BlobServerFactory {
           env.key(),
           env.pwd(),
           env.oauth(),
-          env.disableProductStyleUrl()
+          env.disableProductStyleUrl(),
+          enableBlobEventCapture,
+          blobEventCapturePath
         );
 
         return new SqlBlobServer(config);
@@ -90,6 +114,9 @@ export class BlobServerFactory {
           env.oauth(),
           env.disableProductStyleUrl(),
           env.inMemoryPersistence(),
+          undefined,
+          enableBlobEventCapture,
+          blobEventCapturePath
         );
 
         return new BlobServer(config);

src/blob/IBlobEnvironment.ts

@@ -15,4 +15,6 @@ export default interface IBlobEnvironment {
   inMemoryPersistence(): boolean;
   extentMemoryLimit(): number | undefined;
   disableTelemetry(): boolean;
+  blobEventCapture(): boolean;
+  blobEventCapturePath(): string | undefined;
 }

src/blob/SqlBlobConfiguration.ts

@@ -37,7 +37,9 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
     key: string = "",
     pwd: string = "",
     oauth?: string,
-    disableProductStyleUrl: boolean = false
+    disableProductStyleUrl: boolean = false,
+    public readonly enableBlobEventCapture: boolean = false,
+    public readonly blobEventCapturePath: string = ""
   ) {
     super(
       host,

src/blob/SqlBlobServer.ts

@@ -14,6 +14,8 @@ import IExtentStore from "../common/persistence/IExtentStore";
 import SqlExtentMetadataStore from "../common/persistence/SqlExtentMetadataStore";
 import ServerBase, { ServerStatus } from "../common/ServerBase";
 import BlobRequestListenerFactory from "./BlobRequestListenerFactory";
+import FileBlobEventSink from "./events/FileBlobEventSink";
+import IBlobEventSink from "./events/IBlobEventSink";
 import BlobGCManager from "./gc/BlobGCManager";
 import IBlobMetadataStore from "./persistence/IBlobMetadataStore";
 import SqlBlobMetadataStore from "./persistence/SqlBlobMetadataStore";
@@ -42,6 +44,7 @@ export default class SqlBlobServer extends ServerBase {
   private readonly extentStore: IExtentStore;
   private readonly accountDataStore: IAccountDataStore;
   private readonly gcManager: IGCManager;
+  private readonly eventSink?: IBlobEventSink;
 
   /**
    * Creates an instance of Server.
@@ -89,6 +92,11 @@ export default class SqlBlobServer extends ServerBase {
     // We can also change the HTTP framework here by
     // creating a new XXXListenerFactory implementing IRequestListenerFactory interface
     // and replace the default Express based request listener
+    const eventSink: IBlobEventSink | undefined =
+      configuration.enableBlobEventCapture && configuration.blobEventCapturePath
+        ? new FileBlobEventSink(configuration.blobEventCapturePath, logger)
+        : undefined;
+
     const requestListenerFactory: IRequestListenerFactory =
       new BlobRequestListenerFactory(
         metadataStore,
@@ -99,7 +107,8 @@ export default class SqlBlobServer extends ServerBase {
         configuration.loose,
         configuration.skipApiVersionCheck,
         configuration.getOAuthLevel(),
-        configuration.disableProductStyleUrl
+        configuration.disableProductStyleUrl,
+        eventSink
       );
 
     super(host, port, httpServer, requestListenerFactory, configuration);
@@ -130,6 +139,7 @@ export default class SqlBlobServer extends ServerBase {
     this.extentStore = extentStore;
     this.accountDataStore = accountDataStore;
     this.gcManager = gcManager;
+    this.eventSink = eventSink;
   }
 
   /**
@@ -183,6 +193,10 @@ export default class SqlBlobServer extends ServerBase {
     if (this.gcManager !== undefined) {
       await this.gcManager.start();
     }
+
+    if (this.eventSink !== undefined) {
+      await this.eventSink.init();
+    }
   }
 
   protected async afterStart(): Promise<void> {
@@ -195,6 +209,10 @@ export default class SqlBlobServer extends ServerBase {
   }
 
   protected async afterClose(): Promise<void> {
+    if (this.eventSink !== undefined) {
+      await this.eventSink.close();
+    }
+
     if (this.gcManager !== undefined) {
       await this.gcManager.close();
     }