From f8235331f0a0dddb98a90cd2dab9ee706ea8d8d6 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 25 Jul 2026 15:31:48 -0700 Subject: [PATCH 1/3] Implement Put Block From URL Stage the block by fetching the copy source over loopback so that SAS authentication, x-ms-source-range, and the x-ms-source-if-* conditions are enforced by the existing download path, then persist it through the same extent flow as Put Block. Only sources on the same Azurite instance are supported, matching copyFromURL. The response carries the MD5 of the staged content and source condition failures return 412 SourceConditionNotMet as on the real service. Reject malformed x-ms-source-content-md5 and x-ms-source-content-crc64 up front so the errors name the header the caller sent, then compare the surviving values against the fetched bytes with computeAndValidateTransactionalChecksums, the same path Put Block uses. Mismatches return Md5Mismatch or Crc64Mismatch, and supplying both returns BothCrc64AndMd5HeaderPresent. The response reports the computed CRC64 when no MD5 was supplied, as stageBlock does. Request the source body verbatim and never decompress it. A blob's Content-Encoding is stored metadata rather than a description of the wire framing, so the download echoes it back over the raw stored bytes; decoding here would stage the decompressed content instead of what the source holds, and would fail outright when the property does not match the bytes (issue #646). The source fetch is pinned to the address and port the request arrived on, so over HTTPS it presents whatever certificate Azurite was started with. Skip verification for that self-request, which would otherwise reject the self-signed certificates Azurite is normally run with and make the operation unusable under --cert/--key, and translate transport-level failures into CannotVerifyCopySource rather than letting them escape as a bodiless 500. The same applies to a body that fails partway through being read. Validated with the blockblob test suite and end to end with S3Proxy's native multipart part copy, which previously fell back to streamed emulation on Azurite's 501. Co-Authored-By: Claude Fable 5 --- ChangeLog.md | 1 + README.md | 1 + src/blob/errors/StorageErrorFactory.ts | 9 + src/blob/handlers/BlockBlobHandler.ts | 304 +++++++++++- tests/blob/apis/blockblob.test.ts | 614 +++++++++++++++++++++++++ tests/blob/https.test.ts | 65 +++ 6 files changed, 993 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 48fed1911..021de054f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -58,6 +58,7 @@ Blob: - Fixed issue #2672 startup failures with legacy persisted data by adding backward-compatible restore for persisted `contentMD5` formats. - Added CRC-64/NVME transactional checksum support for `StageBlock`, `PutBlock`, `PutBlob`, `AppendBlock`, and `PutPage` (`x-ms-content-crc64`). - Harden transactional checksum validation for `PutBlob`, `StageBlock`, `AppendBlock`, and `PutPage`: unified MD5/CRC64 validation logic with accurate `InvalidMd5`/`InvalidHeaderValue` (malformed) and `Md5Mismatch`/`Crc64Mismatch` (mismatch) errors, matching real Azure semantics verified against live. +- Implement `PutBlockFromURL` (`Put Block From URL`), which previously returned 501. The source is fetched over loopback so that SAS authentication, `x-ms-source-range`, and the `x-ms-source-if-*` conditions are enforced by the existing download path; unmet source conditions return 412 `SourceConditionNotMet`. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. - Fix `x-ms-blob-content-md5` precedence over `Content-MD5` for `PutBlob` transit integrity verification, matching real Azure behavior. - Make `CopyBlobFromURL` echo back the source `Content-MD5` when supplied via `x-ms-source-content-md5`, matching real Azure behavior. - Added support for the `startFrom` query parameter on `List Blobs` (service version `2026-02-06`), which begins a flat or hierarchical listing at the given blob name. Unlike `marker`, which is exclusive, `startFrom` is inclusive, and the two compose when paging a listing that began at `startFrom`. Previously the parameter was accepted but ignored. diff --git a/README.md b/README.md index bc80aa460..191180e4b 100644 --- a/README.md +++ b/README.md @@ -1076,6 +1076,7 @@ Detailed support matrix: - Copy Blob (Only supports copy within same Azurite instance) - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) + - Put Block From URL (Only supports source within same Azurite instance) - Access control based on conditional headers - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite diff --git a/src/blob/errors/StorageErrorFactory.ts b/src/blob/errors/StorageErrorFactory.ts index a0c4e897b..9c14e6944 100644 --- a/src/blob/errors/StorageErrorFactory.ts +++ b/src/blob/errors/StorageErrorFactory.ts @@ -791,6 +791,15 @@ export default class StorageErrorFactory { ); } + public static getSourceConditionNotMet(contextID: string): StorageError { + return new StorageError( + 412, + "SourceConditionNotMet", + "The source condition specified using HTTP conditional header(s) is not met.", + contextID + ); + } + public static getInvalidResourceName(contextID: string = ""): StorageError { return new StorageError( 400, diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 666a067f0..2f6db0863 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,3 +1,10 @@ +import axios, { AxiosResponse } from "axios"; +import { IncomingMessage } from "http"; +import { Agent } from "https"; + +import { Readable } from "stream"; + +import { IExtentChunk } from "../../common/persistence/IExtentStore"; import { convertRawHeadersToMetadata, getMD5FromString, @@ -18,6 +25,16 @@ import { getTagsFromString } from "../utils/utils"; +/** + * Agent for the loopback self-request stageBlockFromURL makes to read a copy + * source. Shared so requests reuse one Agent rather than allocating their + * own, not for socket reuse: keep-alive stays off, taking a loopback + * handshake per request over an idle socket that the server may close + * mid-reuse, which would surface as a spurious CannotVerifyCopySource. See + * the call site for why verification is off. + */ +const LOOPBACK_HTTPS_AGENT = new Agent({ rejectUnauthorized: false }); + /** * BlobHandler handles Azure Storage BlockBlob related requests. * @@ -261,7 +278,292 @@ export default class BlockBlobHandler options: Models.BlockBlobStageBlockFromURLOptionalParams, context: Context ): Promise { - throw new NotImplementedError(context.contextId); + const blobCtx = new BlobStorageContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + // Put Block From URL carries no request body. + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "Content-Length", + HeaderValue: contentLength.toString() + }); + } + + this.validateBlockId(blockId, blobCtx); + + // Reject malformed source checksum headers before fetching anything. The + // shared validator would catch these too, but it reports the names of the + // transactional headers, and its errors would surface only after the + // source had already been read and staged. + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentcrc64 !== undefined + ) { + throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( + context.contextId + ); + } + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentMD5.length !== 16 + ) { + throw StorageErrorFactory.getInvalidMd5(context.contextId); + } + if ( + options.sourceContentcrc64 !== undefined && + options.sourceContentcrc64.length < 8 + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-source-content-crc64", + HeaderValue: Buffer.from(options.sourceContentcrc64).toString("base64") + }); + } + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + let url: URL; + try { + url = new URL(sourceUrl); + } catch { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source", + HeaderValue: sourceUrl + }); + } + + // Only sources within the same Azurite instance are supported, as with + // copyFromURL. + // Hostnames compare case-insensitively and new URL() lowercases its + // host, so normalize the client-supplied header before comparing. + const currentServer = (blobCtx.request!.getHeader("Host") || "") + .toLowerCase(); + if (currentServer !== url.host) { + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Source ${url} is not on the same Azurite instance as target account ${accountName}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + // The Host header above is client-controlled, so never fetch the + // caller-supplied URL directly; pin the outbound request to the + // loopback address and port this server is actually bound to, keeping + // only the caller's path and query. + const rawRequest = blobCtx.request!.getBodyStream(); + if (!(rawRequest instanceof IncomingMessage) || + rawRequest.socket.localPort === undefined) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; + // Use the local address this request arrived on rather than a + // hard-coded loopback so non-loopback --blobHost binds keep working; + // IPv6 literals need brackets in URLs. + const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; + const localHost = localAddress.includes(":") ? + `[${localAddress}]` : localAddress; + const pinnedUrl = + `${scheme}://${localHost}:${rawRequest.socket.localPort}` + + `${url.pathname}${url.search}`; + + // Fetch the source range over loopback so that SAS authentication, + // range handling, and source conditions reuse the download path. + // Preserve the source URL's host so product-style source URLs still + // resolve their account from the Host header; the connection itself + // stays pinned to this server's bound address. + // A block must be staged as the bytes the source actually stores, so ask + // for the body verbatim rather than letting anything in the path apply + // transfer compression. + const headers: { [key: string]: string } = { + host: url.host, + "accept-encoding": "identity" + }; + if (options.sourceRange !== undefined) { + // The download path ignores malformed Range headers, which would + // silently stage the entire source blob; reject them up front. + // Compare offsets as BigInt: they are 64-bit, and Number rounds + // above 2^53, which would let an end < start range slip through. + const rangeMatch = /^bytes=(\d+)-(\d*)$/.exec(options.sourceRange); + if (rangeMatch === null || + (rangeMatch[2] !== "" && + BigInt(rangeMatch[2]) < BigInt(rangeMatch[1]))) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-source-range", + HeaderValue: options.sourceRange + }); + } + headers.range = options.sourceRange; + } + const sourceConditions = options.sourceModifiedAccessConditions || {}; + if (sourceConditions.sourceIfMatch !== undefined) { + headers["if-match"] = sourceConditions.sourceIfMatch; + } + if (sourceConditions.sourceIfNoneMatch !== undefined) { + headers["if-none-match"] = sourceConditions.sourceIfNoneMatch; + } + if (sourceConditions.sourceIfModifiedSince !== undefined) { + headers["if-modified-since"] = new Date( + sourceConditions.sourceIfModifiedSince + ).toUTCString(); + } + if (sourceConditions.sourceIfUnmodifiedSince !== undefined) { + headers["if-unmodified-since"] = new Date( + sourceConditions.sourceIfUnmodifiedSince + ).toUTCString(); + } + // Note: unlike the Copy Blob operations, Put Block From URL has no + // x-ms-source-if-tags condition; the generated operation spec does not + // deserialize one. + + let sourceResponse: AxiosResponse; + try { + sourceResponse = await axios.get(pinnedUrl, { + headers, + responseType: "stream", + validateStatus: () => true, + // Never decompress. A source blob carries Content-Encoding as a + // stored property, so the download echoes it back even though the + // bytes on the wire are the raw stored ones. Decompressing here + // would stage the decoded content instead of what the source holds, + // and would fail outright when the property does not match the + // bytes. + decompress: false, + // The connection is pinned above to the address and port this very + // request arrived on, so the peer is necessarily this same server. + // Verifying its certificate would only reject the self-signed certs + // Azurite is normally run with under --cert/--key, which would make + // this operation unusable over HTTPS. + httpsAgent: scheme === "https" ? LOOPBACK_HTTPS_AGENT : undefined + }); + } catch (err) { + // Transport-level failures (TLS, connection reset, socket errors) throw + // rather than returning a status. Without this they would escape as a + // bodiless 500 instead of an Azure-shaped error. + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + if (sourceResponse.status === 304 || sourceResponse.status === 412) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); + } + if (sourceResponse.status === 404) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + sourceResponse.status, + "Could not verify the copy source within the specified time." + ); + } + + // The status above only means the response headers arrived; the body can + // still fail midway (socket error, connection reset). Map that to the + // same error as a transport failure rather than letting it escape as a + // bodiless 500, and release the source stream on the way out. + let persistency: IExtentChunk; + try { + persistency = await this.extentStore.appendExtent( + sourceResponse.data, + context.contextId + ); + } catch (err) { + sourceResponse.data.destroy(); + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source body: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // Compare the supplied source checksums against the fetched bytes with + // the same helper Put Block uses. The MD5 is always computed because the + // response echoes it; the CRC64 is computed when it can be reported, + // which mirrors stageBlock (the two checksums are mutually exclusive). + // The header shapes were already rejected above, so the only failures + // left are mismatches, which happen after the stream has been read. + // Destroy it regardless so a throw cannot leave the extent handle open. + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + let calculatedContentMD5: Uint8Array | undefined; + let calculatedContentCRC64: Uint8Array | undefined; + try { + ({ md5: calculatedContentMD5, crc64: calculatedContentCRC64 } = + await computeAndValidateTransactionalChecksums( + stream, + { + md5: options.sourceContentMD5, + crc64: options.sourceContentcrc64 + }, + context.contextId, + { md5: true, crc64: options.sourceContentMD5 === undefined } + )); + } finally { + (stream as Readable).destroy?.(); + } + + const block: BlockModel = { + accountName, + containerName, + blobName, + isCommitted: false, + name: blockId, + size: persistency.count, + persistency + }; + + await this.metadataStore.stageBlock( + context, + block, + options.leaseAccessConditions + ); + + const response: Models.BlockBlobStageBlockFromURLResponse = { + statusCode: 201, + contentMD5: calculatedContentMD5, + xMsContentCrc64: calculatedContentCRC64, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; } public async commitBlockList( diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index fd43e4a1e..1cd19661d 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -6,8 +6,10 @@ import { Tags } from "@azure/storage-blob"; import CustomHeaderPolicyFactory from "../RequestPolicy/CustomHeaderPolicyFactory"; +import axios from "axios"; import * as assert from "assert"; import * as crypto from "crypto"; +import * as zlib from "zlib"; import { configLogger } from "../../../src/common/Logger"; import BlobTestServerFactory from "../../BlobTestServerFactory"; @@ -421,6 +423,618 @@ describe("BlockBlobAPIs", () => { ); }); + it("stageBlockFromURL @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const resultStage = await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + 10 + ); + const expectedMD5 = await getMD5FromString(content.substring(0, 10)); + assert.deepStrictEqual( + Buffer.from(resultStage.contentMD5!), + Buffer.from(expectedMD5) + ); + + await blockBlobClient.stageBlockFromURL( + base64encode("2"), + sourceUrl, + 10, + content.length - 10 + ); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 2); + assert.equal(listResponse.uncommittedBlocks![0].size, 10); + assert.equal( + listResponse.uncommittedBlocks![1].size, + content.length - 10 + ); + + await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL without range copies the entire source @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + await blockBlobClient.commitBlockList([base64encode("1")]); + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL rejects an unmet source condition @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // @azure/storage-blob does not expose x-ms-source-if-* on + // stageBlockFromURL, so issue the request directly. + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-source-if-match": '"0x0000000000000000"', + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 412); + assert.ok(response.data.includes("SourceConditionNotMet")); + }); + + it("stageBlockFromURL rejects a request body @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + "unexpected body", + { + headers: { + "x-ms-copy-source": sourceUrl + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 400); + assert.ok(response.data.includes("InvalidHeaderValue")); + }); + + it("stageBlockFromURL rejects a malformed source range @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + for (const badRange of [ + "bytes=abc", + "bytes=5-2", + "0-10", + // end < start detectable only beyond double precision: both offsets + // round to 2^53 as Numbers, hiding that the range is inverted + "bytes=9007199254740993-9007199254740992" + ]) { + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-source-range": badRange, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 400, badRange); + assert.ok(response.data.includes("InvalidHeaderValue"), badRange); + } + }); + + it("stageBlockFromURL with product-style source URL @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceName = getUniqueName("source"); + const sourceClient = containerClient.getBlockBlobClient(sourceName); + await sourceClient.upload(content, content.length); + const sourceSasQuery = (await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + const destinationSasQuery = (await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + + // Both requests use product-style URLs, where the account comes from + // the Host header rather than the path. + const productHost = + `${EMULATOR_ACCOUNT_NAME}.localhost:${server.config.port}`; + const response = await axios.put( + `http://${server.config.host}:${server.config.port}` + + `/${containerName}/${blobName}?${destinationSasQuery}` + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + host: productHost, + "x-ms-copy-source": + `http://${productHost}/${containerName}/${sourceName}` + + `?${sourceSasQuery}`, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 201); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, content.length); + }); + + it("stageBlockFromURL accepts a mixed-case Host header @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceName = getUniqueName("source"); + const sourceClient = containerClient.getBlockBlobClient(sourceName); + await sourceClient.upload(content, content.length); + const sourceSasQuery = (await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + const destinationSasQuery = (await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + + // Host headers are case-insensitive; the lowercase source URL host + // must match despite the client's casing. + const response = await axios.put( + `http://${server.config.host}:${server.config.port}` + + `/${EMULATOR_ACCOUNT_NAME}/${containerName}/${blobName}` + + `?${destinationSasQuery}` + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + host: `LocalHost:${server.config.port}`, + "x-ms-copy-source": + `http://localhost:${server.config.port}` + + `/${EMULATOR_ACCOUNT_NAME}/${containerName}/${sourceName}` + + `?${sourceSasQuery}`, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 201); + }); + + it("stageBlockFromURL from a missing source returns 404 @loki @sql", async () => { + const missingClient = containerClient.getBlockBlobClient( + getUniqueName("missing") + ); + const sourceUrl = await missingClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl + ); + assert.fail(); + } catch (err: any) { + assert.deepStrictEqual(err.statusCode, 404); + assert.deepStrictEqual(err.details.errorCode, "CannotVerifyCopySource"); + } + }); + + it("stageBlockFromURL stages the stored bytes when the source declares Content-Encoding: gzip @loki @sql", async () => { + // A blob's Content-Encoding is stored metadata, not a description of how + // the body is framed on the wire, so the download echoes it back over the + // raw stored bytes. Staging must copy those bytes verbatim rather than + // decoding them, otherwise the block holds the decompressed content. + const raw = zlib.gzipSync(Buffer.from("HelloWorldFromSourceBlob")); + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(raw, raw.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, raw.length); + + await blockBlobClient.commitBlockList([base64encode("1")]); + const download = await blockBlobClient.download(0); + const chunks: Buffer[] = []; + for await (const chunk of download.readableStreamBody!) { + chunks.push(Buffer.from(chunk)); + } + assert.deepStrictEqual( + Buffer.concat(chunks), + raw, + "Staged block must be the source's stored bytes, not the decoded ones" + ); + }); + + it("stageBlockFromURL succeeds when the source's Content-Encoding does not match its bytes @loki @sql", async () => { + // Nothing validates that a blob's stored Content-Encoding describes its + // content, so a plain body can be labelled gzip. Staging must not try to + // decode it (see issue #646 for the same hazard on copy). + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, content.length); + + await blockBlobClient.commitBlockList([base64encode("1")]); + const result = await blockBlobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL with matching sourceContentMD5 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const resultStage = await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: new Uint8Array(md5) } + ); + + // The response echoes the MD5 the service computed over the staged + // content, which for a matching request equals the supplied value. + assert.deepStrictEqual( + Buffer.from(resultStage.contentMD5!), + Buffer.from(md5) + ); + // The two checksums are mutually exclusive, so no CRC64 is reported + // alongside an MD5, matching stageBlock. + assert.strictEqual((resultStage as any).xMsContentCrc64, undefined); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + }); + + it("stageBlockFromURL with wrong sourceContentMD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // Stage one good block first, so the block list below distinguishes "the + // rejected block was not staged" from "the blob does not exist yet". + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + + // A valid 16-byte MD5 of a *different* body, to exercise the mismatch + // path rather than the InvalidMd5 (wrong-length) path. + const md5 = crypto.createHash("md5").update("anotherBody", "utf8").digest(); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("2"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: new Uint8Array(md5) } + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + + // A rejected block must not be staged. + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal( + listResponse.uncommittedBlocks![0].name, + base64encode("1") + ); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with wrong-length sourceContentMD5 should be rejected @loki @sql", async () => { + // x-ms-source-content-md5 must decode to exactly 16 bytes. This test pins + // which error code the service returns for a malformed (4-byte) value so + // Azurite can be verified against real Azure. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const wrongLengthMd5 = new Uint8Array([0, 0, 0, 0]); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: wrongLengthMd5 } + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with matching sourceContentCrc64 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-crc64", value: crc64 }] + ); + + const resultStage = await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + + // The response reports the CRC64 the service computed over the staged + // content, as stageBlock does. + assert.equal( + Buffer.from((resultStage as any).xMsContentCrc64!).toString("base64"), + crc64 + ); + + const listResponse = await targetClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + }); + + it("stageBlockFromURL with wrong sourceContentCrc64 should throw crc64 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // A valid 8-byte CRC64 of a *different* body, to exercise the mismatch + // path rather than the malformed-header path. + const crc64 = Buffer.from(getCRC64FromString("anotherBody")).toString( + "base64" + ); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-crc64", value: crc64 }] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Crc64Mismatch"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with wrong-length sourceContentCrc64 should be rejected @loki @sql", async () => { + // x-ms-source-content-crc64 must decode to at least 8 bytes. This test + // pins which error code the service returns for a malformed (4-byte) + // value so Azurite can be verified against real Azure. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [ + { + key: "x-ms-source-content-crc64", + value: Buffer.from([1, 2, 3, 4]).toString("base64") + } + ] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidHeaderValue"); + // The error must name the header the caller actually sent, not the + // transactional x-ms-content-crc64 the shared validator reports. + assert.equal( + /([^<]*) { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // Both checksums are correct for the source content; supplying the two + // together is rejected regardless, as on the real service. + const md5 = Buffer.from(await getMD5FromString(content)).toString("base64"); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [ + { key: "x-ms-source-content-md5", value: md5 }, + { key: "x-ms-source-content-crc64", value: crc64 } + ] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "BothCrc64AndMd5HeaderPresent"); + return; + } + assert.fail("Did not throw an exception."); + }); + it("stageBlock with double commit block should work @loki @sql", async () => { const body = "HelloWorld"; diff --git a/tests/blob/https.test.ts b/tests/blob/https.test.ts index 108d71511..ceb89456e 100644 --- a/tests/blob/https.test.ts +++ b/tests/blob/https.test.ts @@ -1,12 +1,16 @@ import { + BlobSASPermissions, BlobServiceClient, newPipeline, StorageSharedKeyCredential } from "@azure/storage-blob"; +import * as assert from "assert"; import { configLogger } from "../../src/common/Logger"; import BlobTestServerFactory from "../BlobTestServerFactory"; import { + base64encode, + bodyToString, EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, getUniqueName @@ -51,4 +55,65 @@ describe("Blob HTTPS", () => { await containerClient.create(); await containerClient.delete(); }); + + it(`stageBlockFromURL should work using HTTPS endpoint @loki @sql`, async () => { + // stageBlockFromURL fetches the copy source from this same server, so on + // an HTTPS endpoint that self-request is itself HTTPS. This pins that the + // scheme is derived from the incoming socket and that the self-request + // succeeds against the certificate Azurite was started with. + // + // Note: the npm test scripts set NODE_TLS_REJECT_UNAUTHORIZED=0, which + // disables certificate verification process-wide, so this test cannot + // detect a regression in the self-request's own TLS handling - it would + // pass even if that request rejected Azurite's self-signed certificate. + // Verifying that requires running the server as a separate process + // without the variable set. + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + } + ) + ); + + const containerClient = serviceClient.getContainerClient( + getUniqueName("container") + ); + await containerClient.create(); + + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + await targetClient.commitBlockList([base64encode("1")]); + + const result = await targetClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + + await containerClient.delete(); + }); }); From c56662696b4130f43b23d17060f4e716c65bc810 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 24 Aug 2026 11:28:51 -0700 Subject: [PATCH 2/3] Pin the loopback self-request to this server's certificate Replace rejectUnauthorized: false with an https.Agent that trusts exactly the certificate this server presents, read from the socket the request arrived on, so a substituted certificate is still rejected. Hostname verification stays off because the request targets the bound address rather than a name. Addresses the CodeQL disabling-certificate-validation alert. Co-Authored-By: Claude Opus 5 (1M context) --- src/blob/handlers/BlockBlobHandler.ts | 58 +++++++++++++++++++++------ tests/blob/https.test.ts | 9 +++-- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 2f6db0863..dfbe53929 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,6 +1,7 @@ import axios, { AxiosResponse } from "axios"; import { IncomingMessage } from "http"; import { Agent } from "https"; +import { TLSSocket } from "tls"; import { Readable } from "stream"; @@ -26,14 +27,46 @@ import { } from "../utils/utils"; /** - * Agent for the loopback self-request stageBlockFromURL makes to read a copy - * source. Shared so requests reuse one Agent rather than allocating their - * own, not for socket reuse: keep-alive stays off, taking a loopback - * handshake per request over an idle socket that the server may close - * mid-reuse, which would surface as a spurious CannotVerifyCopySource. See - * the call site for why verification is off. + * Agents for the loopback self-request stageBlockFromURL makes to read a copy + * source, keyed by the certificate they pin. Shared so requests reuse one + * Agent rather than allocating their own, not for socket reuse: keep-alive + * stays off, taking a loopback handshake per request over an idle socket that + * the server may close mid-reuse, which would surface as a spurious + * CannotVerifyCopySource. */ -const LOOPBACK_HTTPS_AGENT = new Agent({ rejectUnauthorized: false }); +const LOOPBACK_HTTPS_AGENTS = new Map(); + +/** + * Build the Agent for a loopback self-request over HTTPS. + * + * The request is pinned to the address and port it arrived on, so the peer is + * necessarily this same server. Rather than turning certificate validation + * off, trust exactly the one certificate this server presents: it is read + * from the accepted socket, so a certificate substituted on the wire is still + * rejected. Azurite is normally run with a self-signed certificate under + * --cert/--key, which no public trust store would accept. + * + * Hostname verification is skipped because the request deliberately targets + * the bound address rather than a name the certificate could carry, and the + * pinned certificate already identifies the peer. + */ +function getLoopbackHttpsAgent(socket: TLSSocket): Agent { + const certificate = socket.getCertificate(); + if (certificate === null || !("raw" in certificate)) { + throw new Error("Could not read the local TLS certificate"); + } + const der = certificate.raw.toString("base64"); + let agent = LOOPBACK_HTTPS_AGENTS.get(der); + if (agent === undefined) { + const pem = + `-----BEGIN CERTIFICATE-----\n` + + `${der.replace(/(.{64})/g, "$1\n")}\n` + + `-----END CERTIFICATE-----\n`; + agent = new Agent({ ca: pem, checkServerIdentity: () => undefined }); + LOOPBACK_HTTPS_AGENTS.set(der, agent); + } + return agent; +} /** * BlobHandler handles Azure Storage BlockBlob related requests. @@ -442,12 +475,11 @@ export default class BlockBlobHandler // and would fail outright when the property does not match the // bytes. decompress: false, - // The connection is pinned above to the address and port this very - // request arrived on, so the peer is necessarily this same server. - // Verifying its certificate would only reject the self-signed certs - // Azurite is normally run with under --cert/--key, which would make - // this operation unusable over HTTPS. - httpsAgent: scheme === "https" ? LOOPBACK_HTTPS_AGENT : undefined + // Pin trust to the certificate this server itself presents; see + // getLoopbackHttpsAgent(). + httpsAgent: scheme === "https" + ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) + : undefined }); } catch (err) { // Transport-level failures (TLS, connection reset, socket errors) throw diff --git a/tests/blob/https.test.ts b/tests/blob/https.test.ts index ceb89456e..1265637d7 100644 --- a/tests/blob/https.test.ts +++ b/tests/blob/https.test.ts @@ -64,10 +64,11 @@ describe("Blob HTTPS", () => { // // Note: the npm test scripts set NODE_TLS_REJECT_UNAUTHORIZED=0, which // disables certificate verification process-wide, so this test cannot - // detect a regression in the self-request's own TLS handling - it would - // pass even if that request rejected Azurite's self-signed certificate. - // Verifying that requires running the server as a separate process - // without the variable set. + // detect a regression in the self-request pinning the certificate this + // server presents - it would pass even if that request rejected the + // certificate. Verifying that requires running the server without the + // variable set and with a certificate that is still valid, since + // tests/server.cert has expired. const serviceClient = new BlobServiceClient( baseURL, newPipeline( From 153240ddf93218eba5ebf1e0e10cc371ea4230d7 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 24 Aug 2026 11:35:13 -0700 Subject: [PATCH 3/3] Clarify the stageBlockFromURL checksum comment Co-Authored-By: Claude Opus 5 (1M context) --- src/blob/handlers/BlockBlobHandler.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index dfbe53929..3886564e2 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -541,9 +541,11 @@ export default class BlockBlobHandler } // Compare the supplied source checksums against the fetched bytes with - // the same helper Put Block uses. The MD5 is always computed because the - // response echoes it; the CRC64 is computed when it can be reported, - // which mirrors stageBlock (the two checksums are mutually exclusive). + // the same helper Put Block uses. The response always echoes an MD5, so + // that one is always computed. A CRC64 is additionally computed - and so + // additionally echoed - only when no source MD5 was supplied, mirroring + // stageBlock; the two source checksum headers are mutually exclusive, so + // a supplied source MD5 means the caller cannot have asked for CRC64. // The header shapes were already rejected above, so the only failures // left are mismatches, which happen after the stream has been read. // Destroy it regardless so a throw cannot leave the extent handle open.