diff --git a/ChangeLog.md b/ChangeLog.md index 48fed1911..021de054f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -58,6 +58,7 @@ Blob: - Fixed issue #2672 startup failures with legacy persisted data by adding backward-compatible restore for persisted `contentMD5` formats. - Added CRC-64/NVME transactional checksum support for `StageBlock`, `PutBlock`, `PutBlob`, `AppendBlock`, and `PutPage` (`x-ms-content-crc64`). - Harden transactional checksum validation for `PutBlob`, `StageBlock`, `AppendBlock`, and `PutPage`: unified MD5/CRC64 validation logic with accurate `InvalidMd5`/`InvalidHeaderValue` (malformed) and `Md5Mismatch`/`Crc64Mismatch` (mismatch) errors, matching real Azure semantics verified against live. +- Implement `PutBlockFromURL` (`Put Block From URL`), which previously returned 501. The source is fetched over loopback so that SAS authentication, `x-ms-source-range`, and the `x-ms-source-if-*` conditions are enforced by the existing download path; unmet source conditions return 412 `SourceConditionNotMet`. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. - Fix `x-ms-blob-content-md5` precedence over `Content-MD5` for `PutBlob` transit integrity verification, matching real Azure behavior. - Make `CopyBlobFromURL` echo back the source `Content-MD5` when supplied via `x-ms-source-content-md5`, matching real Azure behavior. - Added support for the `startFrom` query parameter on `List Blobs` (service version `2026-02-06`), which begins a flat or hierarchical listing at the given blob name. Unlike `marker`, which is exclusive, `startFrom` is inclusive, and the two compose when paging a listing that began at `startFrom`. Previously the parameter was accepted but ignored. diff --git a/README.md b/README.md index bc80aa460..191180e4b 100644 --- a/README.md +++ b/README.md @@ -1076,6 +1076,7 @@ Detailed support matrix: - Copy Blob (Only supports copy within same Azurite instance) - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) + - Put Block From URL (Only supports source within same Azurite instance) - Access control based on conditional headers - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite diff --git a/src/blob/errors/StorageErrorFactory.ts b/src/blob/errors/StorageErrorFactory.ts index a0c4e897b..9c14e6944 100644 --- a/src/blob/errors/StorageErrorFactory.ts +++ b/src/blob/errors/StorageErrorFactory.ts @@ -791,6 +791,15 @@ export default class StorageErrorFactory { ); } + public static getSourceConditionNotMet(contextID: string): StorageError { + return new StorageError( + 412, + "SourceConditionNotMet", + "The source condition specified using HTTP conditional header(s) is not met.", + contextID + ); + } + public static getInvalidResourceName(contextID: string = ""): StorageError { return new StorageError( 400, diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 666a067f0..3886564e2 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,3 +1,11 @@ +import axios, { AxiosResponse } from "axios"; +import { IncomingMessage } from "http"; +import { Agent } from "https"; +import { TLSSocket } from "tls"; + +import { Readable } from "stream"; + +import { IExtentChunk } from "../../common/persistence/IExtentStore"; import { convertRawHeadersToMetadata, getMD5FromString, @@ -18,6 +26,48 @@ import { getTagsFromString } from "../utils/utils"; +/** + * Agents for the loopback self-request stageBlockFromURL makes to read a copy + * source, keyed by the certificate they pin. Shared so requests reuse one + * Agent rather than allocating their own, not for socket reuse: keep-alive + * stays off, taking a loopback handshake per request over an idle socket that + * the server may close mid-reuse, which would surface as a spurious + * CannotVerifyCopySource. + */ +const LOOPBACK_HTTPS_AGENTS = new Map(); + +/** + * Build the Agent for a loopback self-request over HTTPS. + * + * The request is pinned to the address and port it arrived on, so the peer is + * necessarily this same server. Rather than turning certificate validation + * off, trust exactly the one certificate this server presents: it is read + * from the accepted socket, so a certificate substituted on the wire is still + * rejected. Azurite is normally run with a self-signed certificate under + * --cert/--key, which no public trust store would accept. + * + * Hostname verification is skipped because the request deliberately targets + * the bound address rather than a name the certificate could carry, and the + * pinned certificate already identifies the peer. + */ +function getLoopbackHttpsAgent(socket: TLSSocket): Agent { + const certificate = socket.getCertificate(); + if (certificate === null || !("raw" in certificate)) { + throw new Error("Could not read the local TLS certificate"); + } + const der = certificate.raw.toString("base64"); + let agent = LOOPBACK_HTTPS_AGENTS.get(der); + if (agent === undefined) { + const pem = + `-----BEGIN CERTIFICATE-----\n` + + `${der.replace(/(.{64})/g, "$1\n")}\n` + + `-----END CERTIFICATE-----\n`; + agent = new Agent({ ca: pem, checkServerIdentity: () => undefined }); + LOOPBACK_HTTPS_AGENTS.set(der, agent); + } + return agent; +} + /** * BlobHandler handles Azure Storage BlockBlob related requests. * @@ -261,7 +311,293 @@ export default class BlockBlobHandler options: Models.BlockBlobStageBlockFromURLOptionalParams, context: Context ): Promise { - throw new NotImplementedError(context.contextId); + const blobCtx = new BlobStorageContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + // Put Block From URL carries no request body. + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "Content-Length", + HeaderValue: contentLength.toString() + }); + } + + this.validateBlockId(blockId, blobCtx); + + // Reject malformed source checksum headers before fetching anything. The + // shared validator would catch these too, but it reports the names of the + // transactional headers, and its errors would surface only after the + // source had already been read and staged. + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentcrc64 !== undefined + ) { + throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( + context.contextId + ); + } + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentMD5.length !== 16 + ) { + throw StorageErrorFactory.getInvalidMd5(context.contextId); + } + if ( + options.sourceContentcrc64 !== undefined && + options.sourceContentcrc64.length < 8 + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-source-content-crc64", + HeaderValue: Buffer.from(options.sourceContentcrc64).toString("base64") + }); + } + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + let url: URL; + try { + url = new URL(sourceUrl); + } catch { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source", + HeaderValue: sourceUrl + }); + } + + // Only sources within the same Azurite instance are supported, as with + // copyFromURL. + // Hostnames compare case-insensitively and new URL() lowercases its + // host, so normalize the client-supplied header before comparing. + const currentServer = (blobCtx.request!.getHeader("Host") || "") + .toLowerCase(); + if (currentServer !== url.host) { + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Source ${url} is not on the same Azurite instance as target account ${accountName}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + // The Host header above is client-controlled, so never fetch the + // caller-supplied URL directly; pin the outbound request to the + // loopback address and port this server is actually bound to, keeping + // only the caller's path and query. + const rawRequest = blobCtx.request!.getBodyStream(); + if (!(rawRequest instanceof IncomingMessage) || + rawRequest.socket.localPort === undefined) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; + // Use the local address this request arrived on rather than a + // hard-coded loopback so non-loopback --blobHost binds keep working; + // IPv6 literals need brackets in URLs. + const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; + const localHost = localAddress.includes(":") ? + `[${localAddress}]` : localAddress; + const pinnedUrl = + `${scheme}://${localHost}:${rawRequest.socket.localPort}` + + `${url.pathname}${url.search}`; + + // Fetch the source range over loopback so that SAS authentication, + // range handling, and source conditions reuse the download path. + // Preserve the source URL's host so product-style source URLs still + // resolve their account from the Host header; the connection itself + // stays pinned to this server's bound address. + // A block must be staged as the bytes the source actually stores, so ask + // for the body verbatim rather than letting anything in the path apply + // transfer compression. + const headers: { [key: string]: string } = { + host: url.host, + "accept-encoding": "identity" + }; + if (options.sourceRange !== undefined) { + // The download path ignores malformed Range headers, which would + // silently stage the entire source blob; reject them up front. + // Compare offsets as BigInt: they are 64-bit, and Number rounds + // above 2^53, which would let an end < start range slip through. + const rangeMatch = /^bytes=(\d+)-(\d*)$/.exec(options.sourceRange); + if (rangeMatch === null || + (rangeMatch[2] !== "" && + BigInt(rangeMatch[2]) < BigInt(rangeMatch[1]))) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-source-range", + HeaderValue: options.sourceRange + }); + } + headers.range = options.sourceRange; + } + const sourceConditions = options.sourceModifiedAccessConditions || {}; + if (sourceConditions.sourceIfMatch !== undefined) { + headers["if-match"] = sourceConditions.sourceIfMatch; + } + if (sourceConditions.sourceIfNoneMatch !== undefined) { + headers["if-none-match"] = sourceConditions.sourceIfNoneMatch; + } + if (sourceConditions.sourceIfModifiedSince !== undefined) { + headers["if-modified-since"] = new Date( + sourceConditions.sourceIfModifiedSince + ).toUTCString(); + } + if (sourceConditions.sourceIfUnmodifiedSince !== undefined) { + headers["if-unmodified-since"] = new Date( + sourceConditions.sourceIfUnmodifiedSince + ).toUTCString(); + } + // Note: unlike the Copy Blob operations, Put Block From URL has no + // x-ms-source-if-tags condition; the generated operation spec does not + // deserialize one. + + let sourceResponse: AxiosResponse; + try { + sourceResponse = await axios.get(pinnedUrl, { + headers, + responseType: "stream", + validateStatus: () => true, + // Never decompress. A source blob carries Content-Encoding as a + // stored property, so the download echoes it back even though the + // bytes on the wire are the raw stored ones. Decompressing here + // would stage the decoded content instead of what the source holds, + // and would fail outright when the property does not match the + // bytes. + decompress: false, + // Pin trust to the certificate this server itself presents; see + // getLoopbackHttpsAgent(). + httpsAgent: scheme === "https" + ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) + : undefined + }); + } catch (err) { + // Transport-level failures (TLS, connection reset, socket errors) throw + // rather than returning a status. Without this they would escape as a + // bodiless 500 instead of an Azure-shaped error. + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + if (sourceResponse.status === 304 || sourceResponse.status === 412) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); + } + if (sourceResponse.status === 404) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + sourceResponse.status, + "Could not verify the copy source within the specified time." + ); + } + + // The status above only means the response headers arrived; the body can + // still fail midway (socket error, connection reset). Map that to the + // same error as a transport failure rather than letting it escape as a + // bodiless 500, and release the source stream on the way out. + let persistency: IExtentChunk; + try { + persistency = await this.extentStore.appendExtent( + sourceResponse.data, + context.contextId + ); + } catch (err) { + sourceResponse.data.destroy(); + this.logger.error( + `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source body: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // Compare the supplied source checksums against the fetched bytes with + // the same helper Put Block uses. The response always echoes an MD5, so + // that one is always computed. A CRC64 is additionally computed - and so + // additionally echoed - only when no source MD5 was supplied, mirroring + // stageBlock; the two source checksum headers are mutually exclusive, so + // a supplied source MD5 means the caller cannot have asked for CRC64. + // The header shapes were already rejected above, so the only failures + // left are mismatches, which happen after the stream has been read. + // Destroy it regardless so a throw cannot leave the extent handle open. + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + let calculatedContentMD5: Uint8Array | undefined; + let calculatedContentCRC64: Uint8Array | undefined; + try { + ({ md5: calculatedContentMD5, crc64: calculatedContentCRC64 } = + await computeAndValidateTransactionalChecksums( + stream, + { + md5: options.sourceContentMD5, + crc64: options.sourceContentcrc64 + }, + context.contextId, + { md5: true, crc64: options.sourceContentMD5 === undefined } + )); + } finally { + (stream as Readable).destroy?.(); + } + + const block: BlockModel = { + accountName, + containerName, + blobName, + isCommitted: false, + name: blockId, + size: persistency.count, + persistency + }; + + await this.metadataStore.stageBlock( + context, + block, + options.leaseAccessConditions + ); + + const response: Models.BlockBlobStageBlockFromURLResponse = { + statusCode: 201, + contentMD5: calculatedContentMD5, + xMsContentCrc64: calculatedContentCRC64, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; } public async commitBlockList( diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index fd43e4a1e..1cd19661d 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -6,8 +6,10 @@ import { Tags } from "@azure/storage-blob"; import CustomHeaderPolicyFactory from "../RequestPolicy/CustomHeaderPolicyFactory"; +import axios from "axios"; import * as assert from "assert"; import * as crypto from "crypto"; +import * as zlib from "zlib"; import { configLogger } from "../../../src/common/Logger"; import BlobTestServerFactory from "../../BlobTestServerFactory"; @@ -421,6 +423,618 @@ describe("BlockBlobAPIs", () => { ); }); + it("stageBlockFromURL @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const resultStage = await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + 10 + ); + const expectedMD5 = await getMD5FromString(content.substring(0, 10)); + assert.deepStrictEqual( + Buffer.from(resultStage.contentMD5!), + Buffer.from(expectedMD5) + ); + + await blockBlobClient.stageBlockFromURL( + base64encode("2"), + sourceUrl, + 10, + content.length - 10 + ); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 2); + assert.equal(listResponse.uncommittedBlocks![0].size, 10); + assert.equal( + listResponse.uncommittedBlocks![1].size, + content.length - 10 + ); + + await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL without range copies the entire source @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + await blockBlobClient.commitBlockList([base64encode("1")]); + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL rejects an unmet source condition @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // @azure/storage-blob does not expose x-ms-source-if-* on + // stageBlockFromURL, so issue the request directly. + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-source-if-match": '"0x0000000000000000"', + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 412); + assert.ok(response.data.includes("SourceConditionNotMet")); + }); + + it("stageBlockFromURL rejects a request body @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + "unexpected body", + { + headers: { + "x-ms-copy-source": sourceUrl + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 400); + assert.ok(response.data.includes("InvalidHeaderValue")); + }); + + it("stageBlockFromURL rejects a malformed source range @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + for (const badRange of [ + "bytes=abc", + "bytes=5-2", + "0-10", + // end < start detectable only beyond double precision: both offsets + // round to 2^53 as Numbers, hiding that the range is inverted + "bytes=9007199254740993-9007199254740992" + ]) { + const response = await axios.put( + destinationUrl + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-source-range": badRange, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 400, badRange); + assert.ok(response.data.includes("InvalidHeaderValue"), badRange); + } + }); + + it("stageBlockFromURL with product-style source URL @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceName = getUniqueName("source"); + const sourceClient = containerClient.getBlockBlobClient(sourceName); + await sourceClient.upload(content, content.length); + const sourceSasQuery = (await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + const destinationSasQuery = (await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + + // Both requests use product-style URLs, where the account comes from + // the Host header rather than the path. + const productHost = + `${EMULATOR_ACCOUNT_NAME}.localhost:${server.config.port}`; + const response = await axios.put( + `http://${server.config.host}:${server.config.port}` + + `/${containerName}/${blobName}?${destinationSasQuery}` + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + host: productHost, + "x-ms-copy-source": + `http://${productHost}/${containerName}/${sourceName}` + + `?${sourceSasQuery}`, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 201); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, content.length); + }); + + it("stageBlockFromURL accepts a mixed-case Host header @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceName = getUniqueName("source"); + const sourceClient = containerClient.getBlockBlobClient(sourceName); + await sourceClient.upload(content, content.length); + const sourceSasQuery = (await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + const destinationSasQuery = (await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + })).split("?")[1]; + + // Host headers are case-insensitive; the lowercase source URL host + // must match despite the client's casing. + const response = await axios.put( + `http://${server.config.host}:${server.config.port}` + + `/${EMULATOR_ACCOUNT_NAME}/${containerName}/${blobName}` + + `?${destinationSasQuery}` + + "&comp=block&blockid=" + + encodeURIComponent(base64encode("1")), + undefined, + { + headers: { + host: `LocalHost:${server.config.port}`, + "x-ms-copy-source": + `http://localhost:${server.config.port}` + + `/${EMULATOR_ACCOUNT_NAME}/${containerName}/${sourceName}` + + `?${sourceSasQuery}`, + "Content-Length": "0" + }, + validateStatus: () => true + } + ); + assert.deepStrictEqual(response.status, 201); + }); + + it("stageBlockFromURL from a missing source returns 404 @loki @sql", async () => { + const missingClient = containerClient.getBlockBlobClient( + getUniqueName("missing") + ); + const sourceUrl = await missingClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl + ); + assert.fail(); + } catch (err: any) { + assert.deepStrictEqual(err.statusCode, 404); + assert.deepStrictEqual(err.details.errorCode, "CannotVerifyCopySource"); + } + }); + + it("stageBlockFromURL stages the stored bytes when the source declares Content-Encoding: gzip @loki @sql", async () => { + // A blob's Content-Encoding is stored metadata, not a description of how + // the body is framed on the wire, so the download echoes it back over the + // raw stored bytes. Staging must copy those bytes verbatim rather than + // decoding them, otherwise the block holds the decompressed content. + const raw = zlib.gzipSync(Buffer.from("HelloWorldFromSourceBlob")); + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(raw, raw.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, raw.length); + + await blockBlobClient.commitBlockList([base64encode("1")]); + const download = await blockBlobClient.download(0); + const chunks: Buffer[] = []; + for await (const chunk of download.readableStreamBody!) { + chunks.push(Buffer.from(chunk)); + } + assert.deepStrictEqual( + Buffer.concat(chunks), + raw, + "Staged block must be the source's stored bytes, not the decoded ones" + ); + }); + + it("stageBlockFromURL succeeds when the source's Content-Encoding does not match its bytes @loki @sql", async () => { + // Nothing validates that a blob's stored Content-Encoding describes its + // content, so a plain body can be labelled gzip. Staging must not try to + // decode it (see issue #646 for the same hazard on copy). + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.stageBlockFromURL(base64encode("1"), sourceUrl); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].size, content.length); + + await blockBlobClient.commitBlockList([base64encode("1")]); + const result = await blockBlobClient.download(0); + assert.equal(await bodyToString(result, content.length), content); + }); + + it("stageBlockFromURL with matching sourceContentMD5 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const resultStage = await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: new Uint8Array(md5) } + ); + + // The response echoes the MD5 the service computed over the staged + // content, which for a matching request equals the supplied value. + assert.deepStrictEqual( + Buffer.from(resultStage.contentMD5!), + Buffer.from(md5) + ); + // The two checksums are mutually exclusive, so no CRC64 is reported + // alongside an MD5, matching stageBlock. + assert.strictEqual((resultStage as any).xMsContentCrc64, undefined); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + }); + + it("stageBlockFromURL with wrong sourceContentMD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // Stage one good block first, so the block list below distinguishes "the + // rejected block was not staged" from "the blob does not exist yet". + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + + // A valid 16-byte MD5 of a *different* body, to exercise the mismatch + // path rather than the InvalidMd5 (wrong-length) path. + const md5 = crypto.createHash("md5").update("anotherBody", "utf8").digest(); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("2"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: new Uint8Array(md5) } + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + + // A rejected block must not be staged. + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal( + listResponse.uncommittedBlocks![0].name, + base64encode("1") + ); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with wrong-length sourceContentMD5 should be rejected @loki @sql", async () => { + // x-ms-source-content-md5 must decode to exactly 16 bytes. This test pins + // which error code the service returns for a malformed (4-byte) value so + // Azurite can be verified against real Azure. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const wrongLengthMd5 = new Uint8Array([0, 0, 0, 0]); + + try { + await blockBlobClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length, + { sourceContentMD5: wrongLengthMd5 } + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with matching sourceContentCrc64 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-crc64", value: crc64 }] + ); + + const resultStage = await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + + // The response reports the CRC64 the service computed over the staged + // content, as stageBlock does. + assert.equal( + Buffer.from((resultStage as any).xMsContentCrc64!).toString("base64"), + crc64 + ); + + const listResponse = await targetClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + }); + + it("stageBlockFromURL with wrong sourceContentCrc64 should throw crc64 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // A valid 8-byte CRC64 of a *different* body, to exercise the mismatch + // path rather than the malformed-header path. + const crc64 = Buffer.from(getCRC64FromString("anotherBody")).toString( + "base64" + ); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-crc64", value: crc64 }] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Crc64Mismatch"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlockFromURL with wrong-length sourceContentCrc64 should be rejected @loki @sql", async () => { + // x-ms-source-content-crc64 must decode to at least 8 bytes. This test + // pins which error code the service returns for a malformed (4-byte) + // value so Azurite can be verified against real Azure. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [ + { + key: "x-ms-source-content-crc64", + value: Buffer.from([1, 2, 3, 4]).toString("base64") + } + ] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidHeaderValue"); + // The error must name the header the caller actually sent, not the + // transactional x-ms-content-crc64 the shared validator reports. + assert.equal( + /([^<]*) { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // Both checksums are correct for the source content; supplying the two + // together is rejected regardless, as on the real service. + const md5 = Buffer.from(await getMD5FromString(content)).toString("base64"); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [ + { key: "x-ms-source-content-md5", value: md5 }, + { key: "x-ms-source-content-crc64", value: crc64 } + ] + ); + + try { + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "BothCrc64AndMd5HeaderPresent"); + return; + } + assert.fail("Did not throw an exception."); + }); + it("stageBlock with double commit block should work @loki @sql", async () => { const body = "HelloWorld"; diff --git a/tests/blob/https.test.ts b/tests/blob/https.test.ts index 108d71511..1265637d7 100644 --- a/tests/blob/https.test.ts +++ b/tests/blob/https.test.ts @@ -1,12 +1,16 @@ import { + BlobSASPermissions, BlobServiceClient, newPipeline, StorageSharedKeyCredential } from "@azure/storage-blob"; +import * as assert from "assert"; import { configLogger } from "../../src/common/Logger"; import BlobTestServerFactory from "../BlobTestServerFactory"; import { + base64encode, + bodyToString, EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, getUniqueName @@ -51,4 +55,66 @@ describe("Blob HTTPS", () => { await containerClient.create(); await containerClient.delete(); }); + + it(`stageBlockFromURL should work using HTTPS endpoint @loki @sql`, async () => { + // stageBlockFromURL fetches the copy source from this same server, so on + // an HTTPS endpoint that self-request is itself HTTPS. This pins that the + // scheme is derived from the incoming socket and that the self-request + // succeeds against the certificate Azurite was started with. + // + // Note: the npm test scripts set NODE_TLS_REJECT_UNAUTHORIZED=0, which + // disables certificate verification process-wide, so this test cannot + // detect a regression in the self-request pinning the certificate this + // server presents - it would pass even if that request rejected the + // certificate. Verifying that requires running the server without the + // variable set and with a certificate that is still valid, since + // tests/server.cert has expired. + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + } + ) + ); + + const containerClient = serviceClient.getContainerClient( + getUniqueName("container") + ); + await containerClient.create(); + + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = containerClient.getBlockBlobClient( + getUniqueName("target") + ); + await targetClient.stageBlockFromURL( + base64encode("1"), + sourceUrl, + 0, + content.length + ); + await targetClient.commitBlockList([base64encode("1")]); + + const result = await targetClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + + await containerClient.delete(); + }); });