diff --git a/ChangeLog.md b/ChangeLog.md index b81bb1c6a..874cfa65d 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -61,6 +61,7 @@ Blob: - Fixed issue #2672 startup failures with legacy persisted data by adding backward-compatible restore for persisted `contentMD5` formats. - Added CRC-64/NVME transactional checksum support for `StageBlock`, `PutBlock`, `PutBlob`, `AppendBlock`, and `PutPage` (`x-ms-content-crc64`). - Harden transactional checksum validation for `PutBlob`, `StageBlock`, `AppendBlock`, and `PutPage`: unified MD5/CRC64 validation logic with accurate `InvalidMd5`/`InvalidHeaderValue` (malformed) and `Md5Mismatch`/`Crc64Mismatch` (mismatch) errors, matching real Azure semantics verified against live. +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. - Implement `PutBlockFromURL` (`Put Block From URL`), which previously returned 501. The source is fetched over loopback so that SAS authentication, `x-ms-source-range`, and the `x-ms-source-if-*` conditions are enforced by the existing download path; unmet source conditions return 412 `SourceConditionNotMet`. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. - Fix `x-ms-blob-content-md5` precedence over `Content-MD5` for `PutBlob` transit integrity verification, matching real Azure behavior. - Make `CopyBlobFromURL` echo back the source `Content-MD5` when supplied via `x-ms-source-content-md5`, matching real Azure behavior. diff --git a/README.md b/README.md index 191180e4b..13ff709d9 100644 --- a/README.md +++ b/README.md @@ -1077,6 +1077,7 @@ Detailed support matrix: - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Put Block From URL (Only supports source within same Azurite instance) + - Put Blob From URL (Only supports source within same Azurite instance) - Access control based on conditional headers - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite @@ -1090,7 +1091,6 @@ Detailed support matrix: - Concurrent Append - Blob Expiry - Object Replication Service - - Put Blob From URL - Version Level Worm - Sync copy blob by access source with oauth - Encryption Scope diff --git a/src/blob/authentication/AccountSASAuthenticator.ts b/src/blob/authentication/AccountSASAuthenticator.ts index 48b850b07..7cbb4247a 100644 --- a/src/blob/authentication/AccountSASAuthenticator.ts +++ b/src/blob/authentication/AccountSASAuthenticator.ts @@ -256,6 +256,7 @@ export default class AccountSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || diff --git a/src/blob/authentication/BlobSASAuthenticator.ts b/src/blob/authentication/BlobSASAuthenticator.ts index ac848b0f8..e0d575b61 100644 --- a/src/blob/authentication/BlobSASAuthenticator.ts +++ b/src/blob/authentication/BlobSASAuthenticator.ts @@ -418,6 +418,7 @@ export default class BlobSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || diff --git a/src/blob/authentication/OperationAccountSASPermission.ts b/src/blob/authentication/OperationAccountSASPermission.ts index 56cd123f8..979668078 100644 --- a/src/blob/authentication/OperationAccountSASPermission.ts +++ b/src/blob/authentication/OperationAccountSASPermission.ts @@ -363,6 +363,16 @@ OPERATION_ACCOUNT_SAS_PERMISSIONS.set( ) ); +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create permission is only available for nonexistent block blob. Handle this scenario separately + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + OPERATION_ACCOUNT_SAS_PERMISSIONS.set( Operation.PageBlob_Create, new OperationAccountSASPermission( diff --git a/src/blob/authentication/OperationBlobSASPermission.ts b/src/blob/authentication/OperationBlobSASPermission.ts index ddf21c746..cced5f88f 100644 --- a/src/blob/authentication/OperationBlobSASPermission.ts +++ b/src/blob/authentication/OperationBlobSASPermission.ts @@ -258,6 +258,13 @@ OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) @@ -522,6 +529,13 @@ OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // Create a new blob, must be write + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 3886564e2..6e716076b 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -12,7 +12,6 @@ import { newEtag } from "../../common/utils/utils"; import BlobStorageContext from "../context/BlobStorageContext"; -import NotImplementedError from "../errors/NotImplementedError"; import StorageErrorFactory from "../errors/StorageErrorFactory"; import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; @@ -208,9 +207,222 @@ export default class BlockBlobHandler return response; } - public async putBlobFromUrl(contentLength: number, copySource: string, options: Models.BlockBlobPutBlobFromUrlOptionalParams, context: Context + public async putBlobFromUrl( + contentLength: number, + copySource: string, + options: Models.BlockBlobPutBlobFromUrlOptionalParams, + context: Context ): Promise { - throw new NotImplementedError(context.contextId); + const blobCtx = new BlobStorageContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + const etag = newEtag(); + + // Put Blob From URL carries no request body. + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "Content-Length", + HeaderValue: contentLength.toString() + }); + } + + // Reject a malformed source checksum before fetching anything. The + // shared validator would catch it too, but only once the source had + // already been read. + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentMD5.length !== 16 + ) { + throw StorageErrorFactory.getInvalidMd5(context.contextId); + } + + // The destination's tags are either the source's or the request's, never + // both. + const copySourceTags = + options.copySourceTags === Models.BlobCopySourceTags.COPY; + if (copySourceTags && options.blobTagsString !== undefined) { + throw StorageErrorFactory.getBothUserTagsAndSourceTagsCopyPresentException( + context.contextId! + ); + } + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + // Put Blob From URL always copies the whole source, so no range rides + // along with the conditions. + const sourceResponse = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); + + // The status was only the response headers arriving; the body can still + // fail midway (socket error, connection reset). Map that to the same + // error as a transport failure rather than letting it escape as a + // bodiless 500, and release the source stream on the way out. + let persistency: IExtentChunk; + try { + persistency = await this.extentStore.appendExtent( + sourceResponse.data, + context.contextId + ); + } catch (err) { + sourceResponse.data.destroy(); + this.logger.error( + `BlockBlobHandler:putBlobFromUrl() Failed to read the copy source body: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // The response always echoes an MD5 of what was copied, so it is always + // computed. x-ms-source-content-md5 is this operation's integrity check + // over the bytes that arrived; x-ms-blob-content-md5 gets the same + // treatment Put Blob gives it, since Put Blob From URL follows Put Blob + // for the custom properties. Destroy the stream regardless, so a + // mismatch cannot leave the extent handle open. + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + let calculatedContentMD5: Uint8Array | undefined; + try { + ({ md5: calculatedContentMD5 } = + await computeAndValidateTransactionalChecksums( + stream, + { + md5: + options.sourceContentMD5 ?? + (options.blobHTTPHeaders || {}).blobContentMD5 + }, + context.contextId, + { md5: true } + )); + } finally { + (stream as Readable).destroy?.(); + } + + // COPY reads the source's tags over the same authorized path the content + // came over, so a source that the caller may read but not tag refuses + // the copy rather than leaking them. + const blobTags = copySourceTags + ? await this.readCopySourceTags(context, copySource) + : options.blobTagsString === undefined + ? undefined + : getTagsFromString(options.blobTagsString, context.contextId!); + + // The standard properties are copied from the source unless the request + // turns that off, and a blob content header on the request sets that one + // property either way. The request's own Content-Type is only a + // fallback: a client sends one on a bodiless request without meaning to + // retype the copy. + const copyProperties = options.copySourceBlobProperties !== false; + const sourceProperty = (name: string): string | undefined => + copyProperties ? sourceResponse.headers[name] : undefined; + const blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + blobHTTPHeaders.blobContentType || + sourceProperty("content-type") || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + // Metadata named on the request replaces the source's rather than adding + // to it, and naming none copies the source's. Both are read from raw + // headers, which preserve the case of the names. + const metadata = + convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders(), + context.contextId! + ) ?? + convertRawHeadersToMetadata( + (sourceResponse.data as IncomingMessage).rawHeaders, + context.contextId! + ); + + const blob: BlobModel = { + deleted: false, + metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + // The destination's length is the source's, not the Content-Length + // of this bodiless request. + contentLength: persistency.count, + contentType, + contentEncoding: + blobHTTPHeaders.blobContentEncoding || + sourceProperty("content-encoding"), + contentLanguage: + blobHTTPHeaders.blobContentLanguage || + sourceProperty("content-language"), + contentMD5: calculatedContentMD5, + contentDisposition: + blobHTTPHeaders.blobContentDisposition || + sourceProperty("content-disposition"), + cacheControl: + blobHTTPHeaders.blobCacheControl || sourceProperty("cache-control"), + blobType: Models.BlobType.BlockBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + accessTierChangeTime: date + }, + snapshot: "", + isCommitted: true, + persistency, + blobTags + }; + + if (options.tier !== undefined) { + blob.properties.accessTier = this.parseTier(options.tier); + if (blob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${options.tier}` + }); + } + blob.properties.accessTierInferred = false; + } + + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.BlockBlobPutBlobFromUrlResponse = { + statusCode: 201, + eTag: etag, + lastModified: date, + contentMD5: blob.properties.contentMD5, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; } public async stageBlock( @@ -361,70 +573,7 @@ export default class BlockBlobHandler containerName ); - let url: URL; - try { - url = new URL(sourceUrl); - } catch { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-copy-source", - HeaderValue: sourceUrl - }); - } - - // Only sources within the same Azurite instance are supported, as with - // copyFromURL. - // Hostnames compare case-insensitively and new URL() lowercases its - // host, so normalize the client-supplied header before comparing. - const currentServer = (blobCtx.request!.getHeader("Host") || "") - .toLowerCase(); - if (currentServer !== url.host) { - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Source ${url} is not on the same Azurite instance as target account ${accountName}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - - // The Host header above is client-controlled, so never fetch the - // caller-supplied URL directly; pin the outbound request to the - // loopback address and port this server is actually bound to, keeping - // only the caller's path and query. - const rawRequest = blobCtx.request!.getBodyStream(); - if (!(rawRequest instanceof IncomingMessage) || - rawRequest.socket.localPort === undefined) { - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; - // Use the local address this request arrived on rather than a - // hard-coded loopback so non-loopback --blobHost binds keep working; - // IPv6 literals need brackets in URLs. - const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; - const localHost = localAddress.includes(":") ? - `[${localAddress}]` : localAddress; - const pinnedUrl = - `${scheme}://${localHost}:${rawRequest.socket.localPort}` + - `${url.pathname}${url.search}`; - - // Fetch the source range over loopback so that SAS authentication, - // range handling, and source conditions reuse the download path. - // Preserve the source URL's host so product-style source URLs still - // resolve their account from the Host header; the connection itself - // stays pinned to this server's bound address. - // A block must be staged as the bytes the source actually stores, so ask - // for the body verbatim rather than letting anything in the path apply - // transfer compression. - const headers: { [key: string]: string } = { - host: url.host, - "accept-encoding": "identity" - }; + const headers: { [key: string]: string } = {}; if (options.sourceRange !== undefined) { // The download path ignores malformed Range headers, which would // silently stage the entire source blob; reject them up front. @@ -441,83 +590,24 @@ export default class BlockBlobHandler } headers.range = options.sourceRange; } - const sourceConditions = options.sourceModifiedAccessConditions || {}; - if (sourceConditions.sourceIfMatch !== undefined) { - headers["if-match"] = sourceConditions.sourceIfMatch; - } - if (sourceConditions.sourceIfNoneMatch !== undefined) { - headers["if-none-match"] = sourceConditions.sourceIfNoneMatch; - } - if (sourceConditions.sourceIfModifiedSince !== undefined) { - headers["if-modified-since"] = new Date( - sourceConditions.sourceIfModifiedSince - ).toUTCString(); - } - if (sourceConditions.sourceIfUnmodifiedSince !== undefined) { - headers["if-unmodified-since"] = new Date( - sourceConditions.sourceIfUnmodifiedSince - ).toUTCString(); - } // Note: unlike the Copy Blob operations, Put Block From URL has no // x-ms-source-if-tags condition; the generated operation spec does not // deserialize one. + Object.assign( + headers, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); - let sourceResponse: AxiosResponse; - try { - sourceResponse = await axios.get(pinnedUrl, { - headers, - responseType: "stream", - validateStatus: () => true, - // Never decompress. A source blob carries Content-Encoding as a - // stored property, so the download echoes it back even though the - // bytes on the wire are the raw stored ones. Decompressing here - // would stage the decoded content instead of what the source holds, - // and would fail outright when the property does not match the - // bytes. - decompress: false, - // Pin trust to the certificate this server itself presents; see - // getLoopbackHttpsAgent(). - httpsAgent: scheme === "https" - ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) - : undefined - }); - } catch (err) { - // Transport-level failures (TLS, connection reset, socket errors) throw - // rather than returning a status. Without this they would escape as a - // bodiless 500 instead of an Azure-shaped error. - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source: ${err}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 500, - "Could not verify the copy source within the specified time." - ); - } - - if (sourceResponse.status === 304 || sourceResponse.status === 412) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); - } - if (sourceResponse.status === 404) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - sourceResponse.status, - "Could not verify the copy source within the specified time." - ); - } + const sourceResponse = await this.readCopySource( + context, + "stageBlockFromURL", + sourceUrl, + headers + ); - // The status above only means the response headers arrived; the body can + // The status was only the response headers arriving; the body can // still fail midway (socket error, connection reset). Map that to the // same error as a transport failure rather than letting it escape as a // bodiless 500, and release the source stream on the way out. @@ -829,4 +919,234 @@ export default class BlockBlobHandler ); } } + + /** + * Restate the conditions a copy request names for its source as the + * conditional headers of a read, so that the download path answers them + * the way it answers a client reading the source itself. + * + * x-ms-if-tags only ever appears for Put Blob From URL: the Put Block From + * URL specification deserializes no source tag condition. + * + * @private + * @param {Models.SourceModifiedAccessConditions} [conditions] + * @returns {{ [key: string]: string }} + * @memberof BlockBlobHandler + */ + private static sourceConditionHeaders( + conditions: Models.SourceModifiedAccessConditions = {} + ): { [key: string]: string } { + const headers: { [key: string]: string } = {}; + if (conditions.sourceIfMatch !== undefined) { + headers["if-match"] = conditions.sourceIfMatch; + } + if (conditions.sourceIfNoneMatch !== undefined) { + headers["if-none-match"] = conditions.sourceIfNoneMatch; + } + if (conditions.sourceIfModifiedSince !== undefined) { + headers["if-modified-since"] = new Date( + conditions.sourceIfModifiedSince + ).toUTCString(); + } + if (conditions.sourceIfUnmodifiedSince !== undefined) { + headers["if-unmodified-since"] = new Date( + conditions.sourceIfUnmodifiedSince + ).toUTCString(); + } + if (conditions.sourceIfTags !== undefined) { + headers["x-ms-if-tags"] = conditions.sourceIfTags; + } + return headers; + } + + /** + * Read a copy source with a loopback self-request, so that SAS + * authentication, ranges, and source conditions are answered by the + * download path rather than reimplemented against the store. + * + * Only sources within the same Azurite instance are supported, as with + * copyFromURL. The Host header that decides this is the caller's to + * choose, so the request is never made to the URL they supplied: it is + * pinned to the address and port this server is bound to and keeps only + * their path and query, with the source's own host along as a header so + * that product-style source URLs still resolve their account from it. + * + * @private + * @param {Context} context + * @param {string} operation Handler method name, for log messages + * @param {string} copySource The source URL the request named + * @param {{ [key: string]: string }} headers Conditions, ranges + * @param {string} [subresource] A query to append, such as "comp=tags" + * @returns {Promise} A response whose body is a stream + * @memberof BlockBlobHandler + */ + private async readCopySource( + context: Context, + operation: string, + copySource: string, + headers: { [key: string]: string }, + subresource?: string + ): Promise { + const blobCtx = new BlobStorageContext(context); + + let url: URL; + try { + url = new URL(copySource); + } catch { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source", + HeaderValue: copySource + }); + } + + // Hostnames compare case-insensitively and new URL() lowercases its + // host, so normalize the client-supplied header before comparing. + const currentServer = (blobCtx.request!.getHeader("Host") || "") + .toLowerCase(); + if (currentServer !== url.host) { + this.logger.error( + `BlockBlobHandler:${operation}() Source ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + const rawRequest = blobCtx.request!.getBodyStream(); + if (!(rawRequest instanceof IncomingMessage) || + rawRequest.socket.localPort === undefined) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; + // Use the local address this request arrived on rather than a + // hard-coded loopback so non-loopback --blobHost binds keep working; + // IPv6 literals need brackets in URLs. + const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; + const localHost = localAddress.includes(":") ? + `[${localAddress}]` : localAddress; + // Append rather than rebuild the query: a shared access signature signs + // the exact encoding it arrived in, which re-encoding could disturb. + const query = subresource === undefined + ? url.search + : `${url.search}${url.search === "" ? "?" : "&"}${subresource}`; + const pinnedUrl = + `${scheme}://${localHost}:${rawRequest.socket.localPort}` + + `${url.pathname}${query}`; + + let sourceResponse: AxiosResponse; + try { + sourceResponse = await axios.get(pinnedUrl, { + headers: { + host: url.host, + // A copy must carry the bytes the source actually stores, so ask + // for the body verbatim rather than letting anything in the path + // apply transfer compression. + "accept-encoding": "identity", + ...headers + }, + responseType: "stream", + validateStatus: () => true, + // Never decompress. A source blob carries Content-Encoding as a + // stored property, so the download echoes it back even though the + // bytes on the wire are the raw stored ones. Decompressing here + // would copy the decoded content instead of what the source holds, + // and would fail outright when the property does not match the + // bytes. + decompress: false, + // Pin trust to the certificate this server itself presents; see + // getLoopbackHttpsAgent(). + httpsAgent: scheme === "https" + ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) + : undefined + }); + } catch (err) { + // Transport-level failures (TLS, connection reset, socket errors) throw + // rather than returning a status. Without this they would escape as a + // bodiless 500 instead of an Azure-shaped error. + this.logger.error( + `BlockBlobHandler:${operation}() Failed to read the copy source: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + if (sourceResponse.status === 304 || sourceResponse.status === 412) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); + } + if (sourceResponse.status === 404) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + sourceResponse.status, + "Could not verify the copy source within the specified time." + ); + } + + return sourceResponse; + } + + /** + * Read the tags of a copy source, for the copy that asks to carry them + * over. Real Azure charges this to the caller as its own Get Blob Tags + * request against the source, and so does this: the authorization the + * source URL carries has to allow reading them. + * + * @private + * @param {Context} context + * @param {string} copySource The source URL the request named + * @returns {Promise} + * @memberof BlockBlobHandler + */ + private async readCopySourceTags( + context: Context, + copySource: string + ): Promise { + const response = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + {}, + "comp=tags" + ); + + const chunks: Buffer[] = []; + for await (const chunk of response.data as IncomingMessage) { + chunks.push(Buffer.from(chunk)); + } + const parsed = await parseXML(Buffer.concat(chunks).toString()); + + // parseXML collapses a single element out of its array, and leaves a + // tagless source with no TagSet at all. + const tagSet = parsed.TagSet; + if (tagSet === undefined || tagSet === "" || tagSet.Tag === undefined) { + return undefined; + } + const tags = Array.isArray(tagSet.Tag) ? tagSet.Tag : [tagSet.Tag]; + return { + blobTagSet: tags.map((tag: { Key: string; Value: string }) => ({ + key: tag.Key, + value: tag.Value + })) + }; + } } diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 1cd19661d..212accf12 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1035,6 +1035,509 @@ describe("BlockBlobAPIs", () => { assert.fail("Did not throw an exception."); }); + it("putBlobFromUrl copies the source blob @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const result = await blockBlobClient.syncUploadFromURL(sourceUrl); + // The response echoes the MD5 the service computed over what it copied. + assert.deepStrictEqual( + Buffer.from(result.contentMD5!), + Buffer.from(await getMD5FromString(content)) + ); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + // The destination's length is the source's, not the Content-Length of + // the bodiless request that created it. + assert.equal(download.contentLength, content.length); + }); + + it("putBlobFromUrl overwrites an existing destination @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("overwritten", "overwritten".length); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl copies the source's properties and metadata @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const properties = { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }; + const metadata = { keya: "vala", keyb: "valb" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: properties, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, properties.blobCacheControl); + assert.equal( + download.contentDisposition, + properties.blobContentDisposition + ); + assert.equal(download.contentEncoding, properties.blobContentEncoding); + assert.equal(download.contentLanguage, properties.blobContentLanguage); + assert.equal(download.contentType, properties.blobContentType); + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl with copySourceBlobProperties false leaves the source's properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const metadata = { keya: "vala" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceBlobProperties: false + }); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, undefined); + assert.equal(download.contentDisposition, undefined); + assert.equal(download.contentEncoding, undefined); + assert.equal(download.contentLanguage, undefined); + assert.equal(download.contentType, "application/octet-stream"); + // Metadata answers to its own rule rather than to this header: the + // request named none, so the source's carries over. + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl request headers override the copied properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentType: "text/plain" + } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentType: "application/json" } + }); + + const download = await blobClient.download(0); + assert.equal(download.contentType, "application/json"); + // A property the request did not name still comes from the source. + assert.equal(download.cacheControl, "max-age=3600"); + }); + + it("putBlobFromUrl metadata replaces the source's @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + metadata: { keya: "vala" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + metadata: { keyc: "valc" } + }); + + const download = await blobClient.download(0); + assert.deepStrictEqual(download.metadata, { keyc: "valc" }); + }); + + it("putBlobFromUrl answers the source conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const upload = await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: '"0x0000000000000000"' } + }); + assert.fail("Did not throw an exception."); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 412); + assert.equal(e.code, "SourceConditionNotMet"); + } + + // The same condition naming the source's own ETag admits the copy. + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: upload.etag } + }); + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl answers the destination conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("existing", "existing".length); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + conditions: { ifNoneMatch: "*" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 409); + assert.equal(e.code, "BlobAlreadyExists"); + // The destination the condition protected is untouched. + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, 8), "existing"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects a request body @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // @azure/storage-blob sends no body for this operation, so issue the + // request directly. + const response = await axios.put(destinationUrl, "unexpected body", { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-blob-type": "BlockBlob" + }, + validateStatus: () => true + }); + assert.deepStrictEqual(response.status, 400); + assert.ok(response.data.includes("InvalidHeaderValue")); + }); + + it("putBlobFromUrl from a missing source returns 404 @loki @sql", async () => { + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 404); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl copies the stored bytes when the source declares Content-Encoding: gzip @loki @sql", async () => { + // A blob's Content-Encoding is stored metadata, not a description of how + // the body is framed on the wire, so the download echoes it back over the + // raw stored bytes. The copy must carry those bytes verbatim rather than + // decoding them (see issue #646 for the same hazard on copy). + const raw = zlib.gzipSync(Buffer.from("HelloWorldFromSourceBlob")); + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(raw, raw.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blockBlobClient.download(0); + const chunks: Buffer[] = []; + for await (const chunk of download.readableStreamBody!) { + chunks.push(Buffer.from(chunk)); + } + assert.deepStrictEqual( + Buffer.concat(chunks), + raw, + "The copy must be the source's stored bytes, not the decoded ones" + ); + }); + + it("putBlobFromUrl with matching sourceContentMD5 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const result = await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + assert.deepStrictEqual(Buffer.from(result.contentMD5!), md5); + }); + + it("putBlobFromUrl with wrong sourceContentMD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length sourceContentMD5 should be rejected @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-md5", value: Buffer.from("short").toString("base64") }] + ); + + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl sets the tags the request names @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const tags: Tags = { tag1: "val1", tag2: "val2" }; + await blockBlobClient.syncUploadFromURL(sourceUrl, { tags }); + + // Tags are not copied from the source unless asked for, so the request's + // stand alone. + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copies the source's tags when asked @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const tags: Tags = { sourcetag: "sourcevalue", other: "value" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { tags }); + const sourceUrl = await sourceClient.generateSasUrl({ + // Reading the source's tags is its own permission, as on the service. + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copying the tags of an untagged source @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, {}); + }); + + it("putBlobFromUrl cannot copy tags the source URL may not read @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + // Read permission alone does not extend to the source's tags. + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 403); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects tags alongside copySourceTags COPY @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY", + tags: { tag1: "val1" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "BothUserTagsAndSourceTagsCopyPresentException"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl sets the access tier @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { tier: "Cool" }); + + const properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier, "Cool"); + }); + it("stageBlock with double commit block should work @loki @sql", async () => { const body = "HelloWorld";