diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index c6aa3fc623..ec18a650b4 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -82,3 +82,9 @@ body: label: 复现链接（可选） description: | 请提供能复现此问题的链接。 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此问题是由AI辅助您发现的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index d99968d907..0085d4de53 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -82,3 +82,9 @@ body: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this issue was identified with the assistance of AI, please provide the complete chat log, including information about the model used. diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 8339d947f9..9072402611 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -48,3 +48,9 @@ body: label: 附加信息 description: | 相关的任何其他上下文或截图，或者你觉得有帮助的信息 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此请求是由AI辅助您提交的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 41c9990cfe..393118592f 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -48,3 +48,9 @@ body: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this request was submitted with the assistance of an AI, please provide the complete chat log, including information about the model used. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f1687eabfe..58e122837b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,58 +1,114 @@ + +## Summary / 摘要 + -## Description / 描述 + - +- 列出用户可感知的行为变化。 +- 列出重要实现变化。 +- 如涉及配置、存储、API 或兼容性变化，请明确说明。 +--> -## Motivation and Context / 背景 +- [ ] This PR has breaking changes. + / 此 PR 包含破坏性变更。 +- [ ] This PR changes public API, config, storage format, or migration behavior. + / 此 PR 修改了公开 API、配置、存储格式或迁移行为。 +- [ ] This PR requires corresponding changes in related repositories. + / 此 PR 需要关联仓库同步修改。 - - +Related repository PRs / 关联仓库 PR: - - +- OpenList-Frontend: +- OpenList-Docs: -Closes #XXXX +## Related Issues / 关联 Issue - - + -Relates to #XXXX +## Testing / 测试 -## How Has This Been Tested? / 测试 + - - +- [ ] `go test ./...` +- [ ] Manual test / 手动测试: ## Checklist / 检查清单 - - - - - - -- [ ] I have read the [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) document. - 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) 文档。 -- [ ] I have formatted my code with `go fmt` or [prettier](https://prettier.io/). - 我已使用 `go fmt` 或 [prettier](https://prettier.io/) 格式化提交的代码。 -- [ ] I have added appropriate labels to this PR (or mentioned needed labels in the description if lacking permissions). - 我已为此 PR 添加了适当的标签（如无权限或需要的标签不存在，请在描述中说明，管理员将后续处理）。 -- [ ] I have requested review from relevant code authors using the "Request review" feature when applicable. - 我已在适当情况下使用"Request review"功能请求相关代码作者进行审查。 -- [ ] I have updated the repository accordingly (If it’s needed). - 我已相应更新了相关仓库（若适用）。 - - [ ] [OpenList-Frontend](https://github.com/OpenListTeam/OpenList-Frontend) #XXXX - - [ ] [OpenList-Docs](https://github.com/OpenListTeam/OpenList-Docs) #XXXX +- [ ] I have read [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md). + / 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md)。 +- [ ] I confirm this contribution follows the repository license, contribution policy, and code of conduct. + / 我确认此贡献符合仓库许可证、贡献规范和行为准则。 +- [ ] I have formatted the changed code with `gofmt`, `go fmt`, or `prettier` where applicable. + / 我已按适用情况使用 `gofmt`、`go fmt` 或 `prettier` 格式化变更代码。 +- [ ] I have requested review from relevant maintainers or code owners where applicable. + / 我已在适用情况下请求相关维护者或代码所有者审查。 + +## AI Disclosure / AI 使用声明 + + + +- [ ] This PR includes AI-assisted content. + / 此 PR 包含 AI 辅助内容。 + +Tools used / 使用工具: + +- [ ] ChatGPT +- [ ] Codex +- [ ] GitHub Copilot +- [ ] Claude +- [ ] Gemini +- [ ] Other (please specify) / 其他（请注明）: + +Usage scope / 使用范围: + +- [ ] Code generation / 代码生成 +- [ ] Refactoring / 重构 +- [ ] Documentation / 文档 +- [ ] Tests / 测试 +- [ ] Translation / 翻译 +- [ ] Review assistance / 审查辅助 + +- [ ] I have reviewed and validated all AI-assisted content included in this PR. + / 我已审核并验证此 PR 中的所有 AI 辅助内容。 +- [ ] I have ensured that all AI-assisted commits include `Co-Authored-By` attribution. + / 我已确保所有 AI 辅助提交都包含 `Co-Authored-By` 归属信息。 +- [ ] I can reproduce all AI-assisted content included in this PR without any AI tools. + / 我可以在没有任何 AI 工具的情况下重现此 PR 中包含的所有 AI 辅助内容。 diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index d5817e6b4d..ba64b12f42 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -40,15 +40,18 @@ jobs: npx changelogithub --output CHANGELOG.md - name: Upload assets to beta release - uses: softprops/action-gh-release@v2 - with: - body_path: CHANGELOG.md - files: CHANGELOG.md - prerelease: true - tag_name: beta + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if ! gh release view beta; then + gh release create beta --prerelease --notes-file CHANGELOG.md + else + gh release edit beta --prerelease --notes-file CHANGELOG.md + fi + gh release upload beta "CHANGELOG.md" --clobber - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -110,7 +113,7 @@ jobs: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -121,7 +124,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + uses: OpenListTeam/cgo-actions@v1.2.5 with: targets: ${{ matrix.target }} flags: ${{ matrix.flags || '-ldflags=' }} @@ -137,6 +140,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} env: GOFLAGS: ${{ matrix.goflags }} @@ -164,13 +168,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # See above - name: Upload assets to beta release - uses: softprops/action-gh-release@v2 - with: - files: build/compress/* - prerelease: true - tag_name: beta + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in build/compress/*; do + gh release upload beta "$file" --clobber + done - name: Clean illegal characters from matrix.target id: clean_target_name @@ -182,7 +186,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3a501ffa4..ea8fb5a443 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: strategy: @@ -31,7 +34,7 @@ jobs: id: short-sha - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -42,7 +45,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + uses: OpenListTeam/cgo-actions@v1.2.5 with: targets: ${{ matrix.target }} flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} @@ -55,7 +58,8 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} output: openlist$ext - name: Verify musl binary is static @@ -69,7 +73,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index 1b51e23b9e..bc29e6bf6b 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -16,7 +16,7 @@ jobs: if: github.event_name == 'issues' steps: - name: Check issue for unchecked tasks and reply - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | let comment = ""; @@ -81,7 +81,7 @@ jobs: if: github.event_name == 'pull_request' steps: - name: Check PR title for required prefix and comment - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const title = context.payload.pull_request.title || ""; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fa13af18c..3eed1faf6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '1.25.0' @@ -69,9 +69,9 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload assets - uses: softprops/action-gh-release@v2 - with: - files: build/compress/* - prerelease: false - tag_name: ${{ github.event.release.tag_name }} - + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in build/compress/*; do + gh release upload "${{ github.event.release.tag_name }}" "$file" --clobber + done diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 80bdf9e3c8..4292dacfd2 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -45,13 +45,13 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -85,13 +85,13 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true @@ -147,7 +147,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' @@ -231,7 +231,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 16c2994018..0dc51ef0f1 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,150 +1,195 @@ name: Beta Release (Docker) - on: workflow_dispatch: + inputs: + frontend_repo: + description: 'Frontend repo, e.g. Ironboxplus/OpenList-Frontend' + required: false + default: 'Ironboxplus/OpenList-Frontend' + type: string + frontend_channel: + description: 'Frontend release channel to build' + required: false + default: rolling + type: choice + options: + - rolling + - latest + - both push: branches: - - main - pull_request: - branches: - - main + - feat/dynamic-frontend + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} REGISTRY: ghcr.io - ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} - IMAGE_TAGS_BETA: | - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + ARTIFACT_NAME_PREFIX: 'binaries_docker_release' + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' + # 👇 使用默认的前端仓库 (Ironboxplus/OpenList-Frontend) + # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest + strategy: + matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["rolling"]') }} steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - name: Setup Go + uses: actions/setup-go@v6 with: go-version: '1.25.0' + cache: true + cache-dependency-path: go.sum + + - name: Get Frontend Cache Version + id: frontend-cache + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: ${{ matrix.frontend_channel }} + run: | + frontend_repo="${{ env.FRONTEND_REPO }}" + web_version="$WEB_VERSION" + github_auth_args=() + + if [ -n "$GH_TOKEN" ]; then + github_auth_args=(-H "Authorization: Bearer $GH_TOKEN") + fi + + if [ "$web_version" = "latest" ]; then + frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') + else + # For rolling/dev channels, resolve the actual commit to bust stale caches + release_json=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/tags/$web_version" 2>/dev/null || echo '{}') + asset_name=$(echo "$release_json" | jq -r '.assets[0].name // empty') + frontend_version="${web_version}-${asset_name:-unknown}" + fi + + echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" + echo "version=$frontend_version" >> "$GITHUB_OUTPUT" + echo "Frontend repo: $frontend_repo" + echo "Frontend cache version: $frontend_version" + + - name: Cache Frontend + id: cache-frontend + uses: actions/cache@v5 + with: + path: public/dist + key: frontend-${{ steps.frontend-cache.outputs.repo }}-${{ steps.frontend-cache.outputs.version }} + restore-keys: | + frontend-${{ steps.frontend-cache.outputs.repo }}- - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash build.sh prepare docker-multiplatform - - name: Build go binary (beta) + - name: Build go binary run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} + WEB_VERSION: ${{ matrix.frontend_channel }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} overwrite: true - path: | - build/ - !build/*.tgz - !build/musl-libs/** + path: build/linux/amd64/openlist release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["rolling"]') }} + # 四种变体，各自独立 image 名（推送到独立 GHCR repo） image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - tag_favor: "" + image_name: "openlist" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - tag_favor: "suffix=-ffmpeg,onlatest=true" + image_name: "openlist-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - tag_favor: "suffix=-aria2,onlatest=true" + image_name: "openlist-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - tag_favor: "suffix=-aio,onlatest=true" + image_name: "openlist-aio" steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} - path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} + path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | - ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} - tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }} + tags: | + type=raw,value=front-${{ matrix.frontend_channel }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }}:buildcache-front-${{ matrix.frontend_channel }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }}:buildcache-front-${{ matrix.frontend_channel }},mode=max diff --git a/.gitignore b/.gitignore index 1d71f0d608..d421551100 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .DS_Store output/ /dist/ - +.omx # Binaries for programs and plugins *.exe *.exe~ @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +nul diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..606cf1178d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# Agent Instructions + +## Issues + +Before creating an issue, review the available issue templates in the `.github` directory. + +When drafting the issue: + +- Use the most appropriate template. +- Follow the template structure. +- Fill in all required sections. +- Remove sections that the template explicitly marks as optional or not applicable. +- Do not invent reproduction steps, logs, screenshots, or expected behavior. + +## Pull Requests + +Before creating a pull request, read `.github/PULL_REQUEST_TEMPLATE.md`. + +When drafting the pull request: + +- Follow the template structure. +- Use the title format required by the template. +- Fill in or remove each section according to the template guidance. +- Include testing details, or explain why testing was not run. +- Do not invent testing results. +- Do not claim validation, verification, or review steps that were not actually performed. + +## Git Commits + +When creating commits, follow the repository `git-commit` skill rules: + +- Use Conventional Commits title format: `type(scope): subject`. +- Allowed types: `feat`, `fix`, `refactor`, `perf`, `docs`, `style`, `test`, `build`, `ci`, `chore`, `revert`. +- Use a meaningful scope based on the main module, package, or feature. +- Write the subject in imperative mood and describe the actual change. +- Use a concise Markdown list in the commit body, with each item describing one key change. +- Do not invent changes that are not present in the diff. +- Do not describe behavior, refactors, fixes, or tests that are not reflected in the commit. + +Include at most one `Co-authored-by` trailer that matches the AI assistant actually used to produce the change. + +Examples: + +- `Co-authored-by: Codex <267193182+codex@users.noreply.github.com>` +- `Co-authored-by: GitHub Copilot ` +- `Co-authored-by: Claude <81847+claude@users.noreply.github.com>` + +If you are not one of the listed assistants, do not add a `Co-authored-by` trailer. + +Instead, ask the human collaborator to provide the exact `Co-authored-by` trailer to use. Do not invent, infer, or generate one yourself. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..3a9fad67ed --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,85 @@ +# OpenList Backend (OP) — CLAUDE.md + +Primary development guide for the Go backend. See [../INDEX.MD](../INDEX.MD) for the full subsystem→code map. + +--- + +## Environment Configuration + +| Item | Value | +|------|-------| +| OS | Windows 11 Pro for Workstations (10.0.26100) | +| Go | go1.25.4 windows/amd64 | +| Node.js | v22.19.0 | +| Project Root | `E:\Go\Openlist\` | + +### Repository Map + +| Directory | Repo | Branch | Remote | +|-----------|------|--------|--------| +| `OP/` | [Ironboxplus/OpenList](https://github.com/Ironboxplus/OpenList) | `feat/dynamic-frontend` | origin (fork), op (upstream: OpenListTeam/OpenList) | +| `OpenList-Frontend/` | [Ironboxplus/OpenList-Frontend](https://github.com/Ironboxplus/OpenList-Frontend) | `main` | origin (upstream), ironbox (fork) | +| `movi-player/` | Local fork of [MrUjjwalG/movi-player](https://github.com/MrUjjwalG/movi-player) | `main` | — | +| `115-sdk-go/` | [Ironboxplus/115-sdk-go](https://github.com/Ironboxplus/115-sdk-go) | — | — | + +### Module Replacements (`go.mod`) + +| Module | Replace Target | Notes | +|--------|---------------|-------| +| `github.com/OpenListTeam/115-sdk-go` | `github.com/Ironboxplus/115-sdk-go v0.2.9` | Refresh context isolation, token error code filters, ErrDataEmpty, FlexString CID | +| `github.com/ProtonMail/go-proton-api` | `github.com/henrybear327/go-proton-api v1.0.0` | Community fork | +| `github.com/cronokirby/saferith` | `github.com/Da3zKi7/saferith v0.33.0-fixed` | Bug fix fork | + +--- + +## Core Development Principles + +1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal +2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file +3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload + +## Build and Development Commands + +```bash +# Development +go run main.go # Run backend server (default port 5244) +air # Hot reload during development +./build.sh dev # Build dev version with frontend +./build.sh release # Build release version + +# Testing +go test ./... # Run all tests +go test ./drivers/115_open/ -v # Run tests for a specific driver +go build ./drivers/115_open/... # Quick compile check for a package + +# Docker +docker-compose up && docker build -f Dockerfile . +``` + +- Requires Go 1.24+ (CI uses 1.25.0) +- `build.sh` fetches frontend from `$FRONTEND_REPO` (default: `Ironboxplus/OpenList-Frontend`) and embeds into `public/dist/` + +## Key Naming Conventions + +- Drivers: lowercase with underscores (`baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (`internal/op`) +- Interfaces: PascalCase (`Reader`, `Writer`) +- Logging: `log.Infof("[driver_name] message")` via `logrus` +- Retries: `github.com/avast/retry-go`; HTTP default timeout 30s (`base.RestyClient`) + +--- + +## Documentation Map + +| Document | Content | +|----------|---------| +| [../INDEX.MD](../INDEX.MD) | Subsystem → deep-dive doc → code path index | +| [../OVERVIEW.md](../OVERVIEW.md) | High-level system map | +| [../JOURNAL.md](../JOURNAL.md) | Chronological change log (source of truth) | +| [../PLAN.md](../PLAN.md) | Active development roadmap | +| [docs/architecture.md](docs/architecture.md) | Driver system, request flow, internal packages, startup, common patterns | +| [docs/streaming-and-caching.md](docs/streaming-and-caching.md) | Link types, cache system, RangeReader, SeekableStream pitfalls, HybridCache | +| [docs/uploads.md](docs/uploads.md) | OSS callback validation, Put vs PutResult, merge-task ObjectNotFound tolerance | +| [docs/plugins.md](docs/plugins.md) | Frontend plugin registry/slots + backend yaegi (Go-source) runtime | +| [docs/frontend.md](docs/frontend.md) | Frontend dist serving, movi-player, subtitle handling | +| [docs/new-apis-2026-06-21.md](docs/new-apis-2026-06-21.md) | Storage loading progress, permission bit 16, video_play, favorites, driver_name | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c8344eb6c7..6d4a6ef273 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -106,7 +106,7 @@ Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an +standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within @@ -116,7 +116,7 @@ the community. This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). @@ -124,5 +124,5 @@ enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6064b03cb5..57317c6290 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Prerequisites: - [git](https://git-scm.com) -- [Go 1.24+](https://golang.org/doc/install) +- [Go](https://golang.org/doc/install) version declared in [`go.mod`](./go.mod) - [gcc](https://gcc.gnu.org/) - [nodejs](https://nodejs.org/) @@ -16,8 +16,8 @@ Prerequisites: Fork and clone `OpenList` and `OpenList-Frontend` anywhere: ```shell -$ git clone https://github.com//OpenList.git -$ git clone --recurse-submodules https://github.com//OpenList-Frontend.git +git clone https://github.com//OpenList.git +git clone --recurse-submodules https://github.com//OpenList-Frontend.git ``` ## Creating a branch @@ -25,7 +25,7 @@ $ git clone --recurse-submodules https://github.com//OpenList-Fro Create a new branch from the `main` branch, with an appropriate name. ```shell -$ git checkout -b +git checkout -b ``` ## Preview your change @@ -33,26 +33,36 @@ $ git checkout -b ### backend ```shell -$ go run main.go +go run main.go ``` ### frontend ```shell -$ pnpm dev +pnpm dev ``` ## Add a new driver Copy `drivers/template` folder and rename it, and follow the comments in it. +## Community and policies + +By contributing, you agree to follow the repository's code of conduct and license terms. + +- Code of conduct: [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) +- License: [LICENSE](./LICENSE) +- Security issues: please report privately according to [SECURITY.md](./SECURITY.md) + +If your contribution includes substantial AI-assisted content, disclose the tools used and the scope of assistance in the pull request. + ## Create a commit Commit messages should be well formatted, and to make that "standardized". Submit your pull request. For PR titles, follow [Conventional Commits](https://www.conventionalcommits.org). -https://github.com/OpenListTeam/OpenList/issues/376 + It's suggested to sign your commits. See: [How to sign commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) @@ -72,6 +82,19 @@ At least 1 approving review is required by reviewers with write access. You can (Optional) After your pull request is merged, you can delete your branch. +## AI Disclosure + +If your pull request includes substantial AI-assisted content, disclose it in the PR description. + +Please include: + +- Tools used, such as ChatGPT, GitHub Copilot, Claude, Cursor, or other AI tools. +- Usage scope, such as code generation, refactoring, documentation, tests, translation, or review assistance. +- Confirmation that you have reviewed and validated all AI-assisted content before submission. +- Confirmation that the submitted content complies with this repository's license and contribution policies. + +Minor AI assistance, such as typo fixes, autocomplete, formatting suggestions, or wording polish, does not need to be disclosed. + --- Thank you for your contribution! Let's make OpenList better together! diff --git a/README.md b/README.md index 1ec96df4d9..bcecd83578 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,10 @@ Thank you for your support and understanding of the OpenList project. ## Document -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Docs](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Terms of Use](https://doc.oplist.org/terms) +- 🔒 [Privacy Policy](https://doc.oplist.org/privacy) ## Demo @@ -133,6 +134,14 @@ Please refer to [*Discussions*](https://github.com/OpenListTeam/OpenList/discuss [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## Donors + +Thanks to the following donors for their generous support: + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## License The `OpenList` is open-source software licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) license. @@ -162,3 +171,4 @@ We sincerely thank the author [Xhofe](https://github.com/Xhofe) of the original Thanks goes to these wonderful people: [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) + diff --git a/README_cn.md b/README_cn.md index 1bcfdbed9d..47a8482bc3 100644 --- a/README_cn.md +++ b/README_cn.md @@ -116,14 +116,15 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 ## 文档 -- 🌏 [国内站点](https://doc.oplist.org.cn) -- 📘 [海外站点](https://doc.oplist.org) -- 📚 [备用站点](https://doc.openlist.team) +- 📘 [文档](https://doc.oplist.org) +- 🌏 [中国镜像](https://doc.oplist.org.cn) +- ⚖️ [使用条款](https://doc.oplist.org/terms) +- 🔒 [隐私政策](https://doc.oplist.org/privacy) -## 演示 +## Demo -- 🇨🇳 [国内演示站](https://demo.oplist.org.cn) -- 🌎 [海外演示站](https://demo.oplist.org) +- 🌎 [全球 Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## 讨论 @@ -133,6 +134,14 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## 捐赠者 + +感谢以下捐赠者的慷慨支持： + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## 许可证 `OpenList` 是基于 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证的开源软件。 diff --git a/README_ja.md b/README_ja.md index 3a5d5d19f7..2e4dbfb5af 100644 --- a/README_ja.md +++ b/README_ja.md @@ -116,14 +116,15 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい ## ドキュメント -- 📘 [グローバルサイト](https://doc.oplist.org) -- 📚 [バックアップサイト](https://doc.openlist.team) -- 🌏 [CNサイト](https://doc.oplist.org.cn) +- 📘 [ドキュメント](https://doc.oplist.org) +- 🌏 [中国ミラー](https://doc.oplist.org.cn) +- ⚖️ [利用規約](https://doc.oplist.org/terms) +- 🔒 [プライバシーポリシー](https://doc.oplist.org/privacy) -## デモ +## Demo -- 🌎 [グローバルデモ](https://demo.oplist.org) -- 🇨🇳 [CNデモ](https://demo.oplist.org.cn) +- 🌎 [グローバル Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## ディスカッション @@ -133,6 +134,14 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## 寄付者 + +以下の方々の寛大なご支援に感謝いたします： + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## ライセンス 「OpenList」は [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスの下で公開されているオープンソースソフトウェアです。 diff --git a/README_nl.md b/README_nl.md index 86e90e740c..a7ae49e1c7 100644 --- a/README_nl.md +++ b/README_nl.md @@ -116,9 +116,10 @@ Dank u voor uw ondersteuning en begrip ## Documentatie -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Documentatie](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Gebruiksvoorwaarden](https://doc.oplist.org/terms) +- 🔒 [Privacybeleid](https://doc.oplist.org/privacy) ## Demo @@ -133,6 +134,14 @@ Stel algemene vragen in [*Discussions*](https://github.com/OpenListTeam/OpenList [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## Donatie Bijdragers + +Dank aan de volgende donateurs voor hun steun: + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## Licentie `OpenList` is open-source software onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. diff --git a/build.sh b/build.sh index b6baca1fe7..4e7b4f608d 100644 --- a/build.sh +++ b/build.sh @@ -5,7 +5,7 @@ gitAuthor="The OpenList Projects Contributors " gitCommit=$(git log --pretty=format:"%h" -1) # Set frontend repository, default to OpenListTeam/OpenList-Frontend -frontendRepo="${FRONTEND_REPO:-OpenListTeam/OpenList-Frontend}" +frontendRepo="${FRONTEND_REPO:-Ironboxplus/OpenList-Frontend}" githubAuthArgs="" if [ -n "$GITHUB_TOKEN" ]; then @@ -18,6 +18,8 @@ if [[ "$*" == *"lite"* ]]; then useLite=true fi +skipFrontendFetch="${SKIP_FRONTEND_FETCH:-false}" + if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" @@ -31,6 +33,10 @@ else webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi +if [ -n "$WEB_VERSION" ]; then + webVersion="$WEB_VERSION" +fi + echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then @@ -46,6 +52,7 @@ ldflags="\ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$version' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ +-X 'github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=$frontendRepo' \ " # Keep sqlite driver tag selection centralized to avoid target drift. @@ -97,6 +104,11 @@ AssertStaticBinary() { } FetchWebRolling() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -110,6 +122,11 @@ FetchWebRolling() { } FetchWebRelease() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') @@ -236,8 +253,8 @@ BuildDockerMultiplatform() { docker_lflags="$(GetMuslStaticLdflags)" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -257,15 +274,17 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" - done + # ARM docker variants stay disabled on this branch to keep the workflow x64-only. + # If they are re-enabled later, they should follow the same static-link pattern. + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" + # done } BuildRelease() { @@ -655,7 +674,11 @@ if [ "$buildType" = "dev" ]; then fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then - FetchWebRolling + if [ "$WEB_VERSION" = "latest" ]; then + FetchWebRelease + else + FetchWebRolling + fi else FetchWebRelease fi diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000000..f6377223c7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,171 @@ +# Architecture — Driver System, Request Flow, Internal Packages, Startup + +Deep-dive reference for the OpenList backend's core structure. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` — JSON field name + - `type:"select"` — Input type (select, string, text, bool, number) + - `required:"true"` — Required field + - `options:"a,b,c"` — Dropdown options + - `default:"value"` — Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { /* initialize client */ } +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { /* list files */ } +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { /* return download URL */ } +``` + +--- + +## Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +--- + +## Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +--- + +## Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` — Load config, environment variables +2. `Log()` — Initialize logging +3. `InitDB()` — Connect to database (`_busy_timeout=5000` for SQLite parallel safety) +4. `data.InitData()` — Initialize default data +5. `LoadStorages()` — Load and initialize all storage drivers (parallel by driver type) +6. `InitTaskManager()` — Start background tasks +7. `InitPlugins()` — Start wazero plugin manager +8. `Start()` — Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +--- + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` — Feature not implemented +- `errs.ObjectNotFound` — File/folder not found +- `errs.NotFolder` — Path is not a directory +- `errs.StorageNotInit` — Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { /* refresh link */ } +``` + +### Saving Driver State + +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +```go +type YourDriver struct { limiter *rate.Limiter } + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +### Naming Conventions + +- Drivers: lowercase with underscores (`baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (`internal/op`) +- Interfaces: PascalCase with suffix (`Reader`, `Writer`) +- Driver config: use `driver.RootPath` or `driver.RootID` for root folder; add `omitempty` to optional JSON fields +- Logging: `log.Infof("[driver_name] message")` via `logrus` +- Retries: `github.com/avast/retry-go`; HTTP client timeout 30s via `base.RestyClient` diff --git a/docs/cluster-sync.md b/docs/cluster-sync.md new file mode 100644 index 0000000000..5f7c60a5ea --- /dev/null +++ b/docs/cluster-sync.md @@ -0,0 +1,129 @@ +# Cluster credential sync + +A self-contained plugin (`internal/cluster`) that lets multiple OpenList nodes +share **storage credentials** (tokens / cookies / secrets) in real time, so when +one node refreshes a token the others adopt it automatically. Disabled by +default; configured per-node under **Manage → Plugins → Cluster credential +sharing**. + +The design satisfies three hard requirements from the product owner: + +1. **Sync ONLY the credential** — never the whole storage config. Mount path, + root folder, cache, order, proxy and every other field stay local to each + node. ("除了 key 其他的都不用同步") +2. **Manual, multipartite pairing** — the admin explicitly groups the storages + (across any number of nodes) that represent the same account. A group is a + connected component of a k-partite graph. ("手动选择 storage … 多分图") +3. **Auto-discovery** — no hand-maintained peer list. Any node that authenticates + with the shared key joins; reachable nodes advertise an address and the rest + of the mesh is learned automatically. ("远程连过来，认证过了就加入 peer") + +## Layers + +| File | Responsibility | +|------|----------------| +| `crypto.go` | PSK → HKDF-SHA256 → AES-256-GCM; ed25519 identity (`node_id = base32(sha256(pubkey)[:16])`); sign/verify | +| `identity.go` | Persist a 32-byte seed at `/cluster/identity.key` (0600) | +| `creds.go` | Extract / overlay / hash the **credential subset** of a driver's Addition | +| `state.go` | CRDTs: sync-group document, per-group credential records, node inventory | +| `transport.go` | `syncMessage` wire form + AEAD `envelope` + replay cache | +| `conn.go` | Persistent WebSocket peer connections, registry, dial supervisor, PEX dialing | +| `engine.go` | `Manager`: lifecycle, storage hook, push/pull/apply, relay, anti-entropy, events | +| `status.go` | Redaction-safe overview (nodes/groups/connections/events/stats) for the UI | +| `config.go` | Per-node `Config` (`enabled/key/label/addr/seeds/apply_remote`) + atomic store | + +## What a "credential" is + +OpenList drivers do not tag credential fields, so `creds.go` identifies them by +name over the Addition JSON keys: any key whose normalized name contains +`token, cookie, password, passwd, secret, auth, session, refresh, access, +credential, apikey, appkey, privatekey, signkey, ticket, passport`. Structural +fields (`root_folder_id`, `order_by`, …) never match, so they are never shared. +`applyCreds` overlays only those keys onto a peer's Addition and reports whether +anything actually changed — identical credentials cause no write, no re-init, no +churn. `credHash` (sha256 of the canonicalized credential map) is the +idempotency key. + +## Sync groups + +`group{ id, name, members[]{ node_id, mount_path } }`. The whole set is a +**single LWW document** (`groupDoc`) — signed by its author, versioned by a +Lamport clock, tie-broken by origin id. Admin edits are rare, so whole-doc LWW is +simple and convergent. The document is gossiped to every node; `pruneCreds` +drops credential records for groups that no longer exist. + +## Credential records + +Per group, `credRecord{ group_id, origin_driver, fields[], cred_hash, payload, +version, origin, origin_pub, sig }`. The secret `payload` only ever travels +inside the AEAD-sealed envelope. Records are signed by their origin so they stay +authentic across relays. Merge is LWW by `(version, origin, cred_hash)`, and +idempotent: an identical `cred_hash` is ignored regardless of version. + +### Flow + +1. A driver refreshes its token → `op.MustSaveDriverStorage` → `callStorageHooks("update")`. +2. `onStorageHook` finds the groups containing `(this_node, mount)`. If the + storage is healthy (`status == work`) it extracts the credential, and for each + group `localCredChange` authors a new record (skipped if the hash is + unchanged) and broadcasts a `push`. If the storage is **broken** + (`status != work`) or a driver fired `token-invalid`, it does **not** push — + it `pullGroup`s to recover a good credential from peers instead. ("坏 token 不推") +3. A peer receives the record, `verify`s the signature, `mergeCred`s under LWW, + then `applyCredRecord` overlays the payload onto its own member mounts + (`op.UpdateStorage`, which re-inits the driver with the new token). The + resulting `update` hook is a no-op because the credential now matches the + record's hash — no echo, no loop. +4. The record is `relay`ed to other connections, so two NAT'd nodes converge + through a common reachable peer. + +Health gating ensures a broken token is never propagated; idempotency ensures a +node that already holds the same token does nothing. ("其他节点看到一样的 token 就不用变") + +## Discovery & transport + +The link is a persistent WebSocket at `/api/cluster/ws` (NAT-friendly: a node +behind NAT dials out and keeps the link open; a reachable node accepts inbound +and relays). Authentication is the PSK: any frame that opens the AEAD envelope +admits the sender as a peer. + +- Each node advertises a `nodeInfo` inventory (its `label`, `addr`, and the + `mount_path`/`driver`/`status` of every storage). Inventory is gossiped and + powers the UI graph + group editor. +- **PEX**: nodes exchange the dialable `addr`s they know; the dial supervisor + opens a connection to any advertised peer it is not already connected to. A + `seed` URL is only needed to bootstrap the very first join. +- Per-frame freshness: timestamp window + nonce replay cache. + +## API + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/api/cluster/ws` | Peer WebSocket endpoint (PSK-authenticated) | +| GET | `/api/admin/cluster/config` | Config (key redacted) + live status | +| POST | `/api/admin/cluster/config` | Save config (blank/`********` key keeps current) | +| GET | `/api/admin/cluster/status` | Live status only | +| POST | `/api/admin/cluster/groups` | Replace the sync-group document | + +## Config + +`enabled`, `key` (shared secret), `label` (this node's display name), `addr` +(this node's public URL — advertise to be auto-discovered; leave empty behind +NAT), `seeds` (optional bootstrap URLs), `apply_remote` (adopt peer creds; +default on), `announce_interval_sec`. `active() == enabled && key != ""` — a +reachable node can run accept-only with no seed. + +## State on disk + +`/cluster/identity.key` (seed), `config.json`, `state.json` +(groups + credential records + inventory + Lamport clock). State is written +atomically (temp + rename). + +## Tests + +`*_test.go` cover credential extraction / overlay (node-local fields preserved) / +idempotency, group-doc and credential-record LWW + signature forgery rejection, +inventory merge + dialable-address selection, envelope round-trip / replay / +stale / wrong-key, and `wsURL` / connection registry. A two-node integration run +(discovery → inventory → group propagation → credential overlay on token change → +events) is validated end to end. diff --git a/docs/frontend.md b/docs/frontend.md new file mode 100644 index 0000000000..5ea8d7837e --- /dev/null +++ b/docs/frontend.md @@ -0,0 +1,185 @@ +# Frontend — Dist Serving, Movi-Player Integration, Subtitle Handling + +Deep-dive reference for the OpenList frontend distribution pipeline and the movi-player video subsystem. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Frontend Dist Serving + +**Backend location**: `server/static/static.go`, `internal/frontend/` + +**Stack**: SolidJS + Vite + Hope UI. Builds to `dist/` which gets embedded into the backend binary via `go:embed`. + +The frontend dist has two sources, with a strict priority: + +1. **Embedded dist** (`public/dist/` via `go:embed`): Baked into the binary at build time. Always used on startup. +2. **Dynamic dist** (fetched by watcher): The `frontend.Watcher` checks GitHub every 30 minutes for a newer rolling release. If found, it downloads to `data/frontend_dist/` and hot-swaps the serving FS via `ReloadStatic()`. + +**Key design rule**: `initStatic()` always starts with the embedded dist (or `dist_dir` if configured). The cached dynamic dist in the data volume is **never** read on startup — only the watcher can activate it after verifying a newer version exists on GitHub. This prevents stale cache from overriding a newer Docker image. + +**Configuration** (`config.json`): +- `dist_dir`: Override with a custom local directory (highest priority, skips embedded) +- `frontend_repo`: GitHub repo for the watcher to check (default: `Ironboxplus/OpenList-Frontend`) + +**Startup flow**: +``` +initStatic() → embedded dist (or dist_dir) + ↓ +StartWatcher(ReloadStatic) → background goroutine + ↓ (every 30min) +FetchFromRolling() → compare cache version vs GitHub rolling tag commit + ↓ (if newer) +downloadAndExtract() → atomic swap in data/frontend_dist/dist/ + ↓ +ReloadStatic() → swap staticFS to new dist, re-render index.html +``` + +--- + +## Serving Priority and the In-Memory index.html Gotcha + +There are three sources of truth for the frontend, in strict priority order: + +| Priority | Source | When active | +|-------------|---------------------------------------------------|--------------------------------------------------------------------| +| 1 (highest) | `dist_dir` (config.json) | Set explicitly; `initStatic()` calls `os.DirFS(conf.Conf.DistDir)` | +| 2 | Watcher-fetched dist (`data/frontend_dist/dist`) | `shouldAutoFetch()` true and a newer rolling release was found | +| 3 (lowest) | Embedded dist (`public/dist/` via `go:embed`) | Default fallback when neither of the above applies | + +**Critical implementation detail — index.html lives in memory, not on disk.** + +`initIndex()` reads `index.html` from `staticFS` (or the CDN) once and stores it in `conf.RawIndexHtml`. `UpdateIndex()` derives `conf.ManageHtml` (for `/@manage/*`) and `conf.IndexHtml` (all other routes) from that raw copy. The SPA catch-all handler writes the in-memory string directly to the response — it never opens the file on disk per-request. + +Consequence: **replacing files on disk does not affect what the server returns for `index.html`** until one of the following happens: + +- The process restarts (which calls `initStatic()` / `initIndex()` again), OR +- The watcher downloads a genuinely newer rolling release and calls `ReloadStatic()`. + +Meanwhile, static assets (`/assets/`, `/images/`, `/streamer/`, `/static/`) **do** serve live from `staticFS` via `http.FS`. So new asset chunks load immediately after you copy files to disk — but the old `index.html` still references the old chunk hashes, causing a mix of old UI and new files. + +**Symptom pattern**: new asset requests return 200; the UI loads stale chunks; the browser console shows hash-mismatch errors or the manage page looks like a previous version. + +`ReloadStatic()` (called by the watcher) fixes both: it calls `staticFS.swap(os.DirFS(distPath))` to update the asset FS, then re-runs `initIndex()` to reload index.html into memory. + +--- + +## Deterministic Deploy Procedure (dist_dir method) + +This is the recommended deployment path when you need a specific frontend build on disk — especially on hosts that cannot reach GitHub (e.g., hosts behind a firewall or with restricted egress). + +``` +# 1. Build locally +cd OpenList-Frontend/ +pnpm install && pnpm build +# Output: dist/ + +# 2. Copy to host, atomically swap +# NEVER delete frontend_dist/ itself — the watcher owns that directory. +# Back up the old dist, then rename the new one into place. +REMOTE_DATA=/root/docker/opdata # host-side path; maps to /opt/openlist/data inside container +CONTAINER_DATA=/opt/openlist/data + +scp -r dist/ user@cfscan:${REMOTE_DATA}/frontend_dist/dist.new +ssh user@cfscan " + cd ${REMOTE_DATA}/frontend_dist + mv dist dist.bak.\$(date +%s) # keep old as backup, never delete it + mv dist.new dist +" + +# 3. Set dist_dir in config.json (container path) +# In config.json on the host, set: +# "dist_dir": "/opt/openlist/data/frontend_dist/dist" +# (the container-internal path, not the host path) + +# 4. Restart ONLY the openlist service — never bare `compose up -d` +ssh user@cfscan "docker restart openlist" +# On restart: initStatic() reads from dist_dir, initIndex() loads the new index.html. +``` + +**Production path mapping** (reference): + +| Host | Host path | Container path | +|--------------------------------|------------------------|------------------------| +| cfscan (op.rnarket.com) | `/root/docker/opdata` | `/opt/openlist/data` | +| txcloud (trans.zoterosync.top) | `/root/docker/data` | `/opt/openlist/data` | + +So for both hosts: `dist_dir = /opt/openlist/data/frontend_dist/dist` + +After restart, `initStatic()` picks up `dist_dir` deterministically — no GitHub access required, no watcher race, no index.html staleness. + +**Why not delete `frontend_dist/` entirely?** The watcher goroutine tracks state inside that directory (`.frontend_version` file, the `dist/` subdirectory). Removing the whole `frontend_dist/` directory races with the watcher. Back up `dist/` as `dist.bak.` instead; that's atomic and the watcher ignores the `.bak` sibling. + +--- + +## Frontend Build Commands + +```bash +# From OpenList-Frontend/ +pnpm install # Install dependencies +pnpm dev # Dev server (port 5173) +pnpm build # Production build +pnpm test # Run vitest tests +``` + +--- + +## Preview System + +Previews registered in `src/pages/home/previews/index.ts`. Each preview declares `prior: true/false` for priority ordering. + +Current video priority: **movi-player** (default) > **Artplayer** (fallback). + +--- + +## Movi-Player Integration + +**npm package**: `movi-player ^0.3.2` +**Local reference clone**: `movi-player/` (clean upstream mirror, no local changes) + +movi-player (FFmpeg WASM + WebCodecs) is the default video player. It requires COOP/COEP headers for SharedArrayBuffer — set in backend `server/router.go`: +- `Cross-Origin-Opener-Policy: same-origin` +- `Cross-Origin-Embedder-Policy: credentialless` (chosen over `require-corp` to avoid breaking CDN resources) + +**Subtitle support**: +| Format | Handling | +|--------|---------| +| SRT/VTT (external) | movi-player native parsing | +| ASS (external) | JASSUB (libass WASM) overlay canvas rendering with font fallback | +| PGS/SUP (external) | libpgs overlay canvas rendering | +| All embedded formats | movi-player WASM demuxer | + +**Subtitle manager** (`subtitle-manager.ts`): orchestrates overlay canvases for ASS and PGS formats that movi-player cannot render natively. + +--- + +## Movi-Player npm Package Version + +- Current: **0.3.2** (referenced via `^0.3.2` in package.json) +- Key additions in 0.3.0+: TrueHD/MLP software audio decoding, HDR/DoVi hardware-decode retention, open-GOP HEVC fallback, multichannel passthrough + +**Known Limitations**: +| Limitation | Reason | +|-----------|--------| +| External ASS/PGS not rendered natively | Handled by JASSUB (ASS) and libpgs (PGS) overlay canvases | +| Dolby Vision: purple tint | WASM decoder lacks DV enhancement layer | +| Requires COOP/COEP headers | SharedArrayBuffer for WASM threads | + +--- + +## 115 Official Play Source (`115_video.tsx`) + +Frontend component that renders an Artplayer with quality selector for the `/api/fs/video_play` endpoint. Lists available resolutions sorted high→low. See [new-apis-2026-06-21.md](new-apis-2026-06-21.md) for the backend API details. + +--- + +## Other Frontend Subsystems (vs Upstream) + +| Feature | Key files | +|---------|-----------| +| VideoTreeList | `src/pages/home/previews/video/VideoTreeList.tsx` — tree-structured video browser replacing flat dropdown | +| Mobile toolbar | `src/utils/touch.ts` (triple-detection), `Icon.tsx` (icon+label on touch) | +| Header UserMenu | `user-menu.ts` pure-function entries; `UserMenu.tsx` avatar dropdown; Footer login removed | +| Motion presets | `src/utils/motion-presets.ts` — stagger/fade/scale entry animations | +| Persisted state | `src/utils/persisted.ts` — `createPersistedSignal`; storages driver filter persists across navigation | +| ECharts component | `src/components/EChart.tsx` — tree-shaken PieChart+BarChart, colorMode-aware | +| GridItem overflow | Respects `list_item_filename_overflow` (multi_line / scrollable / ellipsis), consistent with ListItem | diff --git a/docs/new-apis-2026-06-21.md b/docs/new-apis-2026-06-21.md new file mode 100644 index 0000000000..add97be04c --- /dev/null +++ b/docs/new-apis-2026-06-21.md @@ -0,0 +1,73 @@ +# New APIs and Features — 2026-06-21 Session + +Features added in the 2026-06-21 development session. See [INDEX.MD](../../INDEX.MD) for the full subsystem map and [JOURNAL.md](../../JOURNAL.md) for the chronological change log. + +--- + +## Storage Loading Progress — `GET /api/admin/storage/loading` + +Returns a live snapshot of storage initialization state. Requires admin auth. + +```go +// internal/op/loading.go +type LoadingProgress struct { /* thread-safe, states: pending/loading/loaded/failed */ } +func (lp *LoadingProgress) Snapshot() LoadingSnapshot +``` + +**Bootstrap behavior** (`internal/bootstrap/storage.go`): storages grouped by driver type — same driver type loads sequentially (shared rate-limiter / token-refresh safety), different driver types load in parallel. + +**SQLite safety**: `_busy_timeout=5000` set in `internal/bootstrap/db.go` to avoid `SQLITE_BUSY` under parallel writes. + +**Files**: `internal/op/loading.go`, `internal/bootstrap/storage.go`, `server/handles/storage.go` + +--- + +## Permission Bit 16 — `CanManageUserInfo` + +Defined in `internal/model/user.go`. Admin can grant a normal user the ability to edit other users' username/password. Privileged fields (role, permissions, base-path, etc.) are always forced back to original values by `applyUserUpdatePolicy` in `server/handles/user.go`. + +**Route split**: list/get/update under `/api/admin/user` guarded by `AuthUserInfoManage` middleware; create/delete/etc remain admin-only. Frontend hides privileged fields when the editor lacks full admin. + +**Files**: `internal/model/user.go`, `server/middlewares/auth.go`, `server/router.go`, `server/handles/user.go` + +--- + +## `POST /api/fs/video_play` — 115 Official Play Sources + +Optional `VideoPlayer` interface in `internal/driver/video.go`: + +```go +type VideoPlayer interface { + VideoPlay(ctx context.Context, file model.Obj) (*VideoPlayInfo, error) +} +``` + +`drivers/115_open/video.go` implements it via SDK `VideoPlay(pickCode)`. Returns available resolutions sorted high→low. Frontend `115_video.tsx` renders an Artplayer with a quality selector. + +**Files**: `internal/driver/video.go`, `drivers/115_open/video.go`, `server/handles/fsread.go` + +--- + +## Favorites API — `/api/me/favorites` + +User-scoped bookmark store. Model in `internal/model/favorite.go`, CRUD in `internal/db/favorite.go` (registered in `AutoMigrate`), handlers in `server/handles/favorite.go`. + +| Route | Purpose | +|-------|---------| +| `GET /api/me/favorites` | List current user's favorites | +| `POST /api/me/favorites/add` | Add path to favorites (idempotent) | +| `POST /api/me/favorites/delete` | Remove path from favorites | + +**Files**: `internal/model/favorite.go`, `internal/db/favorite.go`, `server/handles/favorite.go`, `server/router.go` + +--- + +## `StorageDetails.DriverName` + +`internal/model/storage.go` — `StorageDetails` now includes a `DriverName string` field populated by `GetStorageDetails` from `storage.Config().Name`. Emitted as `driver_name` in the JSON response. + +The `disk-usage` frontend plugin uses this to display the storage type label. + +**Note**: `/api/fs/list` returns `provider="unknown"` for mount details; `/api/fs/get` returns the real driver name. + +**Files**: `internal/model/storage.go`, `internal/op/` diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 0000000000..5abac61fdf --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,99 @@ +# Plugin System — Frontend Registry/Slots + Backend yaegi Runtime + +Two independent, hot-reloadable plugin planes. The **frontend** plane renders UI +via SolidJS slots; the **backend** plane runs admin-authored Go source in-process +via the [yaegi](https://github.com/traefik/yaegi) interpreter. See +[INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Frontend Plugin System (`OpenList-Frontend/src/plugins/`) + +| File | Role | +|------|------| +| `registry.ts` | Pure TS registry. `register(plugin)` / `unregister(id)` / `selectSlot(name)`. Enable/disable persisted in `localStorage`. Same `id` hot-replaces. | +| `PluginSlot.tsx` | Reactive slot renderer. `` renders all enabled plugins for that slot. | +| `loader.ts` | `loadExternalPlugins(manifest, importer?)` dynamically imports JS assets listed in the backend manifest. | +| `index.ts` | `installBuiltinPlugins()` + `installExternalPlugins()` (fetches `GET /api/plugin/manifest`). | + +**Built-in plugins**: `disk-usage` (header-right; reads `objStore.mountDetails`, updated on every navigation), `favorites` (header-right), `stats` (manage-dashboard ECharts), `media-preview` (behavior-only hover preview). Slot names in use: `header-right`, `manage-dashboard`. + +--- + +## Backend Plugin System (`OP/internal/plugin/`) + +Plugins are plain **`.go` source files** interpreted by yaegi — no compile step, +no WASM toolchain. (Previously this was a wazero/WASM runtime; swapped to yaegi +2026-06-21 so users can author plugins in plain Go.) + +| File | Role | +|------|------| +| `registry.go` | Hook event bus. `Subscribe(pluginID, hook, order, handler)` / `Fire(hook, payload)` / `UnsubscribeAll(pluginID)` / `HasHandlers(hook)`. One handler error never blocks others; `HookContext.Payload` is mutable across the chain. | +| `interp.go` | yaegi `Runtime`: `Load(name, source)` / `Unload` / `Loaded` / `Close`. One interpreter per plugin; same-name `Load` hot-reloads. Exposes `stdlib.Symbols` + a curated `hostSymbols` (this package's `API`, `Handler`, `HookContext`, and `Hook*` constants) — **nothing else of the host**, so plugins cannot import `internal/*`. Defines the `API` interface and per-plugin `pluginAPI`. | +| `hooks.go` | Well-known `Hook*` constants, the `FireHook`/`HasSubscribers` helpers, and injectable capability vars (`SettingGetter`/`SettingSetter`) — kept here so the plugin package never imports `internal/op` (which fires hooks → would be an import cycle). | +| `manager.go` | Watches `/plugins/go/*.go` (mtime+size change detection) and `/plugins/frontend/*.js`. `Start(interval)` polling goroutine; `Sync()` reconciles; tracks per-plugin load errors. `AssetPath` has traversal protection. | +| `manage.go` | User-plugin management: `ListGoPlugins` / `GetGoPluginSource` / `SaveGoPlugin` / `DeleteGoPlugin` / `SetGoPluginEnabled`. Enable/disable toggles the file between `.go` and `.go.disabled`. Name charset is restricted (traversal-proof). | +| `internal/bootstrap/plugin.go` | `InitPlugins()`: wires `SettingGetter/Setter` (via op), bridges `op.RegisterStorageHook` + `op.RegisterObjsUpdateHook` into the plugin registry, starts the watcher. | + +### Plugin contract + +A backend plugin is a single `.go` file: + +```go +package main + +import plugin "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +// Required. Runs once on (re)load. +func OnLoad(api plugin.API) { + api.Log("hello") + api.Subscribe(plugin.HookFsListAfter, 0, func(c *plugin.HookContext) error { + api.Logf("listed %v (%v items)", c.Payload["path"], c.Payload["count"]) + return nil + }) +} + +// Optional. Runs before the plugin is removed or hot-reloaded. +func OnUnload() {} +``` + +`plugin.API` surface: `Name()`, `Subscribe(hook, order, handler)`, `Log(...)`, +`Logf(fmt, ...)`, `GetSetting(key)`, `SetSetting(key, value)`. Plugins may also +import the Go standard library (yaegi blocks `unsafe`, `os/exec`, fork, `os.Exit` +by default). **Trust model**: plugins are admin-installed, trusted code — yaegi is +defense-in-depth, not a hostile-code sandbox. + +### Hook points (fired by the host) + +| Constant | Fired at | Payload | +|----------|----------|---------| +| `HookStorageCreated` / `Updated` / `Deleted` | storage lifecycle (bridged from `op` StorageHook) | `mount_path`, `driver` | +| `HookFsObjsUpdated` | dir contents change (bridged from `op` ObjsUpdateHook) | `path`, `count` | +| `HookFsListAfter` | after a directory is listed (`handles.FsList`) | `path`, `count` | +| `HookFsLinkAfter` | after a link is generated (`handles.Link`) | `path` | +| `HookUserLoginAfter` | after successful login (`handles.loginHash`) | `username` | + +`FireHook` is a no-op when no plugin subscribes, so hot paths gate payload +construction with `HasSubscribers(hook)`. + +### API endpoints + +| Route | Auth | Purpose | +|-------|------|---------| +| `GET /api/plugin/manifest` | Public | Frontend JS asset manifest | +| `GET /api/plugin/asset/:name` | Public | Serve a frontend JS asset (traversal-guarded) | +| `GET /api/admin/plugin/list` | Admin | Backend Go plugins (+ load status/errors) and frontend plugins | +| `GET /api/admin/plugin/get?name=` | Admin | A backend plugin's source | +| `POST /api/admin/plugin/save` | Admin | Create/overwrite a backend plugin (`{name, source}`), loads immediately | +| `POST /api/admin/plugin/delete` | Admin | Delete a backend plugin (`{name}`) | +| `POST /api/admin/plugin/enable` | Admin | Enable/disable (`{name, enabled}`) | + +The management UI is `OpenList-Frontend/src/pages/manage/plugins/Plugins.tsx` +(backend editor + frontend toggles), with the API client in `plugins/api.ts`. + +### Dependency + +`github.com/traefik/yaegi v0.16.1` — builds on the Go 1.25 toolchain; its +interpreter accepts up to **Go 1.22** language level, so plugin source must avoid +1.23+ features (iterators, generic type aliases). Replaces the former +`github.com/tetratelabs/wazero`. diff --git a/docs/streaming-and-caching.md b/docs/streaming-and-caching.md new file mode 100644 index 0000000000..3383929282 --- /dev/null +++ b/docs/streaming-and-caching.md @@ -0,0 +1,146 @@ +# Streaming and Caching — Link Types, Cache System, RangeReader, SeekableStream, HybridCache + +Deep-dive reference for OpenList's streaming pipeline and caching layer. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +--- + +## Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF** — Core interface for range-based reading: + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader** — Wraps RangeReader with automatic link refresh: + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel; assembles final stream + +--- + +## Stream Types and Reader Management + +> **CRITICAL**: `SeekableStream.Reader` must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** — use lazy initialization via `generateReader()` + +**Common Pitfall — Early Reader Creation**: +```go +// WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +--- + +## Hash Calculation for Uploads + +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + +--- + +## Link Refresh Pattern + +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +--- + +## Proxy Function (`server/common/proxy.go`) + +Handles multiple scenarios in priority order: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) — wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +--- + +## HybridCache — Replacing the Old MaxBufferLimit Truncation + +**Background**: Historically `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` capped buffering at `MaxBufferLimit` (~48MB), so non-seekable streams larger than that produced a truncated SHA1 (only the first 48MB hashed) and providers like 115 rejected the upload. The 115_open driver carried a `utils.CreateTempFile` workaround. + +**Current (2026-05-17)**: upstream PR #2460 unified caching via `internal/mem.HybridCache` — three tiers (heap memory → `LinearMemory` → temp file fallback) with 16MB blocks (`MaxBlockLimit`). `CacheFullAndWriter` now writes the *entire* stream regardless of size, so the truncation bug is gone for **all drivers**, not just 115_open. The 115_open workaround was removed; the standard `stream.CacheFullAndHash` path is back. + +**Unaffected paths**: +- `SeekableStream` (copy tasks): uses `RangeRead` directly, never goes through `cache()` +- Form uploads: `c.FormFile()` already stores to a temp file, so `GetFile()` returns non-nil and `CacheFullAndWriter` reads the entire file + +--- + +## Pass 2 Prefetch on `hybridSectionReader` + +For sequential multipart uploads (115_open, baidu_netdisk, aliyundrive_open, etc.), `hybridSectionReader.GetSectionReader` launches a background goroutine to pre-read the next chunk while the caller uploads the current one. The next call picks up the prefetched block; mismatched offsets fall back to synchronous read. Prefetch is clamped to remaining file size, drains on `DiscardSection`, and surfaces errors on the next `GetSectionReader`. Cleanup is registered via `file.Add` so the goroutine is drained before `HybridCache` is freed. + +Drivers that already use `errgroup.Lifecycle` (e.g. 123/upload.go) with `Before` (List/GetSectionReader) and `Do` (upload) get overlap from the lifecycle pattern itself; the section-reader prefetch helps drivers that loop sequentially without Before/Do split (e.g. 115_open, which requires `oss.Sequential()`). diff --git a/docs/uploads.md b/docs/uploads.md new file mode 100644 index 0000000000..2990eb2555 --- /dev/null +++ b/docs/uploads.md @@ -0,0 +1,56 @@ +# Uploads — OSS Callback Validation, Put vs PutResult, Merge-Task Tolerance + +Deep-dive reference for OpenList's upload pipeline. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## OSS Callback Validation + +Drivers that upload via Aliyun OSS (e.g., `115`, `115_open`) use a callback mechanism: after OSS stores the file, it POSTs to the storage provider's callback URL. The provider returns a JSON response indicating whether the file was registered. + +**Critical**: Always capture and validate the callback response: +```go +var bodyBytes []byte +_, err = bucket.CompleteMultipartUpload(imur, parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(callbackVar))), + oss.CallbackResult(&bodyBytes), // ← MUST capture this +) +// Check both OSS error AND callback response +if err != nil { return err } +// Parse bodyBytes to verify {"state": true} +``` + +Without `oss.CallbackResult`, the upload appears successful (OSS returns 200) but the file is never registered on the provider's side. The local cache shows the file temporarily, but it vanishes on refresh. + +--- + +## `Put` vs `PutResult` + +Drivers can implement either: +- `driver.Put` (returns `error`): When `Put` returns nil, `op.Put` creates a temporary object in the directory cache. +- `driver.PutResult` (returns `model.Obj, error`): When `PutResult` returns an actual object, that object is used in the cache instead. + +Use `PutResult` when the storage provider returns metadata about the uploaded file (ID, size, timestamps), so the cache entry is accurate without requiring a re-list. + +--- + +## Merge-Task ObjectNotFound Tolerance + +Merge-mode `FileTransferTask.RunWithNextTaskCallback` calls `op.List(dst)` to build the `existedObjs` skip set. A non-existent dst must be treated as "empty" rather than fatal (otherwise resuming an interrupted merge to a fresh dst fails immediately). The logic is extracted into `existingDstFilesFn` in `internal/fs/copy_move.go`: + +```go +dstObjs, err := listDst(ctx, dstPath) +if err != nil && !errors.Is(err, errs.ObjectNotFound) { + return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) +} +// non-existent dst → empty map, merge proceeds and creates dst on demand +``` + +A previous BFS-style "precreate one level of subdirectories" optimization was removed (2026-05-17, commit `6b3ce577`); directory creation now happens on demand via `op.Put`'s internal `MakeDir(parent)` and `op.MakeDir`'s recursive parent walk. The top-level dst `MakeDir` at `copy_move.go:198` is kept for early failure detection. 12 unit tests on `existingDstFilesFn` (including raw + wrapped `ObjectNotFound`) lock in the contract. + +--- + +## HybridCache and Prefetch + +See [streaming-and-caching.md](streaming-and-caching.md) for the full HybridCache description and `hybridSectionReader` prefetch details, which directly affect upload throughput for multipart drivers. diff --git a/drivers/115/driver.go b/drivers/115/driver.go index d4f5741d0a..7c7c581a8f 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -5,6 +5,7 @@ import ( "strings" "sync" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -68,6 +69,9 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") + if userAgent == "" { + userAgent = base.UserAgent + } downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 278ae0f7fa..69b75e3710 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,23 +2,29 @@ package _115_open import ( "context" + "encoding/json" "fmt" "net/http" + "net/url" stdpath "path" "slices" "strconv" "strings" + "sync/atomic" "time" sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -28,8 +34,20 @@ type Open115 struct { client *sdk.Client limiter *rate.Limiter parentPath string + // tokenInvalid is the edge-trigger latch for the cluster validity protocol: + // set once the refresh_token is found dead, cleared once a request succeeds + // again. It debounces the token-valid/invalid notifications down to real + // transitions instead of firing them on every request. + tokenInvalid atomic.Bool } +var ( + // 回收站列表存在短暂最终一致性延迟，永久删除 fallback 查找增加短重试。 + recycleBinLookupMaxAttempts = 4 + recycleBinLookupRetryDelay = 300 * time.Millisecond + new115SDKClient = sdk.New +) + func (d *Open115) Config() driver.Config { return config } @@ -39,23 +57,59 @@ func (d *Open115) GetAddition() driver.Additional { } func (d *Open115) Init(ctx context.Context) error { - d.client = sdk.New(sdk.WithRefreshToken(d.Addition.RefreshToken), + // This client generation can finish an old request after cluster recovery has + // replaced the storage. The SDK reports the exact access token used by each + // success/failure so callbacks can bind to the current generation. + observedAccessToken := d.Addition.AccessToken + d.client = new115SDKClient(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), sdk.WithOnRefreshToken(func(s1, s2 string) { d.Addition.AccessToken = s1 d.Addition.RefreshToken = s2 op.MustSaveDriverStorage(d) + }), + // Cluster token-validity protocol: a node only shares a token it has + // proven valid, and pulls a fresh one from a healthy peer when its own + // dies. Edge-triggered so the cluster only reacts to real transitions. + sdk.WithOnAccessTokenValid(func(accessToken string) { + st := d.GetStorage() + if st == nil || d.Addition.AccessToken != accessToken { + return + } + currentAddition := st.Addition + currentModified := st.Modified + // A successful authenticated request renews the cluster's in-memory + // proof only. This avoids a periodic refresh/push while preventing a + // working credential from ageing out of recovery advertisement. + op.NotifyStorageTokenHealthyWithSnapshot(d, currentAddition, currentModified) + // A refresh creates a new credential generation even when the mount + // never left WORK. Publish that generation after the retried request + // proves it, and also recover any pre-existing error status. + if accessToken != observedAccessToken || d.shouldNotifyTokenValid() { + op.NotifyStorageTokenValidWithSnapshot(d, currentAddition, currentModified) + } + }), + sdk.WithOnAccessTokenInvalid(func(accessToken string) { + st := d.GetStorage() + if st == nil || d.Addition.AccessToken != accessToken { + return + } + if d.tokenInvalid.CompareAndSwap(false, true) { + op.NotifyStorageTokenInvalidWithSnapshot(d, st.Addition, st.Modified) + } })) + applySDKProxyIfConfigured(d.client) if flags.Debug || flags.Dev { d.client.SetDebug(true) } + d.initLimiter() + if err := d.WaitLimit(ctx); err != nil { + return err + } _, err := d.client.UserInfo(ctx) if err != nil { return err } - if d.Addition.LimitRate > 0 { - d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) - } if d.PageSize <= 0 { d.PageSize = 200 } else if d.PageSize > 1150 { @@ -65,6 +119,9 @@ func (d *Open115) Init(ctx context.Context) error { // add parent path d.parentPath = "/" if d.GetRootId() != d.Config().DefaultRoot { + if err := d.WaitLimit(ctx); err != nil { + return err + } folderInfo, err := d.client.GetFolderInfo(ctx, d.GetRootId()) if err != nil { return err @@ -87,6 +144,32 @@ func (d *Open115) Init(ctx context.Context) error { return nil } +func (d *Open115) shouldNotifyTokenValid() bool { + if d.tokenInvalid.CompareAndSwap(true, false) { + return true + } + return d.GetStorage().Status != op.WORK +} + +func (d *Open115) initLimiter() { + if d.Addition.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) + return + } + d.limiter = nil +} + +func applySDKProxyIfConfigured(client *sdk.Client) { + if client == nil || conf.Conf == nil || strings.TrimSpace(conf.Conf.ProxyAddress) == "" { + return + } + proxyAddress := strings.TrimSpace(conf.Conf.ProxyAddress) + if _, err := url.Parse(proxyAddress); err != nil { + log.Warnf("[115] invalid proxy address ignored: %v", err) + return + } + client.SetProxy(proxyAddress) +} func (d *Open115) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) @@ -99,13 +182,20 @@ func (d *Open115) Drop(ctx context.Context) error { } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + start := time.Now() + log.Infof("[115] List request started for dir: %s (ID: %s)", dir.GetName(), dir.GetID()) + var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) + pageCount := 0 + for { if err := d.WaitLimit(ctx); err != nil { return nil, err } + + pageStart := time.Now() resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -115,7 +205,12 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) // Cur: 1, ShowDir: true, }) + pageDuration := time.Since(pageStart) + pageCount++ + log.Infof("[115] GetFiles page %d took: %v (offset=%d, limit=%d)", pageCount, pageDuration, offset, pageSize) + if err != nil { + log.Errorf("[115] GetFiles page %d failed after %v: %v", pageCount, pageDuration, err) return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { @@ -127,10 +222,17 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } offset += pageSize } + + totalDuration := time.Since(start) + log.Infof("[115] List request completed in %v (%d pages, %d files)", totalDuration, pageCount, len(res)) + return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + start := time.Now() + log.Infof("[115] Link request started for file: %s", file.GetName()) + if err := d.WaitLimit(ctx); err != nil { return nil, err } @@ -146,20 +248,39 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc + + apiStart := time.Now() + log.Infof("[115] Calling DownURL API...") resp, err := d.client.DownURL(ctx, pc, ua) + apiDuration := time.Since(apiStart) + log.Infof("[115] DownURL API took: %v", apiDuration) + if err != nil { + log.Errorf("[115] DownURL API failed after %v: %v", apiDuration, err) return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } - return &model.Link{ + + totalDuration := time.Since(start) + log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) + + link := &model.Link{ URL: u.URL.URL, Header: http.Header{ "User-Agent": []string{ua}, }, - }, nil + } + // Tie the cache TTL to the CDN's own `t=` expiry so OP never serves a + // URL that 115's CDN has already invalidated. Without this, OP would + // hand out a dead URL and 115 responds with 200 + Content-Length: 0, + // which downstream clients see as a corrupt/empty stream. + if ttl, ok := parseCDNExpiry(u.URL.URL); ok { + link.Expiration = &ttl + } + return link, nil } func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { @@ -169,15 +290,14 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { path = stdpath.Join(d.parentPath, path) resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { + if isObjectNotFound(err) { + return nil, errs.ObjectNotFound + } return nil, err } - return &Obj{ - Fid: resp.FileID, - Fn: resp.FileName, - Fc: resp.FileCategory, - Sha1: resp.Sha1, - Pc: resp.PickCode, - }, nil + log.Debugf("[115] GetFolderInfoByPath(%s) => Size=%q FileCategory=%q FileID=%s", + path, resp.Size, resp.FileCategory, resp.FileID) + return fromFolderInfo(resp) } func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -218,7 +338,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ - FileID: srcObj.GetID(), + FileID: srcObj.GetID(), FileName: newName, }) if err != nil { @@ -254,42 +374,265 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if !ok { return fmt.Errorf("can't convert obj") } - _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + resp, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } - return nil + if d.RemoveWay != "delete" { + return nil + } + return d.removePermanently(ctx, _obj, resp) } -func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - err := d.WaitLimit(ctx) +func (d *Open115) removePermanently(ctx context.Context, obj *Obj, deleteResp []string) error { + var directDeleteErr error + for _, tid := range deleteResp { + tid = strings.TrimSpace(tid) + if tid == "" { + continue + } + if err := d.deleteRecycleBinEntry(ctx, tid); err == nil { + return nil + } else if directDeleteErr == nil { + directDeleteErr = err + } + } + + recycleEntry, err := d.findRecycleBinEntryWithRetry(ctx, obj) if err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin candidate: %w; fallback lookup failed: %v", directDeleteErr, err) + } + return err + } + if err := d.deleteRecycleBinEntry(ctx, recycleEntry.ID); err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin entry %s after candidate delete error %v: %w", recycleEntry.ID, directDeleteErr, err) + } return err } + return nil +} + +func (d *Open115) deleteRecycleBinEntry(ctx context.Context, tid string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _, err := d.client.RbDelete(ctx, tid) + return err +} + +func (d *Open115) findRecycleBinEntry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 200 + } else if pageSize > 1150 { + pageSize = 1150 + } + + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.RbList(ctx, pageSize, offset) + if err != nil { + return nil, err + } + if entry := matchRecycleBinEntry(obj, resp.Files); entry != nil { + return entry, nil + } + + count, err := strconv.ParseInt(resp.Count, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse recycle bin count %q: %w", resp.Count, err) + } + offset += pageSize + if offset >= count || len(resp.Files) == 0 { + break + } + } + + return nil, fmt.Errorf("recycle bin entry not found for object id=%s name=%s parent=%s", obj.GetID(), obj.GetName(), obj.Pid) +} + +func isRecycleBinEntryNotFoundErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "recycle bin entry not found") +} + +func (d *Open115) findRecycleBinEntryWithRetry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + attempts := recycleBinLookupMaxAttempts + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for i := 0; i < attempts; i++ { + entry, err := d.findRecycleBinEntry(ctx, obj) + if err == nil { + return entry, nil + } + + lastErr = err + if !isRecycleBinEntryNotFoundErr(err) || i == attempts-1 { + break + } + + wait := recycleBinLookupRetryDelay * time.Duration(i+1) + if wait <= 0 { + continue + } + + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + return nil, lastErr +} + +func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *sdk.RbListResp_FileInfo { + if len(files) == 0 { + return nil + } + if entry, ok := files[obj.GetID()]; ok { + matched := entry + return &matched + } + + size := strconv.FormatInt(obj.GetSize(), 10) + for _, entry := range files { + if entry.ID == obj.GetID() { + matched := entry + return &matched + } + cid := string(entry.CID) + if obj.IsDir() { + if entry.FileName == obj.GetName() && cid == obj.Pid { + matched := entry + return &matched + } + continue + } + if obj.Sha1 != "" && entry.SHA1 != "" && strings.EqualFold(entry.SHA1, obj.Sha1) { + if entry.FileName == obj.GetName() || cid == obj.Pid { + matched := entry + return &matched + } + } + if entry.FileName == obj.GetName() && cid == obj.Pid && entry.FileSize == size { + matched := entry + return &matched + } + } + return nil +} + +func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + var err error sha1 := file.GetHash().GetHash(utils.SHA1) - if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + sha1128k := file.GetHash().GetHash(utils.SHA1_128K) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + if err := d.WaitLimit(ctx); err != nil { + return err + } + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) if err != nil { return err } + if resp.Status == 2 { + up(100) + return nil + } + // 秒传失败，继续后续流程 } - const PreHashSize int64 = 128 * utils.KB - hashSize := PreHashSize - if file.GetSize() < PreHashSize { - hashSize = file.GetSize() - } - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) - if err != nil { - return err + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(sha1) != utils.SHA1.Width { + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + if err != nil { + return err + } + } + // 计算 sha1_128k（如果没有预计算） + if len(sha1128k) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 如果有预计算的 hash，上面已经尝试过秒传了 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + // 秒传失败，需要缓存文件进行实际上传 + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } else { + // 没有预计算的 hash，缓存整个文件并计算 + if len(sha1) != utils.SHA1.Width { + _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + if err != nil { + return err + } + } else if file.GetFile() == nil { + // 有 SHA1 但没有缓存，需要缓存以支持后续 RangeRead + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + // 计算 sha1_128k + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } } - sha1128k, err := utils.HashReader(utils.SHA1, reader) - if err != nil { + + // 1. Init（SeekableStream 或已缓存的 FileStream） + if err := d.WaitLimit(ctx); err != nil { return err } - // 1. Init resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -315,14 +658,17 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } - reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) + signReader, err := file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } - signVal, err := utils.HashReader(utils.SHA1, reader) + signVal, err := utils.HashReader(utils.SHA1, signReader) if err != nil { return err } + if err := d.WaitLimit(ctx); err != nil { + return err + } resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -341,6 +687,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } // 3. get upload token + if err := d.WaitLimit(ctx); err != nil { + return err + } tokenResp, err := d.client.UploadGetToken(ctx) if err != nil { return err @@ -357,15 +706,48 @@ func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir mod return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } +func (d *Open115) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + var envelope sdk.Resp[[]sdk.AddOfflineTaskURIsResp] + response, err := d.client.AuthRequestRaw(ctx, sdk.ApiAddOffline, http.MethodPost, nil, sdk.ReqWithForm(sdk.Form{ + "urls": strings.Join(uris, "\n"), + "wp_path_id": dstDir.GetID(), + })) + if response != nil { + _ = json.Unmarshal(response.Bytes(), &envelope) + } + hashes := make([]string, 0, len(envelope.Data)) + for _, item := range envelope.Data { + if item.State && item.InfoHash != "" { + hashes = append(hashes, item.InfoHash) + } + } + rawResponse := "" + if response != nil { + rawResponse = response.String() + } + return hashes, envelope.Data, rawResponse, err +} + func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 获取第一页 resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } + // 如果有多页，获取所有页面的任务 + if resp.PageCount > 1 { + for page := 2; page <= resp.PageCount; page++ { + pageResp, err := d.client.OfflineTaskList(ctx, int64(page)) + if err != nil { + return nil, err + } + resp.Tasks = append(resp.Tasks, pageResp.Tasks...) + } + } return resp, nil } diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go new file mode 100644 index 0000000000..f3892ef5e9 --- /dev/null +++ b/drivers/115_open/driver_test.go @@ -0,0 +1,1138 @@ +package _115_open + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + driverpkg "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/glebarez/sqlite" + "golang.org/x/time/rate" + "gorm.io/gorm" +) + +type recordedRequest struct { + Path string + Form url.Values + Time time.Time +} + +type rewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = t.target.Scheme + cloned.URL.Host = t.target.Host + return t.base.RoundTrip(cloned) +} + +func TestOpen115RemoveTrashUsesDelFileOnly(t *testing.T) { + driver, requests := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, []string{"rb-123"}) + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete") + assertFormValue(t, requests()[0].Form, "file_ids", "file-1") + assertFormValue(t, requests()[0].Form, "parent_id", "dir-1") +} + +func TestOpen115RemoveDeleteUsesDelFileResponseIDWhenAvailable(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/del": + writeSDKSuccess(t, w, []string{"rb-123"}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del") + assertFormValue(t, requests()[1].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteFallsBackToRecycleBinLookup(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "file_size": "123", + "cid": "dir-1", + "sha1": "sha-demo", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing(t *testing.T) { + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(context.Background(), obj) + if err == nil { + t.Fatalf("expected Remove to fail when recycle-bin entry is missing") + } + if !strings.Contains(err.Error(), "recycle bin entry not found") { + t.Fatalf("expected recycle-bin lookup error, got: %v", err) + } +} + +func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { + info, ok := op.GetDriverInfoMap()["115 Open"] + if !ok { + t.Fatalf("115 Open driver info was not registered") + } + + for _, item := range info.Additional { + if item.Name != "remove_way" { + continue + } + if item.Type != "select" { + t.Fatalf("unexpected remove_way type: %q", item.Type) + } + if item.Options != "trash,delete" { + t.Fatalf("unexpected remove_way options: %q", item.Options) + } + if item.Default != "trash" { + t.Fatalf("unexpected remove_way default: %q", item.Default) + } + if !item.Required { + t.Fatalf("expected remove_way to be required") + } + return + } + + t.Fatalf("remove_way item not found in 115 Open driver info") +} + +func TestOpen115ShouldNotifyTokenValidForInvalidTransitionOrStaleStatus(t *testing.T) { + driver := &Open115{} + driver.Storage.Status = "old init error" + if !driver.shouldNotifyTokenValid() { + t.Fatalf("expected stale storage status to trigger token-valid notification") + } + if driver.tokenInvalid.Load() { + t.Fatalf("stale-status notification should not mark tokenInvalid") + } + + driver.Storage.Status = op.WORK + if driver.shouldNotifyTokenValid() { + t.Fatalf("expected healthy storage with no invalid transition to skip notification") + } + + driver.tokenInvalid.Store(true) + if !driver.shouldNotifyTokenValid() { + t.Fatalf("expected invalid-to-valid transition to trigger token-valid notification") + } + if driver.tokenInvalid.Load() { + t.Fatalf("expected tokenInvalid latch to be cleared") + } +} + +func TestOpen115SDKClientUsesConfiguredProxy(t *testing.T) { + oldConf := conf.Conf + t.Cleanup(func() { + conf.Conf = oldConf + }) + + var gotURL string + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotURL = r.URL.String() + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(proxy.Close) + + conf.Conf = &conf.Config{ProxyAddress: proxy.URL} + client := sdk.New() + applySDKProxyIfConfigured(client) + + resp, err := client.Request( + context.Background(), + "http://openlist-proxy-test.invalid/ping", + http.MethodGet, + ) + if err != nil { + t.Fatalf("expected request to go through configured proxy: %v", err) + } + if resp.StatusCode() != http.StatusOK { + t.Fatalf("unexpected status via proxy: %d", resp.StatusCode()) + } + if gotURL != "http://openlist-proxy-test.invalid/ping" { + t.Fatalf("proxy did not receive the absolute target URL, got %q", gotURL) + } +} + +func TestOpen115InitRateLimitsAuthAndRootInfo(t *testing.T) { + const limitRate = 10.0 + + var ( + mu sync.Mutex + requests []recordedRequest + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + Time: time.Now(), + }) + mu.Unlock() + + switch r.URL.Path { + case "/open/user/info": + writeSDKSuccess(t, w, map[string]any{}) + case "/open/folder/get_info": + writeSDKSuccess(t, w, map[string]any{ + "file_id": "root-1", + "file_name": "Media", + "paths": []map[string]string{ + {"file_id": "0", "file_name": ""}, + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + oldNewClient := new115SDKClient + t.Cleanup(func() { + new115SDKClient = oldNewClient + }) + new115SDKClient = func(opts ...sdk.Option) *sdk.Client { + client := sdk.New(opts...) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + return client + } + + driver := &Open115{Addition: Addition{ + RootID: driverpkg.RootID{RootFolderID: "root-1"}, + AccessToken: "test-access-token", + RefreshToken: "test-refresh-token", + LimitRate: limitRate, + }} + driver.Storage.Status = "old init error" + + if err := driver.Init(context.Background()); err != nil { + t.Fatalf("Init returned error: %v", err) + } + if driver.GetStorage().Status != op.WORK { + t.Fatalf("successful authenticated Init should restore storage status to work, got %q", driver.GetStorage().Status) + } + + mu.Lock() + reqs := append([]recordedRequest(nil), requests...) + mu.Unlock() + + assertRequestPaths(t, reqs, "/open/user/info", "/open/folder/get_info") + gap := reqs[1].Time.Sub(reqs[0].Time) + minGap := time.Duration(float64(time.Second) / limitRate * 0.7) + if gap < minGap { + t.Fatalf("Init SDK requests were too close (%v), expected at least %v; auth/root-info calls are not both rate-limited", gap, minGap) + } +} + +func TestOpen115RefreshRestoresErrorStateAndPublishesNewPair(t *testing.T) { + database, err := gorm.Open(sqlite.Open("file:open115-refresh-recovery?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("open test database: %v", err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + + var refreshCount int + var rejectFresh atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/refreshToken": + refreshCount++ + writeSDKResponse(t, w, map[string]any{ + "state": 1, + "code": 0, + "data": map[string]any{ + "access_token": "fresh-access", + "refresh_token": "fresh-refresh", + "expires_in": 7200, + }, + }) + case "/open/user/info": + if r.Header.Get("Authorization") != "Bearer fresh-access" { + writeSDKError(t, w, 40140125, "access_token invalid") + return + } + if rejectFresh.Load() { + writeSDKError(t, w, 40140120, "refresh token error") + return + } + writeSDKSuccess(t, w, map[string]any{}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + oldNewClient := new115SDKClient + t.Cleanup(func() { new115SDKClient = oldNewClient }) + new115SDKClient = func(opts ...sdk.Option) *sdk.Client { + client := sdk.New(opts...) + client.SetHttpClient(&http.Client{Transport: &rewriteTransport{target: target, base: http.DefaultTransport}}) + return client + } + + driver := &Open115{Addition: Addition{ + RootID: driverpkg.RootID{RootFolderID: "0"}, + AccessToken: "expired-access", + RefreshToken: "live-refresh", + }} + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/refresh-recovery", + Status: "code: 40140125, message: access_token invalid", + Addition: `{"access_token":"expired-access","refresh_token":"live-refresh"}`, + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + driver.Storage = storage + proof := make(chan op.StorageCredentialEvent, 1) + op.RegisterStorageCredentialHook(func(typ string, event op.StorageCredentialEvent) { + if typ != "token-valid" || event.Storage != driver { + return + } + select { + case proof <- event: + default: + } + }) + + if err := driver.Init(context.Background()); err != nil { + t.Fatalf("Init after access-token expiry failed: %v", err) + } + if refreshCount != 1 { + t.Fatalf("refresh count = %d, want exactly 1", refreshCount) + } + if driver.Addition.AccessToken != "fresh-access" || driver.Addition.RefreshToken != "fresh-refresh" { + t.Fatalf("refreshed pair was not installed: access=%q refresh=%q", driver.Addition.AccessToken, driver.Addition.RefreshToken) + } + if driver.Storage.Status != op.WORK { + t.Fatalf("refreshed mount status = %q, want WORK", driver.Storage.Status) + } + persisted, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if persisted.Status != op.WORK || !strings.Contains(persisted.Addition, "fresh-access") { + t.Fatalf("refreshed storage was not durably recovered: %#v", persisted) + } + select { + case event := <-proof: + if !strings.Contains(event.Addition, "fresh-access") || !strings.Contains(event.Addition, "fresh-refresh") { + t.Fatalf("published stale credential generation: %s", event.Addition) + } + case <-time.After(time.Second): + t.Fatal("refreshed and proven pair was not published") + } + + // The same long-lived driver may fail hours after refreshing. The invalid + // callback must bind to fresh-access, not the pre-refresh generation that + // configured the client. + rejectFresh.Store(true) + if _, err := driver.client.UserInfo(context.Background()); err == nil { + t.Fatal("expected the refreshed generation to become invalid") + } + if driver.Storage.Status == op.WORK { + t.Fatal("refreshed generation 401 did not invalidate the mounted storage") + } + persisted, err = db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById after invalidation failed: %v", err) + } + if persisted.Status == op.WORK { + t.Fatal("refreshed generation 401 was not persisted for peer recovery") + } +} + +func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + + var ( + mu sync.Mutex + requests []recordedRequest + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + Time: time.Now(), + }) + mu.Unlock() + responder(w, r) + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + client := sdk.New(sdk.WithAccessToken("test-token")) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + + return &Open115{ + Addition: Addition{ + RemoveWay: removeWay, + PageSize: 1, + }, + client: client, + }, func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + return append([]recordedRequest(nil), requests...) + } +} + +func writeSDKSuccess(t *testing.T, w http.ResponseWriter, data any) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": true, + "data": data, + }) +} + +func writeSDKError(t *testing.T, w http.ResponseWriter, code int64, message string) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": false, + "code": code, + "message": message, + }) +} + +func writeSDKResponse(t *testing.T, w http.ResponseWriter, payload map[string]any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(payload); err != nil { + t.Fatalf("Encode response failed: %v", err) + } +} + +func assertRequestPaths(t *testing.T, requests []recordedRequest, want ...string) { + t.Helper() + got := make([]string, 0, len(requests)) + for _, req := range requests { + got = append(got, req.Path) + } + if !slices.Equal(got, want) { + t.Fatalf("unexpected request paths: got %v want %v", got, want) + } +} + +func assertFormValue(t *testing.T, form url.Values, key, want string) { + t.Helper() + if got := form.Get(key); got != want { + t.Fatalf("unexpected form value for %s: got %q want %q", key, got, want) + } +} + +func cloneValues(src url.Values) url.Values { + dst := make(url.Values, len(src)) + for key, values := range src { + dst[key] = append([]string(nil), values...) + } + return dst +} + +// --- FlexString / numeric CID tests --- + +func TestFlexStringUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"string value", `{"cid":"dir-1"}`, "dir-1"}, + {"integer value", `{"cid":3383942108160578280}`, "3383942108160578280"}, + {"large integer", `{"cid":9999999999999999999}`, "9999999999999999999"}, + {"zero", `{"cid":0}`, "0"}, + {"negative", `{"cid":-123}`, "-123"}, + {"float", `{"cid":1.5}`, "1.5"}, + {"empty string", `{"cid":""}`, ""}, + {"null", `{"cid":null}`, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if got := string(v.CID); got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestFlexStringUnmarshalInvalid(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"boolean", `{"cid":true}`}, + {"array", `{"cid":[1]}`}, + {"object", `{"cid":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err == nil { + t.Fatalf("expected error for input %s", tt.input) + } + }) + } +} + +func TestRbListRespUnmarshalCIDAsString(t *testing.T) { + raw := `{"id":"rb-1","file_name":"demo.txt","cid":"dir-1","file_size":"123"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "dir-1" { + t.Fatalf("got CID %q, want %q", string(info.CID), "dir-1") + } +} + +func TestRbListRespUnmarshalCIDAsNumber(t *testing.T) { + raw := `{"id":"rb-1","file_name":"MyFolder","cid":3383942108160578280,"file_size":"0"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "3383942108160578280" { + t.Fatalf("got CID %q, want %q", string(info.CID), "3383942108160578280") + } +} + +// --- matchRecycleBinEntry with numeric CID --- + +func TestMatchRecycleBinEntryDirMatchWithNumericCID(t *testing.T) { + // obj represents a directory with Pid (parent ID) as a large number + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("3383942108160578280"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match for directory with numeric CID, got nil") + } + if result.ID != "rb-folder-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-folder-1") + } +} + +func TestMatchRecycleBinEntryDirNoMatchWhenCIDWrong(t *testing.T) { + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("9999999999"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result != nil { + t.Fatalf("expected no match, got %+v", result) + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchWithNumericCID(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+CID, got nil") + } + if result.ID != "rb-file-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-file-1") + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchByNameOnly(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "wrong-pid", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+name, got nil") + } +} + +func TestMatchRecycleBinEntryFileNameSizeCIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "doc.pdf", Fc: "1", FS: 500} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "doc.pdf", + CID: sdk.FlexString("3383942108160578280"), + FileSize: "500", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via name+size+CID, got nil") + } +} + +func TestMatchRecycleBinEntryDirectIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + files := map[string]sdk.RbListResp_FileInfo{ + "file-1": { + ID: "rb-123", + FileName: "demo.txt", + CID: sdk.FlexString("dir-1"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected direct ID match, got nil") + } + if result.ID != "rb-123" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-123") + } +} + +func TestMatchRecycleBinEntryEmptyFiles(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + result := matchRecycleBinEntry(obj, nil) + if result != nil { + t.Fatalf("expected nil for nil files, got %+v", result) + } + result = matchRecycleBinEntry(obj, map[string]sdk.RbListResp_FileInfo{}) + if result != nil { + t.Fatalf("expected nil for empty files, got %+v", result) + } +} + +// --- Full Remove flow with numeric CID in recycle bin --- + +func TestOpen115RemoveDeleteWithNumericCID(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"folder-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "folder-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-folder-1"}) + case "/open/rb/list": + // CID returned as number (the real bug scenario) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-folder-1": map[string]any{ + "id": "rb-folder-1", + "file_name": "MyFolder", + "cid": 3383942108160578280, + "file_size": "0", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-folder-1") +} + +func TestOpen115RemoveDeleteWithStringCIDStillWorks(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + // CID returned as string (normal case) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteRetriesRecycleBinLookupUntilVisible(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 3 + recycleBinLookupRetryDelay = time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + rbListCalls := 0 + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + rbListCalls++ + if rbListCalls < 3 { + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + return + } + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + if rbListCalls != 3 { + t.Fatalf("rbListCalls = %d, want 3", rbListCalls) + } + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/list", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[5].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteStopsRetryWhenContextCancelled(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 5 + recycleBinLookupRetryDelay = 50 * time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + writeSDKError(t, w, 404, "not found") + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(ctx, obj) + if err == nil { + t.Fatalf("expected Remove to fail due to context cancellation") + } + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("expected context deadline exceeded, got: %v", err) + } +} + +// --- Put rate-limiting tests --- + +// mockFileStreamer satisfies model.FileStreamer with pre-computed hashes for testing. +type mockFileStreamer struct { + name string + size int64 + hashInfo utils.HashInfo + data []byte +} + +func (m *mockFileStreamer) Read(p []byte) (int, error) { return 0, io.EOF } +func (m *mockFileStreamer) Close() error { return nil } +func (m *mockFileStreamer) Add(_ io.Closer) {} +func (m *mockFileStreamer) AddIfCloser(_ any) {} +func (m *mockFileStreamer) GetSize() int64 { return m.size } +func (m *mockFileStreamer) GetName() string { return m.name } +func (m *mockFileStreamer) ModTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) CreateTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) IsDir() bool { return false } +func (m *mockFileStreamer) GetHash() utils.HashInfo { return m.hashInfo } +func (m *mockFileStreamer) GetID() string { return "" } +func (m *mockFileStreamer) GetPath() string { return "" } +func (m *mockFileStreamer) GetMimetype() string { return "application/octet-stream" } +func (m *mockFileStreamer) NeedStore() bool { return false } +func (m *mockFileStreamer) IsForceStreamUpload() bool { return false } +func (m *mockFileStreamer) GetExist() model.Obj { return nil } +func (m *mockFileStreamer) SetExist(_ model.Obj) {} +func (m *mockFileStreamer) GetFile() model.File { return nil } +func (m *mockFileStreamer) RangeRead(_ http_range.Range) (io.Reader, error) { + return strings.NewReader(string(m.data)), nil +} +func (m *mockFileStreamer) CacheFullAndWriter(_ *model.UpdateProgress, _ io.Writer) (model.File, error) { + return nil, nil +} + +func newTestOpen115WithRateLimit(t *testing.T, limitRate float64, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + driver, requests := newTestOpen115(t, "trash", responder) + driver.limiter = rate.NewLimiter(rate.Limit(limitRate), 1) + return driver, requests +} + +func TestPutRateLimitsEverySDKCall(t *testing.T) { + // Rate limit: 10 req/s → each WaitLimit blocks ~100ms + const limitRate = 10.0 + + driver, requests := newTestOpen115WithRateLimit(t, limitRate, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/upload/init": + // First call: status=1 (not rapid), second call: status=2 (rapid success) + if r.FormValue("sign_key") != "" { + writeSDKSuccess(t, w, map[string]any{"status": 2}) + } else { + writeSDKSuccess(t, w, map[string]any{ + "status": 7, + "sign_key": "test-key", + "sign_check": "0-10", + }) + } + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + stream := &mockFileStreamer{ + name: "test.txt", + size: 100, + hashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + utils.SHA1_128K: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }), + data: make([]byte, 100), + } + dstDir := &model.Object{ID: "0", Name: "root", IsFolder: true} + up := func(float64) {} + + start := time.Now() + err := driver.Put(context.Background(), dstDir, stream, up) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + + reqs := requests() + // Expect: pre-hash UploadInit + main UploadInit + sign-check UploadInit = 3 calls + if len(reqs) < 3 { + t.Fatalf("expected at least 3 requests, got %d", len(reqs)) + } + assertRequestPaths(t, reqs, "/open/upload/init", "/open/upload/init", "/open/upload/init") + + // With 2 SDK calls each preceded by WaitLimit(10/s), minimum elapsed is ~100ms. + // Without WaitLimit, both calls fire instantly (<10ms). + minExpected := time.Duration(float64(time.Second) / limitRate * float64(len(reqs)-1)) + tolerance := minExpected * 7 / 10 // 70% to account for timing jitter + if elapsed < tolerance { + t.Fatalf("Put completed too fast (%v), expected at least %v — WaitLimit likely missing before some SDK calls", elapsed, tolerance) + } + + // Also verify individual request spacing + for i := 1; i < len(reqs); i++ { + gap := reqs[i].Time.Sub(reqs[i-1].Time) + gapMin := time.Duration(float64(time.Second) / limitRate * 0.7) + if gap < gapMin { + t.Fatalf("gap between request %d and %d is %v, expected at least %v — WaitLimit missing", i-1, i, gap, gapMin) + } + } +} + +func TestPutRateLimitsPreHashPath(t *testing.T) { + const limitRate = 10.0 + + driver, requests := newTestOpen115WithRateLimit(t, limitRate, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/upload/init": + // Rapid upload success on first try + writeSDKSuccess(t, w, map[string]any{"status": 2}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + stream := &mockFileStreamer{ + name: "test.txt", + size: 100, + hashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + utils.SHA1_128K: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }), + data: make([]byte, 100), + } + dstDir := &model.Object{ID: "0", Name: "root", IsFolder: true} + + err := driver.Put(context.Background(), dstDir, stream, func(float64) {}) + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + + reqs := requests() + if len(reqs) != 1 { + t.Fatalf("expected 1 request, got %d: %v", len(reqs), reqs) + } + assertRequestPaths(t, reqs, "/open/upload/init") +} + +func TestGetReturnsObjectNotFoundForEmptyData(t *testing.T) { + driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + // 115 returns empty array when path doesn't exist + writeSDKSuccess(t, w, []any{}) + }) + driver.parentPath = "" + + _, err := driver.Get(context.Background(), "/nonexistent/path") + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, errs.ObjectNotFound) { + t.Fatalf("expected ObjectNotFound, got: %v", err) + } +} + +func TestCheckUploadCallbackSuccess(t *testing.T) { + body := []byte(`{"state":true,"code":0,"message":"success","data":{"pick_code":"abc","file_name":"test.txt","file_size":100,"file_id":"123","sha1":"da39a3ee","cid":"456"}}`) + if err := checkUploadCallback(body); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } +} + +func TestCheckUploadCallbackStateFalse(t *testing.T) { + body := []byte(`{"state":false,"code":990009,"message":"upload failed"}`) + err := checkUploadCallback(body) + if err == nil { + t.Fatal("expected error for state=false, got nil") + } + if !strings.Contains(err.Error(), "990009") || !strings.Contains(err.Error(), "upload failed") { + t.Fatalf("error should contain code and message, got: %v", err) + } +} + +func TestCheckUploadCallbackEmptyBody(t *testing.T) { + err := checkUploadCallback([]byte{}) + if err == nil { + t.Fatal("expected error for empty body, got nil") + } + if !strings.Contains(err.Error(), "empty") { + t.Fatalf("error should mention empty, got: %v", err) + } +} + +func TestCheckUploadCallbackInvalidJSON(t *testing.T) { + err := checkUploadCallback([]byte(`not json`)) + if err == nil { + t.Fatal("expected error for invalid JSON, got nil") + } + if !strings.Contains(err.Error(), "parse error") { + t.Fatalf("error should mention parse error, got: %v", err) + } +} + +func TestGetReturnsObjForExistingFolder(t *testing.T) { + driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, map[string]any{ + "file_id": "99999", + "file_name": "my_folder", + "pick_code": "pc-123", + "file_category": "0", // folder; Fix 4 rejects file responses with NotImplement + }) + }) + driver.parentPath = "" + + obj, err := driver.Get(context.Background(), "/my_folder") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if obj.GetID() != "99999" || obj.GetName() != "my_folder" { + t.Fatalf("unexpected obj: id=%s name=%s", obj.GetID(), obj.GetName()) + } +} diff --git a/drivers/115_open/get.go b/drivers/115_open/get.go new file mode 100644 index 0000000000..3f18d18a1f --- /dev/null +++ b/drivers/115_open/get.go @@ -0,0 +1,62 @@ +package _115_open + +import ( + "errors" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// isObjectNotFound reports whether err means the 115 API could not find the +// requested path, so driver.Get can translate it into errs.ObjectNotFound and +// let op.Get fall through instead of surfacing a raw SDK error. The SDK +// surfaces "not found" two ways and both must be honored: +// - ErrDataEmpty: GetFolderInfoByPath returned an empty array. This is the +// realistic case — request.go collapses an empty/`[]` payload to +// ErrDataEmpty before the folder-info unmarshal can run. +// - ErrObjectNotFound: upstream #2596's mapping of SDK not-found errors. +// Currently shadowed for this call path but kept as a forward-compatible +// second arm so a future SDK that surfaces it directly still works. +func isObjectNotFound(err error) bool { + return errors.Is(err, sdk.ErrObjectNotFound) || errors.Is(err, sdk.ErrDataEmpty) +} + +// folderInfoToObj extracts the fields driver.Get needs from a +// GetFolderInfoByPath response. We only look at FileID + FileName + +// FileCategory — Sha1 / PickCode come along because the struct already +// carries them, but Size is intentionally dropped: the API is +// folder-oriented and resp.Size is empty/garbage for file paths. +// fromFolderInfo rejects files outright so the parsed value would never +// be read anyway. +func folderInfoToObj(resp *sdk.GetFolderInfoResp) *Obj { + return &Obj{ + Fid: resp.FileID, + Fn: resp.FileName, + Fc: resp.FileCategory, + Sha1: resp.Sha1, + Pc: resp.PickCode, + } +} + +// fromFolderInfo is the gateway used by driver.Get. Files are rejected +// with errs.NotImplement so op.Get falls through to its list-based +// path — that response carries GetFilesResp_File.FS (int64) and is +// always correct. Folders take the fast path: one +// GetFolderInfoByPath call is enough to build the Obj that +// op.list will pass to driver.List as the parent directory. +// +// Trade-off: cold-cache file access pays one wasted +// GetFolderInfoByPath (this call) + one folder GetFolderInfoByPath +// (parent) + one GetFiles (list parent) + the eventual DownURL = 4 +// WaitLimit-gated SDK calls. Default limit_rate is 1 req/s so this +// is ~3s of pure rate-limit wait on top of network. Steady-state +// access within dirCache TTL (5 min) collapses back to a single +// DownURL call. +func fromFolderInfo(resp *sdk.GetFolderInfoResp) (model.Obj, error) { + obj := folderInfoToObj(resp) + if !obj.IsDir() { + return nil, errs.NotImplement + } + return obj, nil +} diff --git a/drivers/115_open/get_test.go b/drivers/115_open/get_test.go new file mode 100644 index 0000000000..d53423f2aa --- /dev/null +++ b/drivers/115_open/get_test.go @@ -0,0 +1,88 @@ +package _115_open + +import ( + "errors" + "fmt" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +// driver.Get's only job after Fix 4 is "is this a folder, and if so what's +// its FileID/name". Files are rejected with errs.NotImplement so op.Get +// falls back to the list-based path — that response carries +// GetFilesResp_File.FS as int64 and is always correct. The 115 +// "Get folder info" API (yuque rl8zrhe2nag21dfw) is folder-oriented and +// resp.Size is unreliable for file paths anyway. + +func TestFolderInfoToObj_Folder(t *testing.T) { + resp := &sdk.GetFolderInfoResp{ + FileID: "1234", FileName: "Movies", FileCategory: "0", + } + obj := folderInfoToObj(resp) + if !obj.IsDir() { + t.Fatalf("IsDir() = false, want true (FileCategory 0 = folder)") + } + if obj.GetID() != "1234" || obj.GetName() != "Movies" { + t.Fatalf("obj = %+v, want id=1234 name=Movies", obj) + } +} + +func TestFromFolderInfo_FileReturnsNotImplement(t *testing.T) { + // File path response: even with sha1/pickcode populated, fast-path + // must defer to list — Size is not byte-accurate for files. + resp := &sdk.GetFolderInfoResp{ + FileID: "3370024725891437210", FileName: "Django Unchained 2012.mkv", + FileCategory: "1", Sha1: "17C6301550DE8E22D477EB9BA3901A99B9961494", + PickCode: "cqghjg71avvncddhi", Size: "36767958354", + } + obj, err := fromFolderInfo(resp) + if obj != nil { + t.Fatalf("obj = %+v, want nil (files must defer to List)", obj) + } + if !errors.Is(err, errs.NotImplement) { + t.Fatalf("err = %v, want errs.NotImplement (so op.Get falls through to list path)", err) + } +} + +// driver.Get maps "path not found" to errs.ObjectNotFound so op.Get can fall +// through cleanly instead of surfacing a raw SDK error. The SDK surfaces this +// two ways and both must be honored: ErrDataEmpty (GetFolderInfoByPath returns +// an empty array — the realistic case) and ErrObjectNotFound (upstream #2596's +// SDK not-found mapping, kept as a forward-compatible second arm). Any other +// error must NOT be swallowed. +func TestIsObjectNotFound(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"ErrDataEmpty (empty folder-info array)", sdk.ErrDataEmpty, true}, + {"ErrObjectNotFound (upstream #2596 mapping)", sdk.ErrObjectNotFound, true}, + {"wrapped ErrDataEmpty", fmt.Errorf("GetFolderInfoByPath: %w", sdk.ErrDataEmpty), true}, + {"wrapped ErrObjectNotFound", fmt.Errorf("GetFolderInfoByPath: %w", sdk.ErrObjectNotFound), true}, + {"unrelated error must propagate", errors.New("429 rate limited"), false}, + {"nil is not not-found", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isObjectNotFound(tt.err); got != tt.want { + t.Fatalf("isObjectNotFound(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +func TestFromFolderInfo_FolderReturnsObj(t *testing.T) { + resp := &sdk.GetFolderInfoResp{ + FileID: "1234", FileName: "Movies", FileCategory: "0", + } + obj, err := fromFolderInfo(resp) + if err != nil { + t.Fatalf("unexpected err = %v", err) + } + if obj == nil || !obj.IsDir() || obj.GetID() != "1234" { + t.Fatalf("obj = %+v, want non-nil folder with ID 1234", obj) + } +} diff --git a/drivers/115_open/link_expiry.go b/drivers/115_open/link_expiry.go new file mode 100644 index 0000000000..f92f401c4a --- /dev/null +++ b/drivers/115_open/link_expiry.go @@ -0,0 +1,40 @@ +package _115_open + +import ( + "net/url" + "strconv" + "time" +) + +// 115 CDN download URLs carry `?t=` marking when the signed URL stops +// serving bytes. The real lifetime is often a few minutes, well below +// OpenList's default link-cache TTL — when the cache outlives the URL, OP +// hands out an expired CDN link and the response is "200 OK + empty body". +// +// parseCDNExpiry extracts that timestamp and turns it into a TTL suitable +// for model.Link.Expiration, with a safety margin so OP refreshes slightly +// before the CDN actually rejects. +const ( + cdnExpirySafetyMargin = 60 * time.Second + cdnExpiryMinimum = 1 * time.Second +) + +func parseCDNExpiry(rawURL string) (time.Duration, bool) { + parsed, err := url.Parse(rawURL) + if err != nil { + return 0, false + } + tStr := parsed.Query().Get("t") + if tStr == "" { + return 0, false + } + tUnix, err := strconv.ParseInt(tStr, 10, 64) + if err != nil { + return 0, false + } + remaining := time.Until(time.Unix(tUnix, 0)) - cdnExpirySafetyMargin + if remaining < cdnExpiryMinimum { + return cdnExpiryMinimum, true + } + return remaining, true +} diff --git a/drivers/115_open/link_expiry_test.go b/drivers/115_open/link_expiry_test.go new file mode 100644 index 0000000000..4a19686ce0 --- /dev/null +++ b/drivers/115_open/link_expiry_test.go @@ -0,0 +1,150 @@ +package _115_open + +import ( + "fmt" + "testing" + "time" +) + +// 115 CDN download URLs carry a Unix timestamp in `t=...` that marks when +// the signed URL stops accepting requests. The actual lifetime is typically +// just a few minutes — much shorter than OpenList's default link-cache TTL. +// `parseCDNExpiry` reads `t=` and returns the time-to-live to plug into +// model.Link.Expiration so OP refreshes the link before the CDN does. + +func TestParseCDNExpiry_FutureTimestamp(t *testing.T) { + future := time.Now().Add(30 * time.Minute).Unix() + url := fmt.Sprintf("https://cdnfhnfdfs.115cdn.net/group518/foo.mkv?t=%d&u=123&s=52428800", future) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true for URL with future t=") + } + // Expect 30min minus the safety margin. Allow ±5s slack for the + // time.Now() jitter between Unix() above and time.Until() inside. + want := 30*time.Minute - cdnExpirySafetyMargin + if d < want-5*time.Second || d > want+5*time.Second { + t.Fatalf("d = %v, want ~%v", d, want) + } +} + +func TestParseCDNExpiry_PastTimestamp_ClampedToMinimum(t *testing.T) { + past := time.Now().Add(-10 * time.Minute).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", past) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true even for expired t= (caller decides what to do)") + } + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } +} + +func TestParseCDNExpiry_AboutToExpire_ClampedToMinimum(t *testing.T) { + // 30 seconds in the future, less than safety margin (60s) → would yield + // a negative duration. Should clamp. + soon := time.Now().Add(30 * time.Second).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", soon) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true") + } + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } +} + +func TestParseCDNExpiry_MissingT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?u=123&s=52428800") + if ok { + t.Fatalf("ok = true, want false for URL without t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_MalformedT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=notanumber&u=123") + if ok { + t.Fatalf("ok = true, want false for malformed t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_EmptyT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=&u=123") + if ok { + t.Fatalf("ok = true, want false for empty t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_NegativeT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=-1") + if ok { + // negative parses as Int64 but maps to a past time → ok=true with clamp + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } + } else { + // alternative acceptable behavior: reject negative as malformed. + // Either policy is defensible; pin whichever the implementation chose. + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } + } +} + +func TestParseCDNExpiry_InvalidURL(t *testing.T) { + // url.Parse is permissive — most "garbage in" still parses, but the + // query bag is empty so t= is missing. + d, ok := parseCDNExpiry("not_a_url_at_all") + if ok { + t.Fatalf("ok = true, want false for garbage URL with no query") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_RealWorld115URL(t *testing.T) { + // Anchored on the actual URL the user pasted, but with t= rewritten to a + // known future point so the test isn't time-bombed. + future := time.Now().Add(2 * time.Hour).Unix() + url := fmt.Sprintf( + "https://cdnfhnfdfs.115cdn.net/group518/M00/5B/9F/tzyQp1JGFCUAAAAIj4qFUklVFs09176478/Django%%20Unchained%%202012.mkv?t=%d&u=103088508&s=52428800&d=vip-2559104837-cqghjg71avvncddhi-1-100195313&c=2&f=1&k=44423951d519b201c3b42e03556e724b&us=62914560&uc=10&v=1", + future, + ) + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false on real-world URL") + } + if d <= 0 || d > 2*time.Hour { + t.Fatalf("d = %v, want in (0, 2h]", d) + } +} + +func TestParseCDNExpiry_SafetyMarginApplied(t *testing.T) { + // Verify the margin actually shrinks the returned duration. Use a fixed + // offset much larger than the safety margin so jitter doesn't matter. + raw := 1 * time.Hour + future := time.Now().Add(raw).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", future) + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false") + } + if d >= raw { + t.Fatalf("d = %v, expected strictly less than raw=%v (safety margin not applied)", d, raw) + } + if d < raw-2*cdnExpirySafetyMargin { + t.Fatalf("d = %v, expected ≥ raw-2*margin=%v (margin too aggressive)", d, raw-2*cdnExpirySafetyMargin) + } +} diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index ed908e2e62..0479ac1fdb 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -11,6 +11,7 @@ type Addition struct { // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete" default:"trash"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` @@ -18,9 +19,10 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", - LinkCacheMode: driver.LinkCacheUA, + Name: "115 Open", + DefaultRoot: "0", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/types.go b/drivers/115_open/types.go index 0bd95bfd19..7bc7443ba5 100644 --- a/drivers/115_open/types.go +++ b/drivers/115_open/types.go @@ -55,5 +55,80 @@ func (o *Obj) ModTime() time.Time { return time.Unix(o.Upt, 0) } +// Extra surfaces 115-specific metadata the universal Obj cannot carry: media +// duration (seconds), video resolution label, the starred flag and file tags. +// Only non-empty values are included, so the frontend renders whatever is +// present and ignores the rest — adding/removing keys never breaks the client, +// and a changed 115 payload degrades to "no extra info" rather than an error. +func (o *Obj) Extra() map[string]any { + extra := map[string]any{} + if o.Ism == "1" { + extra["starred"] = true + } + if d := o.durationSec(); d > 0 { + extra["duration"] = d + } + if label := defLabel(o.maxDef()); label != "" { + extra["resolution"] = label + } + if len(o.Fl) > 0 { + tags := make([]string, 0, len(o.Fl)) + for _, t := range o.Fl { + if t.Name != "" { + tags = append(tags, t.Name) + } + } + if len(tags) > 0 { + extra["tags"] = tags + } + } + if len(extra) == 0 { + return nil + } + return extra +} + +// durationSec parses 115's play_long (json.Number, seconds) defensively. +func (o *Obj) durationSec() float64 { + if o.PlayLong == "" { + return 0 + } + f, err := o.PlayLong.Float64() + if err != nil || f <= 0 { + return 0 + } + return f +} + +// maxDef prefers the higher of the two resolution fields 115 reports. +func (o *Obj) maxDef() int64 { + if o.Def2 > o.Def { + return o.Def2 + } + return o.Def +} + +// defLabel maps 115's video-definition code to a short badge. Unknown codes +// (including 0 for non-videos) return "" so no badge is shown. +func defLabel(def int64) string { + switch def { + case 1: + return "SD" + case 2: + return "HD" + case 3: + return "FHD" + case 4: + return "1080P" + case 5: + return "4K" + case 100: + return "原画" + default: + return "" + } +} + var _ model.Obj = (*Obj)(nil) var _ model.Thumb = (*Obj)(nil) +var _ model.ObjExtra = (*Obj)(nil) diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2c4..8669cc81bc 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -3,7 +3,10 @@ package _115_open import ( "context" "encoding/base64" + "encoding/json" + "fmt" "io" + "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" @@ -14,8 +17,55 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" ) +type UploadCallbackResult struct { + State bool `json:"state"` + Code int `json:"code"` + Message string `json:"message"` + Data struct { + PickCode string `json:"pick_code"` + FileName string `json:"file_name"` + FileSize int64 `json:"file_size"` + FileID string `json:"file_id"` + Sha1 string `json:"sha1"` + Cid string `json:"cid"` + } `json:"data"` +} + +func checkUploadCallback(bodyBytes []byte) error { + if len(bodyBytes) == 0 { + return fmt.Errorf("115 upload callback returned empty response") + } + var result UploadCallbackResult + if err := json.Unmarshal(bodyBytes, &result); err != nil { + return fmt.Errorf("115 upload callback response parse error: %w (body: %s)", err, string(bodyBytes)) + } + if !result.State { + return fmt.Errorf("115 upload callback failed: code=%d, message=%s", result.Code, result.Message) + } + return nil +} + +// isTokenExpiredError 检测是否为OSS凭证过期错误 +func isTokenExpiredError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "SecurityTokenExpired") || + strings.Contains(errStr, "InvalidAccessKeyId") +} + +// isPartAlreadyExistError 检测是否为分片已存在错误（超时后重试时 OSS 返回 409） +func isPartAlreadyExistError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "PartAlreadyExist") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -46,36 +96,29 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp return err } + var bodyBytes []byte err = bucket.PutObject(initResp.Object, tempF, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + oss.CallbackResult(&bodyBytes), ) - - return err + if err != nil { + return err + } + return checkUploadCallback(bodyBytes) } -// type CallbackResult struct { -// State bool `json:"state"` -// Code int `json:"code"` -// Message string `json:"message"` -// Data struct { -// PickCode string `json:"pick_code"` -// FileName string `json:"file_name"` -// FileSize int64 `json:"file_size"` -// FileID string `json:"file_id"` -// ThumbURL string `json:"thumb_url"` -// Sha1 string `json:"sha1"` -// Aid int `json:"aid"` -// Cid string `json:"cid"` -// } `json:"data"` -// } - func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err + // 创建OSS客户端的辅助函数 + createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { + ossClient, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return nil, err + } + return ossClient.Bucket(initResp.Bucket) } - bucket, err := ossClient.Bucket(initResp.Bucket) + + bucket, err := createBucket(tokenResp) if err != nil { return err } @@ -112,6 +155,21 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, rd.Seek(0, io.SeekStart) part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) if err != nil { + if isPartAlreadyExistError(err) { + log.Infof("115 OSS part %d already exists, retrieving from ListUploadedParts", i) + lpr, listErr := bucket.ListUploadedParts(imur) + if listErr != nil { + return fmt.Errorf("part %d already exists but ListUploadedParts failed: %w", i, listErr) + } + for _, p := range lpr.UploadedParts { + if p.PartNumber == int(i) { + parts[i-1] = oss.UploadPart{PartNumber: p.PartNumber, ETag: p.ETag} + log.Infof("115 OSS part %d recovered: ETag=%s", i, p.ETag) + return nil + } + } + return fmt.Errorf("part %d reported as existing but not found in ListUploadedParts", i) + } return err } parts[i-1] = part @@ -120,7 +178,23 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second)) + retry.Delay(time.Second), + retry.OnRetry(func(n uint, err error) { + if isTokenExpiredError(err) { + log.Warnf("115 OSS token expired, refreshing token...") + if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { + tokenResp = newToken + if newBucket, bucketErr := createBucket(tokenResp); bucketErr == nil { + bucket = newBucket + log.Infof("115 OSS token refreshed successfully") + } else { + log.Errorf("Failed to create new bucket with refreshed token: %v", bucketErr) + } + } else { + log.Errorf("Failed to refresh 115 OSS token: %v", refreshErr) + } + } + })) ss.FreeSectionReader(rd) if err != nil { return err @@ -134,17 +208,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up(float64(offset) * 100 / float64(fileSize)) } - // callbackRespBytes := make([]byte, 1024) + var bodyBytes []byte _, err = bucket.CompleteMultipartUpload( imur, parts, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), - // oss.CallbackResult(&callbackRespBytes), + oss.CallbackResult(&bodyBytes), ) if err != nil { return err } - - return nil + return checkUploadCallback(bodyBytes) } diff --git a/drivers/115_open/upload_test.go b/drivers/115_open/upload_test.go new file mode 100644 index 0000000000..5ad2ff1a06 --- /dev/null +++ b/drivers/115_open/upload_test.go @@ -0,0 +1,29 @@ +package _115_open + +import "testing" + +func TestIsPartAlreadyExistError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"generic error", errStr("some random error"), false}, + {"timeout", errStr("net/http: timeout awaiting response headers"), false}, + {"part already exist", errStr(`oss: service returned error: StatusCode=409, ErrorCode=PartAlreadyExist, ErrorMessage="For sequential multipart upload, you can't overwrite uploaded parts."`), true}, + {"partial match", errStr("PartAlreadyExist"), true}, + {"case sensitive", errStr("partalreadyexist"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isPartAlreadyExistError(tt.err); got != tt.want { + t.Errorf("isPartAlreadyExistError() = %v, want %v", got, tt.want) + } + }) + } +} + +type errStr string + +func (e errStr) Error() string { return string(e) } diff --git a/drivers/115_open/video.go b/drivers/115_open/video.go new file mode 100644 index 0000000000..6c6f23c4d3 --- /dev/null +++ b/drivers/115_open/video.go @@ -0,0 +1,107 @@ +package _115_open + +import ( + "context" + "sort" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// toVideoPlayInfos maps the SDK video-play URLs to driver.VideoPlayInfo, +// sorted by Definition descending (highest quality first). +func toVideoPlayInfos(urls []sdk.VideoPlayURL) []driver.VideoPlayInfo { + infos := make([]driver.VideoPlayInfo, 0, len(urls)) + for _, u := range urls { + infos = append(infos, driver.VideoPlayInfo{ + Resolution: u.Desc, + Definition: u.Definition, + URL: u.URL, + }) + } + sort.SliceStable(infos, func(i, j int) bool { + return infos[i].Definition > infos[j].Definition + }) + return infos +} + +// toVideoSubtitleInfos maps the SDK subtitle list to driver.VideoSubtitleInfo, +// dropping caption-map placeholder entries and any without a downloadable URL. +func toVideoSubtitleInfos(list []sdk.SubtitleItem) []driver.VideoSubtitleInfo { + infos := make([]driver.VideoSubtitleInfo, 0, len(list)) + for _, s := range list { + if s.URL == "" || s.IsCaptionMap == 1 { + continue + } + title := s.Title + if title == "" { + title = s.Language + } + infos = append(infos, driver.VideoSubtitleInfo{ + Language: s.Language, + Title: title, + URL: s.URL, + Type: s.Type, + }) + } + return infos +} + +// VideoSubtitle returns the provider's subtitle tracks for a video. 115 extracts +// a container's embedded (and any sidecar) subtitles during transcoding and +// serves them as standalone files, so these work on every play source — +// including the transcoded HLS streams that drop the original embedded tracks. +func (d *Open115) VideoSubtitle(ctx context.Context, file model.Obj) ([]driver.VideoSubtitleInfo, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + obj, ok := file.(*Obj) + if !ok { + return nil, errors.New("can't convert obj") + } + pc := obj.Pc + if pc == "" { + return nil, errors.New("can't get pick code") + } + + resp, err := d.client.VideoSubtitle(ctx, pc) + if err != nil { + log.Errorf("[115] VideoSubtitle API failed: %v", err) + return nil, errors.WithStack(err) + } + if resp == nil { + return nil, nil + } + + return toVideoSubtitleInfos(resp.List), nil +} + +// VideoPlay returns the official online-play (transcoded streaming) sources +// for a video at multiple resolutions. +func (d *Open115) VideoPlay(ctx context.Context, file model.Obj) ([]driver.VideoPlayInfo, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + obj, ok := file.(*Obj) + if !ok { + return nil, errors.New("can't convert obj") + } + pc := obj.Pc + if pc == "" { + return nil, errors.New("can't get pick code") + } + + resp, err := d.client.VideoPlay(ctx, pc) + if err != nil { + log.Errorf("[115] VideoPlay API failed: %v", err) + return nil, errors.WithStack(err) + } + if resp == nil || len(resp.VideoURL) == 0 { + return nil, errors.New("no official play sources") + } + + return toVideoPlayInfos(resp.VideoURL), nil +} diff --git a/drivers/115_open/video_test.go b/drivers/115_open/video_test.go new file mode 100644 index 0000000000..f63bec0ef9 --- /dev/null +++ b/drivers/115_open/video_test.go @@ -0,0 +1,48 @@ +package _115_open + +import ( + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" +) + +func TestToVideoPlayInfos_SortAndMap(t *testing.T) { + urls := []sdk.VideoPlayURL{ + {URL: "http://example.com/720", Definition: 2, Desc: "720P"}, + {URL: "http://example.com/1080", Definition: 4, Desc: "1080P"}, + {URL: "http://example.com/360", Definition: 1, Desc: "360P"}, + } + + got := toVideoPlayInfos(urls) + + if len(got) != 3 { + t.Fatalf("expected 3 infos, got %d", len(got)) + } + + // Sorted by Definition descending (highest quality first). + wantDefinitions := []int{4, 2, 1} + wantResolutions := []string{"1080P", "720P", "360P"} + wantURLs := []string{"http://example.com/1080", "http://example.com/720", "http://example.com/360"} + + for i := range got { + if got[i].Definition != wantDefinitions[i] { + t.Errorf("index %d: Definition = %d, want %d", i, got[i].Definition, wantDefinitions[i]) + } + if got[i].Resolution != wantResolutions[i] { + t.Errorf("index %d: Resolution = %q, want %q", i, got[i].Resolution, wantResolutions[i]) + } + if got[i].URL != wantURLs[i] { + t.Errorf("index %d: URL = %q, want %q", i, got[i].URL, wantURLs[i]) + } + } +} + +func TestToVideoPlayInfos_Empty(t *testing.T) { + got := toVideoPlayInfos(nil) + if got == nil { + t.Fatalf("expected non-nil empty slice, got nil") + } + if len(got) != 0 { + t.Fatalf("expected empty slice, got %d elements", len(got)) + } +} diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index fe8b7733a4..8f8cac01e1 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -2,6 +2,7 @@ package _115_share import ( "context" + "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -96,8 +97,12 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err != nil { return nil, err } - - return &model.Link{URL: downloadInfo.URL.URL}, nil + header := http.Header{} + header.Set("User-Agent", ua) + return &model.Link{ + URL: downloadInfo.URL.URL, + Header: header, + }, nil } func (d *Pan115Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/123/util.go b/drivers/123/util.go index 5cce49b65b..d53a3d9db8 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -181,7 +181,7 @@ func (d *Pan123) login() error { return err } if utils.Json.Get(res.Body(), "code").ToInt() != 200 { - err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString()) + err = fmt.Errorf("%s", utils.Json.Get(res.Body(), "message").ToString()) } else { d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString() } diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 78ff272b9b..9adb1aed08 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -175,7 +175,7 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { @@ -197,14 +197,49 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(etag) >= utils.MD5.Width { + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return nil, err + } + if createResp.Data.Reuse && createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + // 秒传失败，继续后续流程 + } + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(etag) < utils.MD5.Width { + etag, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return nil, err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 秒传失败或没有 hash，缓存整个文件并计算 MD5 _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } } + + // 2. 创建上传任务（或再次尝试秒传） createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -223,13 +258,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/139/util.go b/drivers/139/util.go index 85c798cc7c..c026de52a2 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1180,7 +1180,6 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", "Content-Type": "text/plain;charset=UTF-8", "Host": "user-njs.yun.139.com", - "Connection": "Keep-Alive", "Accept-Encoding": "gzip", "User-Agent": "okhttp/3.12.2", } diff --git a/drivers/189/torrent.go b/drivers/189/torrent.go new file mode 100644 index 0000000000..4e41bf4905 --- /dev/null +++ b/drivers/189/torrent.go @@ -0,0 +1,149 @@ +package _189 + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +func (d *Cloud189) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte) error { + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 获取 sessionKey + sessionKey, err := d.getSessionKey() + if err != nil { + return err + } + d.sessionKey = sessionKey + + // 初始化上传 + res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + "parentFolderId": dstDir.GetID(), + "fileName": encode(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(cas.SliceSize), + "lazyCheck": "1", + }, nil) + if err != nil { + return fmt.Errorf("初始化上传失败: %w", err) + } + + uploadFileId := utils.Json.Get(res, "data", "uploadFileId").ToString() + + // 提交上传（使用 CAS 信息秒传） + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": cas.FileMD5, + "sliceMd5": cas.SliceMD5, + "lazyCheck": "1", + "opertype": "3", + }, nil) + if err != nil { + return fmt.Errorf("秒传提交失败: %w", err) + } + + return nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} diff --git a/drivers/189/util.go b/drivers/189/util.go index bb9a6adb4f..5fad92c1dd 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/md5" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "errors" @@ -18,6 +19,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" @@ -235,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } - log.Debugf(res.String()) + log.Debugf("%s", res.String()) return errors.New(res.String()) } @@ -311,48 +314,107 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } d.sessionKey = sessionKey const DEFAULT int64 = 10485760 - count := int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + fileSize := file.GetSize() + count := int64(math.Ceil(float64(fileSize) / float64(DEFAULT))) - res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + // 先计算文件完整MD5和分片MD5，用于秒传判断 + fileMd5Hex := file.GetHash().GetHash(utils.MD5) + sliceMd5Hex := "" + md5s := make([]string, 0) + + if len(fileMd5Hex) < utils.MD5.Width { + // 没有MD5，先缓存流并同时计算文件MD5和分片MD5 + fileMd5Hash := md5.New() + sliceMd5Hash := md5.New() + var finish int64 + cache, err := file.CacheFullAndWriter(nil, io.MultiWriter(fileMd5Hash, &sliceHashWriter{ + hash: sliceMd5Hash, + md5s: &md5s, + sliceSize: DEFAULT, + finish: &finish, + fileSize: fileSize, + up: up, + ctx: ctx, + })) + if err != nil { + return err + } + // 处理最后一个分片的MD5 + if finish%DEFAULT != 0 || finish == 0 { + md5s = append(md5s, strings.ToUpper(hex.EncodeToString(sliceMd5Hash.Sum(nil)))) + } + fileMd5Hex = hex.EncodeToString(fileMd5Hash.Sum(nil)) + + // seek回起始位置，供后续上传使用 + if _, err := cache.Seek(0, io.SeekStart); err != nil { + return err + } + } + + // 计算sliceMd5 + if fileSize > DEFAULT && len(md5s) > 0 { + sliceMd5Hex = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + } else { + sliceMd5Hex = fileMd5Hex + } + + // 带fileMd5调用initMultiUpload，支持秒传 + initParams := map[string]string{ "parentFolderId": dstDir.GetID(), "fileName": encode(file.GetName()), - "fileSize": strconv.FormatInt(file.GetSize(), 10), + "fileSize": strconv.FormatInt(fileSize, 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), - "lazyCheck": "1", - }, nil) + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + } + + res, err := d.uploadRequest("/person/initMultiUpload", initParams, nil) if err != nil { return err } uploadFileId := jsoniter.Get(res, "data", "uploadFileId").ToString() - //_, err = d.uploadRequest("/person/getUploadedPartsInfo", map[string]string{ - // "uploadFileId": uploadFileId, - //}, nil) + fileDataExists := jsoniter.Get(res, "data", "fileDataExists").ToInt() + + // 秒传成功，直接提交 + if fileDataExists == 1 { + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": "3", + }, nil) + return err + } + + // 非秒传，需要上传分片 var finish int64 = 0 var i int64 var byteSize int64 - md5s := make([]string, 0) - md5Sum := md5.New() + + // 额外计算 SHA-1 piece hash 用于生成 torrent + pieceSHA1Hashes := make([]byte, 0, int(count)*20) + for i = 1; i <= count; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize = file.GetSize() - finish + byteSize = fileSize - finish if DEFAULT < byteSize { byteSize = DEFAULT } - // log.Debugf("%d,%d", byteSize, finish) byteData := make([]byte, byteSize) n, err := io.ReadFull(file, byteData) - // log.Debug(err, n) if err != nil { return err } finish += int64(n) md5Bytes := getMd5(byteData) - md5Hex := hex.EncodeToString(md5Bytes) md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) - md5s = append(md5s, strings.ToUpper(md5Hex)) - md5Sum.Write(byteData) + + // 计算 SHA-1 piece hash + sha1Hash := sha1Pkg.Sum(byteData) + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Hash[:]...) var resp UploadUrlsResp res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{ "partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64), @@ -379,21 +441,58 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } log.Debugf("%+v %+v", r, r.Request.Header) _ = r.Body.Close() - up(float64(i) * 100 / float64(count)) - } - fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) - sliceMd5 := fileMd5 - if file.GetSize() > DEFAULT { - sliceMd5 = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + up(50 + float64(i)*50/float64(count)) } res, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ "uploadFileId": uploadFileId, - "fileMd5": fileMd5, - "sliceMd5": sliceMd5, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "opertype": "3", }, nil) - return err + if err != nil { + return err + } + + // 生成 torrent 文件（异步，不影响上传结果） + capturedDstDir := dstDir + capturedFileName := file.GetName() + capturedFileSize := fileSize + capturedFileMd5Hex := fileMd5Hex + capturedMd5s := md5s + go func() { + fileMD5Upper := strings.ToUpper(capturedFileMd5Hex) + torrentData, err := GenerateTorrent(capturedFileName, capturedFileSize, fileMD5Upper, capturedMd5s, DEFAULT, pieceSHA1Hashes) + if err != nil { + log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录 + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + uploadErr := d.oldUpload(capturedDstDir, torrentFileStream) + if uploadErr != nil { + log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(d, capturedDstDir.GetPath()) + } + }() + + return nil } func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { @@ -406,3 +505,52 @@ func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { } return &resp, nil } + +// sliceHashWriter 在写入过程中按分片大小自动切分并计算每个分片的MD5， +// 同时支持进度回调和取消检查。 +type sliceHashWriter struct { + hash io.Writer // 当前分片的MD5 hash + md5s *[]string // 收集每个分片的MD5十六进制字符串 + sliceSize int64 // 分片大小 + finish *int64 // 已写入的总字节数 + fileSize int64 // 文件总大小 + up driver.UpdateProgress + ctx context.Context +} + +func (w *sliceHashWriter) Write(p []byte) (int, error) { + if utils.IsCanceled(w.ctx) { + return 0, w.ctx.Err() + } + total := len(p) + written := 0 + for written < total { + // 当前分片还能写入的字节数 + sliceRemain := w.sliceSize - (*w.finish % w.sliceSize) + toWrite := int64(total - written) + if toWrite > sliceRemain { + toWrite = sliceRemain + } + n, err := w.hash.Write(p[written : written+int(toWrite)]) + if err != nil { + return written, err + } + written += n + *w.finish += int64(n) + + // 当前分片写满，记录MD5并重置 + if *w.finish%w.sliceSize == 0 { + if h, ok := w.hash.(interface{ Sum([]byte) []byte }); ok { + *w.md5s = append(*w.md5s, strings.ToUpper(hex.EncodeToString(h.Sum(nil)))) + } + if resetter, ok := w.hash.(interface{ Reset() }); ok { + resetter.Reset() + } + } + } + // 报告进度（缓存阶段占50%） + if w.fileSize > 0 && w.up != nil { + w.up(float64(*w.finish) / float64(w.fileSize) * 50) + } + return total, nil +} diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index af719401d3..82aa1c1af5 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -87,7 +87,12 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { } // 先尝试用Token刷新，之后尝试登陆 - if y.Addition.RefreshToken != "" { + if y.Addition.AccessToken != "" { + y.tokenInfo = &AppSessionResp{AccessToken: y.Addition.AccessToken, RefreshToken: y.Addition.RefreshToken} + if err = y.refreshSession(); err != nil { + return err + } + } else if y.Addition.RefreshToken != "" { y.tokenInfo = &AppSessionResp{RefreshToken: y.Addition.RefreshToken} if err = y.refreshToken(); err != nil { return err @@ -288,7 +293,7 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin req.SetContext(ctx).SetQueryParams(queryParam) }, nil, resp, isFamily) if err != nil { - if resp.ResCode == "FileAlreadyExists" { + if code, ok := resp.ResCode.(string); ok && code == "FileAlreadyExists" { return nil, errs.ObjectAlreadyExists } return nil, err @@ -338,6 +343,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // 响应时间长,按需启用 if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { + // 尝试妙传 if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } @@ -346,10 +352,11 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil uploadMethod := y.UploadMethod if stream.IsForceStreamUpload() { uploadMethod = "stream" - } - - // 旧版上传家庭云也有限制 - if uploadMethod == "old" { + } else if y.Addition.RapidUpload && stream.GetFile() != nil { + // 文件流支持随机读取，走FastUpload计算MD5并尝试秒传 + uploadMethod = "rapid" + } else if uploadMethod == "old" { + // 旧版上传家庭云也有限制 return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) } diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 670b991163..3a5e02979d 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -10,17 +10,19 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` VCode string `json:"validate_code"` + AccessToken string `json:"access_token" required:"false"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - Type string `json:"type" type:"select" options:"personal,family" default:"personal"` - FamilyID string `json:"family_id"` - UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` - UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` - FamilyTransfer bool `json:"family_transfer"` - RapidUpload bool `json:"rapid_upload"` - NoUseOcr bool `json:"no_use_ocr"` + OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + Type string `json:"type" type:"select" options:"personal,family" default:"personal"` + FamilyID string `json:"family_id"` + UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + FamilyTransfer bool `json:"family_transfer"` + RapidUpload bool `json:"rapid_upload"` + NoUseOcr bool `json:"no_use_ocr"` + GenerateTorrent bool `json:"generate_torrent" help:"Generate torrent file with CAS extension after upload"` } var config = driver.Config{ diff --git a/drivers/189pc/torrent.go b/drivers/189pc/torrent.go new file mode 100644 index 0000000000..3068a0f7b9 --- /dev/null +++ b/drivers/189pc/torrent.go @@ -0,0 +1,296 @@ +package _189pc + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/url" + "strings" + + "github.com/go-resty/resty/v2" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +// fileMD5: 整文件 MD5（大写十六进制） +// sliceMD5s: 每个分片的 MD5 列表（大写十六进制） +// sliceSize: 分片大小 +// pieceHashes: SHA-1 piece hashes 拼接（每 20 字节一个） +// fileName: 文件名 +// fileSize: 文件大小 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +// 返回值：上传成功的文件对象、错误 +func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte, overwrite bool) (model.Obj, error) { + isFamily := y.isFamily() + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return nil, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 统一 MD5 为大写（与正常上传保持一致，天翼云盘要求大写） + fileMD5Upper := strings.ToUpper(cas.FileMD5) + + // 优先使用 torrent 中嵌入的分片大小，与生成时保持一致 + sliceSize := cas.SliceSize + if sliceSize <= 0 { + sliceSize = partSize(fileSize) + } + + // 计算 sliceMd5（与上传时一致的算法） + // 优先使用 torrent 中已有的 SliceMD5；仅当有多分片列表时才重新计算 + sliceMd5Hex := strings.ToUpper(cas.SliceMD5) + if sliceMd5Hex == "" { + sliceMd5Hex = fileMD5Upper + } + if len(cas.SliceMD5s) > 1 { + // 分片 MD5 也需要统一大写后再拼接计算 + upperSliceMD5s := make([]string, len(cas.SliceMD5s)) + for i, s := range cas.SliceMD5s { + upperSliceMD5s[i] = strings.ToUpper(s) + } + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n"))) + } + + + // 使用与 Web 端一致的三步秒传流程 + fullUrl := "https://upload.cloud.189.cn" + if isFamily { + fullUrl += "/family" + } else { + fullUrl += "/person" + } + + // Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck） + initParams := Params{ + "parentFolderId": dstDir.GetID(), + "fileName": url.QueryEscape(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(sliceSize), + "lazyCheck": "1", + } + if isFamily { + initParams.Set("familyId", y.FamilyID) + } + + + var uploadInfo InitMultiUploadResp + _, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, initParams, &uploadInfo, isFamily) + if err != nil { + return nil, fmt.Errorf("initMultiUpload 失败: %w", err) + } + + + uploadFileId := uploadInfo.Data.UploadFileID + + // Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传） + checkParams := Params{ + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "uploadFileId": uploadFileId, + } + + + var checkResp struct { + Data struct { + FileDataExists int `json:"fileDataExists"` + } `json:"data"` + } + _, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, checkParams, &checkResp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("秒传检查失败: %w", err) + } + + + if checkResp.Data.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize) + } + + // Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5） + + var resp CommitMultiUploadFileResp + commitParams := Params{ + "uploadFileId": uploadFileId, + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": IF(overwrite, "3", "1"), + } + + _, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, commitParams, &resp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return resp.toFile(), nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +// 适用于：已有文件需要生成 torrent 的场景（如下载完成后生成） +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +// 返回：CAS 信息、文件名、文件大小、错误 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// InjectCASIntoTorrent 向已有的 torrent 文件注入 CAS 扩展信息 +// 用于：下载完成后，计算了 MD5 信息，写回到 torrent 中 +func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string, sliceSize int64) ([]byte, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + // 注入 CAS 信息 + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + // 同时更新 info 中的 md5sum 字段 + if t.Info.MD5Sum == "" { + t.Info.MD5Sum = fileMD5 + } + + return t.Encode() +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} + +// ComputeSliceMD5sFromReader 从 reader 中计算每个 10MB 分片的 MD5 +// 返回：整文件 MD5、分片 MD5 列表 +func ComputeSliceMD5sFromReader(reader io.Reader, sliceSize int64) (string, []string, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + fileMD5Hash := utils.MD5.NewFunc() + sliceMD5s := make([]string, 0) + + buf := make([]byte, sliceSize) + for { + n, err := io.ReadFull(reader, buf) + if n > 0 { + chunk := buf[:n] + fileMD5Hash.Write(chunk) + // 计算该分片的 MD5 + sliceMD5 := strings.ToUpper(utils.HashData(utils.MD5, chunk)) + sliceMD5s = append(sliceMD5s, sliceMD5) + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return "", nil, err + } + } + + fileMD5Hex := strings.ToUpper(hex.EncodeToString(fileMD5Hash.Sum(nil))) + return fileMD5Hex, sliceMD5s, nil +} diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index eed447e25b..7952cb78a7 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -432,7 +432,7 @@ type RenameResp struct { ResMsg string `json:"res_message"` CreateDate Time `json:"createDate"` FileCate int `json:"fileCata"` - ID string `json:"id"` + ID String `json:"id"` LastOpTime Time `json:"lastOpTime"` MD5 string `json:"md5"` MediaType int `json:"mediaType"` @@ -441,12 +441,12 @@ type RenameResp struct { ParentID int64 `json:"parentId"` Rev string `json:"rev"` Size int64 `json:"size"` - ResCode string `json:"res_code"` + ResCode any `json:"res_code"` // int or string } func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { return &Cloud189File{ - ID: String(r.ID), + ID: r.ID, Name: r.Name, Size: r.Size, Md5: r.MD5, @@ -458,7 +458,7 @@ func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { func (r *RenameResp) toFolder() *Cloud189Folder { return &Cloud189Folder{ - ID: String(r.ID), + ID: r.ID, Name: r.Name, ParentID: r.ParentID, LastOpTime: r.LastOpTime, diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 08ee658ca0..86cf834e4e 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -3,6 +3,7 @@ package _189pc import ( "bytes" "context" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "encoding/xml" @@ -353,9 +354,10 @@ func (y *Cloud189PC) loginByPassword() (err error) { return &erron } if tokenInfo.ResCode != 0 { - err = fmt.Errorf(tokenInfo.ResMessage) + err = fmt.Errorf("%s", tokenInfo.ResMessage) return err } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) @@ -412,8 +414,9 @@ func (y *Cloud189PC) loginByQRCode() error { return err } if tokenInfo.ResCode != 0 { - return fmt.Errorf(tokenInfo.ResMessage) + return fmt.Errorf("%s", tokenInfo.ResMessage) } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) @@ -661,6 +664,7 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { return y.login() } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) @@ -739,6 +743,10 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo silceMd5 := utils.MD5.NewFunc() var writers io.Writer = silceMd5 + // 如果启用了 torrent 生成，额外计算 SHA-1 piece hash + generateTorrent := y.Addition.GenerateTorrent + pieceSHA1Hashes := make([]byte, 0, count*20) + fileMd5Hex := file.GetHash().GetHash(utils.MD5) var fileMd5 hash.Hash if len(fileMd5Hex) != utils.MD5.Width { @@ -763,7 +771,18 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo return err } silceMd5.Reset() - w, err := utils.CopyWithBuffer(writers, reader) + + // 如果需要生成 torrent，同时计算 SHA-1 + var sha1Writer hash.Hash + var multiWriter io.Writer + if generateTorrent { + sha1Writer = sha1Pkg.New() + multiWriter = io.MultiWriter(writers, sha1Writer) + } else { + multiWriter = writers + } + + w, err := utils.CopyWithBuffer(multiWriter, reader) if w != partSize { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) } @@ -771,6 +790,11 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo md5Bytes := silceMd5.Sum(nil) silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) + + // 收集 SHA-1 piece hash + if generateTorrent && sha1Writer != nil { + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Writer.Sum(nil)...) + } return nil }, Do: func(ctx context.Context) (err error) { @@ -824,6 +848,45 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return nil, err } + + // 生成 torrent 文件（异步，不影响上传结果） + if generateTorrent && len(pieceSHA1Hashes) > 0 { + // 捕获必要的变量 + capturedDstDir := dstDir + capturedIsFamily := isFamily + capturedFileName := file.GetName() + go func() { + torrentData, err := GenerateTorrent(capturedFileName, fileSize, fileMd5Hex, silceMd5Hexs, sliceSize, pieceSHA1Hashes) + if err != nil { + utils.Log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + utils.Log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录（使用 FastUpload，因为 torrent 文件很小） + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + _, uploadErr := y.FastUpload(context.Background(), capturedDstDir, torrentFileStream, func(p float64) {}, capturedIsFamily, false) + if uploadErr != nil { + utils.Log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + utils.Log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(y, capturedDstDir.GetPath()) + } + }() + } + return resp.toFile(), nil } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8e5eb8a843..b378543940 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -40,7 +40,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if !withDetails || len(v) != 1 { continue } - remoteDriver, err := op.GetStorageByMountPath(v[0]) + remoteDriver, err := fs.GetStorage(v[0], &fs.GetStoragesArgs{}) if err != nil { continue } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index a4a6c1de1f..5f02c75f57 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -163,21 +163,29 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) + + // 检查是否是可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") - // read 1024 bytes to calculate pre hash - reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) - if err != nil { - return nil, err - } - hash, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return nil, err + // 优先使用预计算的 pre_hash + preHash := stream.GetHash().GetHash(utils.PRE_HASH) + if len(preHash) != utils.PRE_HASH.Width { + // 没有预计算的 pre_hash，使用 RangeRead 计算 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) + if err != nil { + return nil, err + } + preHash, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } } createData["size"] = stream.GetSize() - createData["pre_hash"] = hash + createData["pre_hash"] = preHash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { @@ -191,9 +199,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { - _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 StreamHashFile（RangeRead），不缓存 + hash, err = streamPkg.StreamHashFile(stream, utils.SHA1, 100, &up) + if err != nil { + return nil, err + } + } else { + // 不可重复读取的流，缓存并计算 + _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) + if err != nil { + return nil, err + } } } diff --git a/drivers/all.go b/drivers/all.go index fb68d03950..4af88dc00c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" + _ "github.com/OpenListTeam/OpenList/v4/drivers/emby" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/ftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/github" diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index fe77aca38b..474dd2b987 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,30 +1,18 @@ package baidu_netdisk import ( - "bytes" "context" - "crypto/md5" - "encoding/hex" "errors" - "io" - "mime/multipart" - "net/http" "net/url" - "os" stdpath "path" "strconv" - "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" - "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -37,6 +25,7 @@ type BaiduNetdisk struct { } var ErrUploadIDExpired = errors.New("uploadid expired") +var ErrUploadURLExpired = errors.New("upload url expired or unavailable") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -199,80 +188,26 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - var ( - cache = stream.GetFile() - tmpF *os.File - err error - ) - if cache == nil { - tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - defer func() { - _ = tmpF.Close() - _ = os.Remove(tmpF.Name()) - }() - cache = tmpF - } - streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } - lastBlockSize := streamSize % sliceSize - if lastBlockSize == 0 { - lastBlockSize = sliceSize - } - - // cal md5 for first 256k data - const SliceSize int64 = 256 * utils.KB - blockList := make([]string, 0, count) - byteSize := sliceSize - fileMd5H := md5.New() - sliceMd5H := md5.New() - sliceMd5H2 := md5.New() - slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) - writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} - if tmpF != nil { - writers = append(writers, tmpF) - } - written := int64(0) - for i := 1; i <= count; i++ { - if utils.IsCanceled(ctx) { - return nil, ctx.Err() - } - if i == count { - byteSize = lastBlockSize - } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) - written += n - if err != nil && err != io.EOF { - return nil, err - } - blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) - sliceMd5H.Reset() - } - if tmpF != nil { - if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) - } - _, err = tmpF.Seek(0, io.SeekStart) - if err != nil { - return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") - } - } - contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) - sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) - blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 尝试读取已保存进度 + // step.1 流式计算MD5哈希值（使用 RangeRead，不会消耗流） + contentMd5, sliceMd5, blockList, err := d.calculateHashesStream(ctx, stream, sliceSize, &up) + if err != nil { + return nil, err + } + + blockListStr, _ := utils.Json.MarshalToString(blockList) + + // step.2 尝试读取已保存进度或执行预上传 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 @@ -288,6 +223,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { if precreateResp.UploadURL != "" { return @@ -295,58 +231,20 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } - // step.2 上传分片 + // step.3 流式上传分片 + // 创建 StreamSectionReader 用于上传 + ss, err := streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } + uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() - // 并发上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(UPLOAD_RETRY_COUNT), - retry.Delay(UPLOAD_RETRY_WAIT_TIME), - retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), - retry.DelayType(retry.BackOffDelay), - retry.RetryIf(func(err error) bool { - return !errors.Is(err, ErrUploadIDExpired) - }), - retry.LastErrorOnly(true)) - - totalParts := len(precreateResp.BlockList) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break - } - if partseq < 0 { - continue - } - i, partseq := i, partseq - offset, size := int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - size = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), - } - section := io.NewSectionReader(cache, offset, size) - err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) - if err != nil { - return err - } - precreateResp.BlockList[i] = -1 - progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) - up(progress) - return nil - }) - } - err = threadG.Wait() + // 流式并发上传 + err = d.uploadChunksStream(ctx, ss, stream, precreateResp, path, sliceSize, count, up) if err == nil { break uploadLoop } @@ -372,13 +270,19 @@ uploadLoop: precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + // 尝试重新创建 StreamSectionReader（如果流支持重新读取） + ss, err = streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } continue uploadLoop } return nil, err } defer up(100) - // step.3 创建文件 + // step.4 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { @@ -427,68 +331,6 @@ func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize in return &precreateResp, nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { - b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - mw := multipart.NewWriter(b) - _, err := mw.CreateFormFile("file", fileName) - if err != nil { - return err - } - headSize := b.Len() - err = mw.Close() - if err != nil { - return err - } - head := bytes.NewReader(b.Bytes()[:headSize]) - tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) - if err != nil { - return err - } - query := req.URL.Query() - for k, v := range params { - query.Set(k, v) - } - req.URL.RawQuery = query.Encode() - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(b.Len()) + file.Size() - - client := net.NewHttpClient() - if d.UploadSliceTimeout > 0 { - client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) - } else { - client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b.Reset() - _, err = b.ReadFrom(resp.Body) - if err != nil { - return err - } - body := b.Bytes() - respStr := string(body) - log.Debugln(respStr) - lower := strings.ToLower(respStr) - // 合并 uploadid 过期检测逻辑 - if strings.Contains(lower, "uploadid") && - (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { - return ErrUploadIDExpired - } - - errCode := utils.Json.Get(body, "error_code").ToInt() - errNo := utils.Json.Get(body, "errno").ToInt() - if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) - } - return nil -} - func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 3f3bed0229..499fcd8a87 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -31,8 +31,8 @@ type Addition struct { const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) - DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 - UPLOAD_RETRY_COUNT = 3 + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 180 // 上传分片请求默认超时时间（增加到3分钟以应对慢速网络） + UPLOAD_RETRY_COUNT = 5 // 增加重试次数以提高成功率 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go new file mode 100644 index 0000000000..5283ffe325 --- /dev/null +++ b/drivers/baidu_netdisk/upload.go @@ -0,0 +1,311 @@ +package baidu_netdisk + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" +) + +// calculateHashesStream 流式计算文件的MD5哈希值 +// 返回：文件MD5、前256KB的MD5、每个分片的MD5列表 +// 注意：此函数使用 RangeRead 读取数据，不会消耗流 +func (d *BaiduNetdisk) calculateHashesStream( + ctx context.Context, + stream model.FileStreamer, + sliceSize int64, + up *driver.UpdateProgress, +) (contentMd5 string, sliceMd5 string, blockList []string, err error) { + streamSize := stream.GetSize() + count := 1 + if streamSize > sliceSize { + count = int((streamSize + sliceSize - 1) / sliceSize) + } + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 前256KB的MD5 + const SliceSize int64 = 256 * utils.KB + blockList = make([]string, 0, count) + fileMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceWritten := int64(0) + + // 使用固定大小的缓冲区进行流式哈希计算 + // 这样可以利用 readFullWithRangeRead 的链接刷新逻辑 + const chunkSize = 10 * 1024 * 1024 // 10MB per chunk + buf := make([]byte, chunkSize) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return "", "", nil, ctx.Err() + } + + offset := int64(i) * sliceSize + length := sliceSize + if i == count-1 { + length = lastBlockSize + } + + // 计算分片MD5 + sliceMd5Calc := md5.New() + + // 分块读取并计算哈希 + var sliceOffset int64 = 0 + for sliceOffset < length { + readSize := chunkSize + if length-sliceOffset < int64(chunkSize) { + readSize = int(length - sliceOffset) + } + + // 使用 readFullWithRangeRead 读取数据，自动处理链接刷新 + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset+sliceOffset) + if err != nil { + return "", "", nil, err + } + + // 同时写入多个哈希计算器 + fileMd5H.Write(buf[:n]) + sliceMd5Calc.Write(buf[:n]) + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + if int64(n) > remaining { + sliceMd5H2.Write(buf[:remaining]) + sliceWritten += remaining + } else { + sliceMd5H2.Write(buf[:n]) + sliceWritten += int64(n) + } + } + + sliceOffset += int64(n) + } + + blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) + + // 更新进度（哈希计算占总进度的一小部分） + if up != nil { + progress := float64(i+1) * 10 / float64(count) + (*up)(progress) + } + } + + return hex.EncodeToString(fileMd5H.Sum(nil)), + hex.EncodeToString(sliceMd5H2.Sum(nil)), + blockList, nil +} + +// uploadChunksStream 流式上传所有分片 +func (d *BaiduNetdisk) uploadChunksStream( + ctx context.Context, + ss streamPkg.StreamSectionReader, + stream model.FileStreamer, + precreateResp *PrecreateResp, + path string, + sliceSize int64, + count int, + up driver.UpdateProgress, +) error { + streamSize := stream.GetSize() + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 使用 OrderedGroup 保证 Before 阶段有序 + thread := min(d.uploadThread, len(precreateResp.BlockList)) + threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.OnRetry(func(n uint, err error) { + // 重试前检测是否需要刷新上传 URL + if errors.Is(err, ErrUploadURLExpired) { + log.Infof("[baidu_netdisk] refreshing upload URL due to error: %v", err) + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + + i, partseq := i, partseq + offset := int64(partseq) * sliceSize + size := sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + var reader io.ReadSeeker + + threadG.GoWithLifecycle(errgroup.Lifecycle{ + Before: func(ctx context.Context) error { + var err error + reader, err = ss.GetSectionReader(offset, size) + return err + }, + Do: func(ctx context.Context) error { + reader.Seek(0, io.SeekStart) + err := d.uploadSliceStream(ctx, precreateResp.UploadURL, path, + precreateResp.Uploadid, partseq, stream.GetName(), reader, size) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度从10%开始（前10%是哈希计算） + progress := 10 + float64(threadG.Success()+1)*90/float64(totalParts+1) + up(progress) + return nil + }, + After: func(err error) { + ss.FreeSectionReader(reader) + }, + }) + } + + return threadG.Wait() +} + +// uploadSliceStream 上传单个分片（接受io.ReadSeeker） +func (d *BaiduNetdisk) uploadSliceStream( + ctx context.Context, + uploadUrl string, + path string, + uploadid string, + partseq int, + fileName string, + reader io.ReadSeeker, + size int64, +) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": uploadid, + "partseq": strconv.Itoa(partseq), + } + + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() + if err != nil { + return err + } + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + size + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + // 检测超时或网络错误，标记需要刷新上传 URL + if isUploadURLError(err) { + log.Warnf("[baidu_netdisk] upload slice failed with network error: %v, will refresh upload URL", err) + return errors.Join(err, ErrUploadURLExpired) + } + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + if errCode != 0 || errNo != 0 { + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) + } + return nil +} + +// isUploadURLError 判断是否为需要刷新上传 URL 的错误 +// 包括：超时、连接被拒绝、连接重置、DNS 解析失败等网络错误 +func isUploadURLError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + // 超时错误 + if strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "deadline exceeded") { + return true + } + // 连接错误 + if strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "connection reset") || + strings.Contains(errStr, "no such host") || + strings.Contains(errStr, "network is unreachable") { + return true + } + // EOF 错误（连接被服务器关闭） + if strings.Contains(errStr, "eof") || + strings.Contains(errStr, "broken pipe") { + return true + } + return false +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 0e27fb3053..a66204fa5f 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -207,7 +207,24 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) - res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) + + // Retry HEAD request with longer timeout to avoid client-side errors + // Create a client with longer timeout (base.NoRedirectClient doesn't have timeout set) + client := base.NoRedirectClient.SetTimeout(60 * time.Second) + var res *resty.Response + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + res, err = client.R(). + SetHeader("User-Agent", "pan.baidu.com"). + Head(u) + if err == nil { + break + } + if i < maxRetries-1 { + log.Warnf("HEAD request failed (attempt %d/%d): %v, retrying...", i+1, maxRetries, err) + time.Sleep(time.Duration(i+1) * 2 * time.Second) // Exponential backoff: 2s, 4s, 6s, 8s + } + } if err != nil { return nil, err } @@ -215,11 +232,13 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li u = res.Header().Get("location") //} + exp := time.Hour return &model.Link{ URL: u, Header: http.Header{ "User-Agent": []string{"pan.baidu.com"}, }, + Expiration: &exp, }, nil } diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index 5d218ace9b..bee9ebb266 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -72,7 +72,7 @@ func (c *File) Thumb() string { } func (c *File) GetHash() utils.HashInfo { - return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) + return utils.HashInfo{} } /*相册部分*/ diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index f47eb1748b..a7cee84b84 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "time" "unicode" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -411,12 +412,14 @@ func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkAr // } // location := resp.Header().Get("Location") + exp := time.Hour link := &model.Link{ URL: downloadUrl.Dlink, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, }, + Expiration: &exp, } return link, nil } diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index dfd25d1957..54345ff125 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -55,13 +55,13 @@ func (d *ChaoXing) refreshCookie() error { func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } }) return nil diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index ca171c4d50..dcbded0d8a 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -105,44 +105,82 @@ func (ios *int_str) UnmarshalJSON(data []byte) error { return nil } +// 时间戳同样存在字符串/数字两种形式，且毫秒级时间戳会超出 int32 范围，需用 int64 解析 +// 超星 API 返回的 uploadDate 有三种格式： +// 1. 数字时间戳：1780191356415 +// 2. 字符串时间戳："1780191356415" +// 3. 日期字符串："2024-11-06 07:49" +type int64_str int64 + +func (ios *int64_str) UnmarshalJSON(data []byte) error { + // 去除引号 + str := string(bytes.Trim(data, "\"")) + + // 尝试直接解析为数字（处理格式 1 和 2） + intValue, err := strconv.ParseInt(str, 10, 64) + if err == nil { + *ios = int64_str(intValue) + return nil + } + + // 尝试解析日期字符串（处理格式 3） + // 支持格式："2024-11-06 07:49" 或 "2024-11-06 07:49:30" + layouts := []string{ + "2006-01-02 15:04", + "2006-01-02 15:04:05", + } + + for _, layout := range layouts { + if t, err := time.Parse(layout, str); err == nil { + // 转换为毫秒时间戳 + *ios = int64_str(t.UnixMilli()) + return nil + } + } + + // 所有格式都失败，返回 0（避免因时间格式问题导致整个解析失败） + *ios = 0 + return nil +} + type File struct { Cataid int `json:"cataid"` Cfid int `json:"cfid"` Content struct { - Cfid int `json:"cfid"` - Pid int `json:"pid"` - FolderName string `json:"folderName"` - ShareType int `json:"shareType"` - Preview string `json:"preview"` - Filetype string `json:"filetype"` - PreviewURL string `json:"previewUrl"` - IsImg bool `json:"isImg"` - ParentPath string `json:"parentPath"` - Icon string `json:"icon"` - Suffix string `json:"suffix"` - Duration int `json:"duration"` - Pantype string `json:"pantype"` - Puid int_str `json:"puid"` - Filepath string `json:"filepath"` - Crc string `json:"crc"` - Isfile bool `json:"isfile"` - Residstr string `json:"residstr"` - ObjectID string `json:"objectId"` - Extinfo string `json:"extinfo"` - Thumbnail string `json:"thumbnail"` - Creator int `json:"creator"` - ResTypeValue int `json:"resTypeValue"` - UploadDateFormat string `json:"uploadDateFormat"` - DisableOpt bool `json:"disableOpt"` - DownPath string `json:"downPath"` - Sort int `json:"sort"` - Topsort int `json:"topsort"` - Restype string `json:"restype"` - Size int_str `json:"size"` - UploadDate int64 `json:"uploadDate"` - FileSize string `json:"fileSize"` - Name string `json:"name"` - FileID string `json:"fileId"` + Cfid int `json:"cfid"` + Pid int `json:"pid"` + FolderName string `json:"folderName"` + ShareType int `json:"shareType"` + Preview string `json:"preview"` + Filetype string `json:"filetype"` + PreviewURL string `json:"previewUrl"` + IsImg bool `json:"isImg"` + ParentPath string `json:"parentPath"` + Icon string `json:"icon"` + Suffix string `json:"suffix"` + Duration int `json:"duration"` + Pantype string `json:"pantype"` + Puid int_str `json:"puid"` + Filepath string `json:"filepath"` + Crc string `json:"crc"` + Isfile bool `json:"isfile"` + Residstr string `json:"residstr"` + ObjectID string `json:"objectId"` + Extinfo string `json:"extinfo"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + ResTypeValue int `json:"resTypeValue"` + UploadDateFormat string `json:"uploadDateFormat"` + DisableOpt bool `json:"disableOpt"` + DownPath string `json:"downPath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + Restype string `json:"restype"` + Size int_str `json:"size"` + UploadDate int64_str `json:"uploadDate"` + FileSize string `json:"fileSize"` + Name string `json:"name"` + FileID string `json:"fileId"` } `json:"content"` CreatorID int `json:"creatorId"` DesID string `json:"des_id"` @@ -265,7 +303,7 @@ func fileToObj(f File) *model.Object { IsFolder: true, } } - paserTime := time.UnixMilli(f.Content.UploadDate) + paserTime := time.UnixMilli(int64(f.Content.UploadDate)) return &model.Object{ ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), Name: f.Content.Name, diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go index 2a90cf16cb..f84828f0e1 100644 --- a/drivers/doubao_new/driver.go +++ b/drivers/doubao_new/driver.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "net/url" + "os" "sort" "strings" "sync" @@ -300,7 +301,10 @@ func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileSt if err != nil { return nil, err } - defer tmpFile.Close() + defer func() { + _ = tmpFile.Close() + _ = os.Remove(tmpFile.Name()) + }() blockSize := uploadPrep.BlockSize totalSize := file.GetSize() diff --git a/drivers/emby/driver.go b/drivers/emby/driver.go new file mode 100644 index 0000000000..b2406aa6e7 --- /dev/null +++ b/drivers/emby/driver.go @@ -0,0 +1,261 @@ +package emby + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Emby struct { + model.Storage + Addition + + client *http.Client + token string + userID string +} + +func (d *Emby) Config() driver.Config { + return config +} + +func (d *Emby) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Emby) Init(ctx context.Context) error { + d.URL = strings.TrimRight(strings.TrimSpace(d.URL), "/") + if d.URL == "" { + return fmt.Errorf("url is required") + } + + if strings.TrimSpace(d.RootFolderID) == "" { + d.RootFolderID = "1" + } + + d.client = base.HttpClient + d.token = strings.TrimSpace(d.ApiKey) + d.userID = strings.TrimSpace(d.UserID) + + if d.token != "" { + if d.userID == "" { + return fmt.Errorf("user_id is required when api_key is set") + } + op.MustSaveDriverStorage(d) + return nil + } + + if strings.TrimSpace(d.Username) == "" || strings.TrimSpace(d.Password) == "" { + return fmt.Errorf("please provide api_key+user_id or username+password") + } + + if err := d.login(ctx); err != nil { + return err + } + + d.ApiKey = d.token + d.UserID = d.userID + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Emby) Drop(ctx context.Context) error { + return nil +} + +func (d *Emby) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentID := strings.TrimSpace(d.RootFolderID) + if dir != nil && strings.TrimSpace(dir.GetID()) != "" { + parentID = strings.TrimSpace(dir.GetID()) + } + + items, err := d.getItems(ctx, parentID) + if err != nil { + return nil, err + } + + parentPath := "/" + if dir != nil && strings.TrimSpace(dir.GetPath()) != "" { + parentPath = dir.GetPath() + } + + objs := make([]model.Obj, 0, len(items.Items)) + for _, it := range items.Items { + modified := time.Now() + if it.DateCreated != "" { + if t, parseErr := time.Parse(time.RFC3339Nano, it.DateCreated); parseErr == nil { + modified = t + } + } + + name := strings.TrimSpace(it.Name) + id := strings.TrimSpace(it.ID) + displayName := name + if name != "" && id != "" { + if it.IsFolder { + displayName = fmt.Sprintf("%s (ID%s)", name, id) + } else { + ext := path.Ext(strings.TrimSpace(it.Path)) + if ext == "" { + ext = path.Ext(name) + } + + base := strings.TrimSpace(strings.TrimSuffix(name, ext)) + episodeCode := "" + if m := episodeCodeRegexp.FindString(base); m != "" { + episodeCode = strings.ToUpper(m) + } else if it.ParentIndex > 0 && it.IndexNumber > 0 { + episodeCode = fmt.Sprintf("S%02dE%02d", it.ParentIndex, it.IndexNumber) + } + + title := strings.TrimSpace(base) + if episodeCode != "" { + title = strings.TrimSpace(episodeCodeRegexp.ReplaceAllString(title, "")) + title = strings.TrimSpace(strings.Trim(title, "-_:[]() ")) + } + + series := strings.TrimSpace(it.SeriesName) + if series == "" && episodeCode != "" { + if idx := strings.Index(title, " - "); idx > 0 { + series = strings.TrimSpace(title[:idx]) + title = strings.TrimSpace(title[idx+3:]) + } + } + + core := title + if series != "" { + if title == "" || strings.EqualFold(series, title) { + core = series + } else { + core = series + " " + title + } + } + if core == "" { + core = base + } + + if episodeCode != "" { + core = fmt.Sprintf("%s - [%s]", core, episodeCode) + } + if ext == "" { + displayName = fmt.Sprintf("%s (ID%s)", core, id) + } else { + displayName = fmt.Sprintf("%s (ID%s)%s", core, id, ext) + } + } + } + + obj := &model.Object{ + ID: id, + Name: displayName, + Path: path.Join(parentPath, displayName), + Size: it.Size, + Modified: modified, + IsFolder: it.IsFolder, + } + if it.IsFolder { + obj.Size = 0 + } + objs = append(objs, obj) + } + return objs, nil +} + +func (d *Emby) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := strings.TrimSpace(file.GetID()) + if fileID == "" { + return nil, fmt.Errorf("invalid file id") + } + + u, err := url.Parse(d.URL) + if err != nil { + return nil, err + } + linkMethod := strings.ToLower(strings.TrimSpace(d.LinkMethod)) + useDownload := linkMethod == "download" + + mediaSourceID := "" + mediaContainer := "" + if !useDownload { + detailURL, parseErr := url.Parse(d.URL + "/Users/" + d.userID + "/Items/" + fileID) + if parseErr == nil { + q := detailURL.Query() + q.Set("Fields", "MediaSources") + q.Set("api_key", d.token) + detailURL.RawQuery = q.Encode() + + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, detailURL.String(), nil) + if reqErr == nil { + resp, doErr := d.client.Do(req) + if doErr == nil { + func() { + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return + } + var detail itemDetailResp + if decodeErr := json.NewDecoder(resp.Body).Decode(&detail); decodeErr != nil || len(detail.MediaSources) == 0 { + return + } + for i := range detail.MediaSources { + if strings.TrimSpace(detail.MediaSources[i].ID) != "" && detail.MediaSources[i].SupportsDirectStream { + mediaSourceID = strings.TrimSpace(detail.MediaSources[i].ID) + mediaContainer = strings.TrimSpace(detail.MediaSources[i].Container) + return + } + } + for i := range detail.MediaSources { + if strings.TrimSpace(detail.MediaSources[i].ID) != "" { + mediaSourceID = strings.TrimSpace(detail.MediaSources[i].ID) + mediaContainer = strings.TrimSpace(detail.MediaSources[i].Container) + return + } + } + }() + } + } + } + } + + if useDownload { + u.Path = path.Join(u.Path, "/Items", fileID, "Download") + } else { + if mediaContainer != "" { + u.Path = path.Join(u.Path, "/Videos", fileID, "stream."+mediaContainer) + } else { + u.Path = path.Join(u.Path, "/Videos", fileID, "stream") + } + } + q := u.Query() + q.Set("api_key", d.token) + if mediaSourceID != "" { + q.Set("MediaSourceId", mediaSourceID) + } + if !useDownload { + q.Set("Static", "true") + } + u.RawQuery = q.Encode() + + return &model.Link{ + URL: u.String(), + Header: http.Header{ + "User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +var _ driver.Driver = (*Emby)(nil) diff --git a/drivers/emby/meta.go b/drivers/emby/meta.go new file mode 100644 index 0000000000..632c98b7a9 --- /dev/null +++ b/drivers/emby/meta.go @@ -0,0 +1,30 @@ +package emby + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootID + URL string `json:"url" required:"true"` + ApiKey string `json:"api_key"` + UserID string `json:"user_id"` + Username string `json:"username"` + Password string `json:"password"` + LinkMethod string `json:"link_method" type:"select" options:"stream,download" default:"stream"` +} + +var config = driver.Config{ + Name: "Emby", + LocalSort: true, + NoUpload: true, + DefaultRoot: "1", + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Emby{} + }) +} diff --git a/drivers/emby/types.go b/drivers/emby/types.go new file mode 100644 index 0000000000..5f05158296 --- /dev/null +++ b/drivers/emby/types.go @@ -0,0 +1,41 @@ +package emby + +type authReq struct { + Username string `json:"Username"` + Pw string `json:"Pw"` +} + +type authResp struct { + AccessToken string `json:"AccessToken"` + User struct { + ID string `json:"Id"` + } `json:"User"` +} + +type listResp struct { + Items []embyItem `json:"Items"` + TotalRecordCount int `json:"TotalRecordCount"` +} + +type embyItem struct { + Name string `json:"Name"` + ID string `json:"Id"` + Type string `json:"Type"` + Path string `json:"Path"` + SeriesName string `json:"SeriesName"` + IndexNumber int `json:"IndexNumber"` + ParentIndex int `json:"ParentIndexNumber"` + IsFolder bool `json:"IsFolder"` + Size int64 `json:"Size"` + DateCreated string `json:"DateCreated"` +} + +type itemDetailResp struct { + MediaSources []embyMediaSource `json:"MediaSources"` +} + +type embyMediaSource struct { + ID string `json:"Id"` + Container string `json:"Container"` + SupportsDirectStream bool `json:"SupportsDirectStream"` +} diff --git a/drivers/emby/util.go b/drivers/emby/util.go new file mode 100644 index 0000000000..788fb4be43 --- /dev/null +++ b/drivers/emby/util.go @@ -0,0 +1,91 @@ +package emby + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" +) + +var episodeCodeRegexp = regexp.MustCompile(`(?i)\bS\d{1,2}E\d{1,2}\b`) + +func (d *Emby) login(ctx context.Context) error { + payload, err := json.Marshal(authReq{ + Username: d.Username, + Pw: d.Password, + }) + if err != nil { + return err + } + + endpoint := d.URL + "/Users/AuthenticateByName" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Emby-Authorization", `MediaBrowser Client="OpenList", Device="OpenList", DeviceId="openlist-emby", Version="1.0.0"`) + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("emby auth failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var data authResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return err + } + if strings.TrimSpace(data.AccessToken) == "" || strings.TrimSpace(data.User.ID) == "" { + return fmt.Errorf("emby auth response missing access token or user id") + } + + d.token = data.AccessToken + d.userID = data.User.ID + return nil +} + +func (d *Emby) getItems(ctx context.Context, parentID string) (*listResp, error) { + u, err := url.Parse(d.URL + "/Users/" + d.userID + "/Items") + if err != nil { + return nil, err + } + q := u.Query() + q.Set("ParentId", parentID) + q.Set("Recursive", "false") + q.Set("Fields", "Path,Size,DateCreated,SeriesName,IndexNumber,ParentIndexNumber") + q.Set("api_key", d.token) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("emby list failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var data listResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, err + } + return &data, nil +} diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 4845337861..0d3e803e03 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -5,6 +5,7 @@ import ( "errors" "io" stdpath "path" + "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -51,13 +52,35 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if err := d.login(); err != nil { return nil, err } - entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + + path := encode(dir.GetPath(), d.Encoding) + + var entries []*ftp.Entry + var err error + + if d.CwdList { + origDir, cwdErr := d.conn.CurrentDir() + if cwdErr != nil { + return nil, cwdErr + } + if cwdErr = d.conn.ChangeDir(path); cwdErr != nil { + return nil, cwdErr + } + entries, err = d.conn.List("") + if restoreErr := d.conn.ChangeDir(origDir); restoreErr != nil { + d.conn = nil + } + } else { + entries, err = d.conn.List(path) + } + if err != nil { return nil, err } + res := make([]model.Obj, 0) for _, entry := range entries { - if entry.Name == "." || entry.Name == ".." { + if entry.Name == "." || entry.Name == ".." || strings.Contains(entry.Name, "/") { continue } name := decode(entry.Name, d.Encoding) diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 0ec0e735eb..6dc0d2cb61 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -27,6 +27,7 @@ type Addition struct { Encoding string `json:"encoding" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` + CwdList bool `json:"cwd_list" type:"bool" default:"false" help:"enter directory before listing"` driver.RootPath } diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index 8a8025c5aa..92d94b4dcf 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" ) type GithubReleases struct { @@ -45,12 +46,25 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis point := &d.points[i] if !d.Addition.ShowAllVersion { // latest - point.RequestRelease(d.GetRequest, args.Refresh) + err := point.RequestRelease(d.GetRequest, args.Refresh) + if err != nil { + log.Warnf("failed to request release for %s: %v", point.Repo, err) + } if point.Point == path { // 与仓库路径相同 + if point.Release == nil { + if err != nil { + return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get release for %s: unknown error", point.Repo) + } files = append(files, point.GetLatestRelease()...) if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) + if err != nil { + return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + } + files = append(files, otherFiles...) } if d.Addition.ShowSourceCode { files = append(files, point.GetSourceCode()...) @@ -60,6 +74,9 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if nextDir == "" { continue } + if err != nil { + return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) + } hasSameDir := false for index := range files { @@ -70,30 +87,51 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } } if !hasSameDir { + var updateAt, createAt string + if point.Release != nil { + updateAt = point.Release.PublishedAt + createAt = point.Release.CreatedAt + } files = append(files, File{ Path: stdpath.Join(path, nextDir), FileName: nextDir, Size: point.GetLatestSize(), - UpdateAt: point.Release.PublishedAt, - CreateAt: point.Release.CreatedAt, + UpdateAt: updateAt, + CreateAt: createAt, Type: "dir", Url: "", }) } } } else { // all version - point.RequestReleases(d.GetRequest, args.Refresh) + err := point.RequestReleases(d.GetRequest, args.Refresh) + if err != nil { + log.Warnf("failed to request releases for %s: %v", point.Repo, err) + } if point.Point == path { // 与仓库路径相同 + if point.Releases == nil { + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) + } files = append(files, point.GetAllVersion()...) if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) + if err != nil { + return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + } + files = append(files, otherFiles...) } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } hasSameDir := false for index := range files { @@ -104,12 +142,17 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } } if !hasSameDir { + var updateAt, createAt string + if point.Releases != nil && len(*point.Releases) > 0 { + updateAt = (*point.Releases)[0].PublishedAt + createAt = (*point.Releases)[0].CreatedAt + } files = append(files, File{ FileName: nextDir, Path: stdpath.Join(path, nextDir), Size: point.GetAllVersionSize(), - UpdateAt: (*point.Releases)[0].PublishedAt, - CreateAt: (*point.Releases)[0].CreatedAt, + UpdateAt: updateAt, + CreateAt: createAt, Type: "dir", Url: "", }) @@ -119,6 +162,12 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if tagName == "" { continue } + if point.Releases == nil { + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) + } files = append(files, point.GetReleaseByTagName(tagName)...) diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 663aec77e2..abc024cc8d 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -19,33 +19,54 @@ type MountPoint struct { } // 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) error { if m.Repo == "" { - return + return nil } if m.Release == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") - m.Release = new(Release) - json.Unmarshal(resp.Body(), m.Release) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") + if err != nil { + m.Release = nil + return err + } + release := new(Release) + if err := json.Unmarshal(resp.Body(), release); err != nil { + m.Release = nil + return err + } + m.Release = release } + return nil } // 请求所有版本 -func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) error { if m.Repo == "" { - return + return nil } if m.Releases == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases") - m.Releases = new([]Release) - json.Unmarshal(resp.Body(), m.Releases) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases") + if err != nil { + m.Releases = nil + return err + } + releases := new([]Release) + if err := json.Unmarshal(resp.Body(), releases); err != nil { + m.Releases = nil + return err + } + m.Releases = releases } + return nil } // 获取最新版本 func (m *MountPoint) GetLatestRelease() []File { + if m.Release == nil { + return nil + } files := make([]File, 0, len(m.Release.Assets)) for _, asset := range m.Release.Assets { files = append(files, File{ @@ -63,6 +84,9 @@ func (m *MountPoint) GetLatestRelease() []File { // 获取最新版本大小 func (m *MountPoint) GetLatestSize() int64 { + if m.Release == nil { + return 0 + } size := int64(0) for _, asset := range m.Release.Assets { size += asset.Size @@ -72,6 +96,9 @@ func (m *MountPoint) GetLatestSize() int64 { // 获取所有版本 func (m *MountPoint) GetAllVersion() []File { + if m.Releases == nil { + return nil + } files := make([]File, 0) for _, release := range *m.Releases { file := File{ @@ -93,6 +120,9 @@ func (m *MountPoint) GetAllVersion() []File { // 根据版本号获取版本 func (m *MountPoint) GetReleaseByTagName(tagName string) []File { + if m.Releases == nil { + return nil + } for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) @@ -145,6 +175,9 @@ func (m *MountPoint) GetAllVersionSize() int64 { } func (m *MountPoint) GetSourceCode() []File { + if m.Release == nil { + return nil + } files := make([]File, 0) // 无法获取文件大小，此处设为 1 @@ -171,6 +204,9 @@ func (m *MountPoint) GetSourceCode() []File { } func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { + if m.Releases == nil { + return nil + } for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) @@ -198,11 +234,19 @@ func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { return nil } -func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { +func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) ([]File, error) { if m.OtherFile == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") - m.OtherFile = new([]FileInfo) - json.Unmarshal(resp.Body(), m.OtherFile) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/contents") + if err != nil { + m.OtherFile = nil + return nil, err + } + otherFile := new([]FileInfo) + if err := json.Unmarshal(resp.Body(), otherFile); err != nil { + m.OtherFile = nil + return nil, err + } + m.OtherFile = otherFile } files := make([]File, 0) @@ -220,7 +264,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), }) } } - return files + return files, nil } type File struct { diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index 91a4b3a231..a6401e6cd7 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -7,7 +7,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) // 发送 GET 请求 @@ -23,7 +22,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { return nil, err } if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) + return nil, fmt.Errorf("github api error: status %d", res.StatusCode()) } return res, nil } diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 94ef854f2f..0e1c04f46c 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -3,17 +3,29 @@ package google_drive import ( "context" "fmt" + "io" "net/http" "strconv" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) +// mkdirLocks prevents race conditions when creating folders with the same name +// Google Drive allows duplicate folder names, so we need application-level locking +var mkdirLocks sync.Map // map[string]*sync.Mutex - key is parentID + "/" + dirName + type GoogleDrive struct { model.Storage Addition @@ -67,15 +79,79 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // Use per-folder lock to prevent concurrent creation of same folder + // This is critical because Google Drive allows duplicate folder names + lockKey := parentDir.GetID() + "/" + dirName + lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + lock.Lock() + defer func() { + lock.Unlock() + mkdirLocks.Delete(lockKey) + }() + + // Check if folder already exists with retry to handle API eventual consistency + escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") + query := map[string]string{ + "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), + "fields": "files(id)", + } + + var existingFiles Files + err := retry.Do(func() error { + var checkErr error + _, checkErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + return checkErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(200*time.Millisecond), + ) + + // If query succeeded and folder exists, return success (idempotent) + if err == nil && len(existingFiles.Files) > 0 { + log.Debugf("[google_drive] Folder '%s' already exists in parent %s, skipping creation", dirName, parentDir.GetID()) + return nil + } + // If query failed, return error to prevent duplicate creation + if err != nil { + return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) + } + + // Create new folder (only when confirmed folder doesn't exist) data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { - req.SetBody(data) - }, nil) - return err + + var createErr error + err = retry.Do(func() error { + _, createErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return createErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) + + if err != nil { + return err + } + + // Wait for API eventual consistency before releasing lock + // This helps prevent race conditions where a concurrent request + // checks for folder existence before the newly created folder is visible + // 500ms is needed because Google Drive API has significant sync delay + time.Sleep(500 * time.Millisecond) + + return nil } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -111,8 +187,50 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj := stream.GetExist() +const maxPutAuthRetries = 2 + +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + return d.putWithRetry(ctx, dstDir, file, up, 0) +} + +func (d *GoogleDrive) putWithRetry(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, authRetries int) error { + // 1. 准备MD5（用于完整性校验） + md5Hash := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验（Google Drive会自动校验） + } + } else { + // 不可重复读取的流（如 HTTP body） + if len(md5Hash) != utils.MD5.Width { + // 缓存整个文件并计算 MD5 + var err error + _, md5Hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验 + } else if file.GetFile() == nil { + // 有 MD5 但没有缓存，需要缓存以支持后续 RangeRead + _, err := file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + } + + // 2. 初始化可恢复上传会话 + obj := file.GetExist() var ( e Error url string @@ -125,7 +243,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi data = base.Json{} } else { data = base.Json{ - "name": stream.GetName(), + "name": file.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" @@ -133,8 +251,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, - "X-Upload-Content-Type": stream.GetMimetype(), - "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + "X-Upload-Content-Type": file.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(file.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { @@ -146,25 +264,45 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi return err } if e.Error.Code != 0 { - if e.Error.Code == 401 { + if e.Error.Code == 401 && authRetries < maxPutAuthRetries { err = d.refreshToken() if err != nil { return err } - return d.Put(ctx, dstDir, stream, up) + return d.putWithRetry(ctx, dstDir, file, up, authRetries+1) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } + + // 3. 上传文件内容 putUrl := res.Header().Get("location") - if stream.GetSize() < d.ChunkSize*1024*1024 { - _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, stream)) - }, nil) + if file.GetSize() < d.ChunkSize*1024*1024 { + // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） + err = retry.Do(func() error { + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) + }, nil) + return err + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + return err } else { - err = d.chunkUpload(ctx, stream, putUrl, up) + // 大文件分片上传 + return d.chunkUpload(ctx, file, putUrl, up) } - return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { diff --git a/drivers/google_drive/driver_test.go b/drivers/google_drive/driver_test.go new file mode 100644 index 0000000000..b98bdcf146 --- /dev/null +++ b/drivers/google_drive/driver_test.go @@ -0,0 +1,71 @@ +package google_drive + +import ( + "sync" + "testing" +) + +func TestMaxPutAuthRetriesIsBounded(t *testing.T) { + if maxPutAuthRetries < 1 { + t.Fatalf("maxPutAuthRetries=%d, must be >= 1", maxPutAuthRetries) + } + if maxPutAuthRetries > 5 { + t.Fatalf("maxPutAuthRetries=%d, must be <= 5 to prevent excessive retries", maxPutAuthRetries) + } +} + +func TestMkdirLocksCleanedUpAfterUse(t *testing.T) { + // Reset state + mkdirLocks = sync.Map{} + + key1 := "parent-1/folder-a" + key2 := "parent-2/folder-b" + + // Simulate two MakeDir calls storing locks + mkdirLocks.LoadOrStore(key1, &sync.Mutex{}) + mkdirLocks.LoadOrStore(key2, &sync.Mutex{}) + + // Both should exist + if _, ok := mkdirLocks.Load(key1); !ok { + t.Fatal("key1 should exist") + } + if _, ok := mkdirLocks.Load(key2); !ok { + t.Fatal("key2 should exist") + } + + // After MakeDir completes, locks should be cleaned up + mkdirLocks.Delete(key1) + mkdirLocks.Delete(key2) + + if _, ok := mkdirLocks.Load(key1); ok { + t.Fatal("key1 should be deleted after cleanup") + } + if _, ok := mkdirLocks.Load(key2); ok { + t.Fatal("key2 should be deleted after cleanup") + } +} + +func TestMkdirLocksNoCrossContamination(t *testing.T) { + mkdirLocks = sync.Map{} + + key := "parent/shared-folder" + lockVal, _ := mkdirLocks.LoadOrStore(key, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + + // Simulate concurrent access: lock should be shared for same key + lockVal2, loaded := mkdirLocks.LoadOrStore(key, &sync.Mutex{}) + if !loaded { + t.Fatal("second LoadOrStore should return existing entry") + } + lock2 := lockVal2.(*sync.Mutex) + if lock != lock2 { + t.Fatal("same key should return same mutex instance") + } + + // Different key should get different lock + lockVal3, _ := mkdirLocks.LoadOrStore("other-parent/other-folder", &sync.Mutex{}) + lock3 := lockVal3.(*sync.Mutex) + if lock == lock3 { + t.Fatal("different keys should have different mutex instances") + } +} diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa44..53695deee0 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -296,9 +296,60 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { res = append(res, resp.Files...) } + + // Handle duplicate filenames by adding suffixes like (1), (2), etc. + // Google Drive allows multiple files with the same name in one folder, + // but OpenList uses path-based file system which requires unique names + res = handleDuplicateNames(res) + return res, nil } +// handleDuplicateNames adds suffixes to duplicate filenames to make them unique +// For example: file.txt, file (1).txt, file (2).txt +func handleDuplicateNames(files []File) []File { + if len(files) <= 1 { + return files + } + + // Track how many files with each name we've seen + nameCount := make(map[string]int) + + // First pass: count occurrences of each name + for _, file := range files { + nameCount[file.Name]++ + } + + // Second pass: add suffixes to duplicates + nameIndex := make(map[string]int) + for i := range files { + name := files[i].Name + if nameCount[name] > 1 { + index := nameIndex[name] + nameIndex[name]++ + + if index > 0 { + // Add suffix for all except the first occurrence + // Split name into base and extension + ext := "" + base := name + for j := len(name) - 1; j >= 0; j-- { + if name[j] == '.' { + ext = name[j:] + base = name[:j] + break + } + } + + // Add (1), (2), etc. suffix + files[i].Name = fmt.Sprintf("%s (%d)%s", base, index, ext) + } + } + } + + return files +} + // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 3a9b66ab2e..81d05b4929 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error { return err } if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 9a15d428e5..37844500ea 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -91,7 +91,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, if info == "" { info = utils.Json.Get(data, "info").ToString() } - return data, fmt.Errorf(info) + return data, fmt.Errorf("%s", info) } } diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 7189648a7a..ba7c0850b7 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -389,6 +389,9 @@ func (d *Local) Remove(ctx context.Context, obj model.Obj) error { if err != nil { return err } + if !obj.IsDir() { + d.removeThumbCache(obj.GetPath()) + } if obj.IsDir() { if d.directoryMap.Has(obj.GetPath()) { d.directoryMap.DeleteDirNode(obj.GetPath()) diff --git a/drivers/local/remove_test.go b/drivers/local/remove_test.go new file mode 100644 index 0000000000..86ec510819 --- /dev/null +++ b/drivers/local/remove_test.go @@ -0,0 +1,44 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestRemoveDeletesThumbCache(t *testing.T) { + root := t.TempDir() + cacheDir := filepath.Join(root, "thumbs") + if err := os.Mkdir(cacheDir, 0o755); err != nil { + t.Fatal(err) + } + filePath := filepath.Join(root, "photo.jpg") + if err := os.WriteFile(filePath, []byte("image"), 0o666); err != nil { + t.Fatal(err) + } + + driver := &Local{ + Addition: Addition{ + ThumbCacheFolder: cacheDir, + }, + } + thumbPath := driver.thumbCachePath(filePath) + if err := os.WriteFile(thumbPath, []byte("thumb"), 0o666); err != nil { + t.Fatal(err) + } + + err := driver.Remove(context.Background(), &model.Object{ + Path: filePath, + Name: filepath.Base(filePath), + }) + if err != nil { + t.Fatal(err) + } + + if _, err = os.Stat(thumbPath); !os.IsNotExist(err) { + t.Fatalf("expected thumb cache to be removed, got %v", err) + } +} diff --git a/drivers/local/util.go b/drivers/local/util.go index be86d9c9b0..f9f4f4c45f 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -24,6 +24,8 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) +const thumbPrefix = "openlist_thumb_" + func isSymlinkDir(f fs.FileInfo, path string) bool { if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular == os.ModeIrregular) { // os.ModeIrregular is Junction bit in Windows @@ -110,16 +112,29 @@ func readDir(dirname string) ([]fs.FileInfo, error) { return list, nil } +func (d *Local) thumbCachePath(fullPath string) string { + if d.ThumbCacheFolder == "" { + return "" + } + return filepath.Join(d.ThumbCacheFolder, thumbPrefix+utils.GetMD5EncodeStr(fullPath)+".png") +} + +func (d *Local) removeThumbCache(fullPath string) { + thumbPath := d.thumbCachePath(fullPath) + if thumbPath == "" { + return + } + _ = os.Remove(thumbPath) +} + func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() - thumbPrefix := "openlist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { return nil, &fullPath, nil } - thumbPath := filepath.Join(d.ThumbCacheFolder, thumbName) + thumbPath := d.thumbCachePath(fullPath) if utils.Exists(thumbPath) { return nil, &thumbPath, nil } @@ -151,7 +166,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { return nil, nil, err } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) + err = os.WriteFile(d.thumbCachePath(fullPath), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 0bc3e79b53..a975772ac0 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -1,27 +1,41 @@ package onedrive_sharelink import ( + "bytes" "context" + "encoding/json" "fmt" "io" "net/http" + "net/url" stdpath "path" + "regexp" "strings" "sync" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) -const headerTTL = 25 * time.Minute +const ( + headerTTL = 25 * time.Minute + driveTokenTTL = 20 * time.Minute + directLinkTTL = 20 * time.Minute + simpleUploadLimit = 250 * 1024 * 1024 + uploadSessionChunk = 10 * 1024 * 1024 +) type OnedriveSharelink struct { model.Storage @@ -78,10 +92,17 @@ func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model. if err != nil { return nil, err } + folderSizes, err := d.driveChildrenFolderSizes(ctx, dir.GetPath()) + if err != nil { + log.Warnf("onedrive_sharelink: failed to get folder sizes for %s: %+v", dir.GetPath(), err) + } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { obj := fileToObj(src) + if size, ok := folderSizes[obj.GetName()]; ok { + obj.Size = size + } obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) return obj, nil }) @@ -99,6 +120,18 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model return nil, err } + if args.Redirect { + directURL, err := d.resolveDirectDownloadURL(ctx, file, url, header) + if err != nil { + return nil, err + } + expiration := directLinkTTL + return &model.Link{ + URL: directURL, + Expiration: &expiration, + }, nil + } + return &model.Link{ URL: url, Header: header, @@ -109,8 +142,30 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model } func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - // TODO create folder, optional - return errs.NotImplement + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return err + } + apiURL := injectAccessToken(d.drivePathAPIURL(parentDir.GetPath())+"/children", token) + body := map[string]any{ + "name": dirName, + "folder": map[string]any{}, + "@microsoft.graph.conflictBehavior": "fail", + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + return nil + case http.StatusConflict: + return errs.ObjectAlreadyExists + default: + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create folder, status code: %d, body: %s", resp.StatusCode, string(data)) + } } func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -134,8 +189,422 @@ func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error { } func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO upload file, optional - return errs.NotImplement + info, err := d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), stream.GetName()), stream.GetSize()) + if err != nil { + return err + } + if info.ChunkSize == 0 { + return d.uploadContent(ctx, info.UploadURL, stream, up) + } + return d.uploadToSession(ctx, info.UploadURL, stream, up) +} + +func (d *OnedriveSharelink) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if d.DisableDiskUsage { + return nil, errs.NotImplement + } + size, err := d.driveItemSize(ctx, "/") + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: size, + UsedSpace: size, + }, + }, nil +} + +func (d *OnedriveSharelink) driveItemSize(ctx context.Context, path string) (int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return 0, err + } + apiURL := injectAccessToken(d.drivePathAPIURL(path), token) + resp, err := d.doJSON(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("failed to get folder details, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var item struct { + Size int64 `json:"size"` + } + if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { + return 0, err + } + return item.Size, nil +} + +func (d *OnedriveSharelink) driveChildrenFolderSizes(ctx context.Context, path string) (map[string]int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + rawURL := d.drivePathAPIURL(path) + "/children?$select=name,size,folder" + sizes := make(map[string]int64) + for rawURL != "" { + resp, err := d.doJSON(ctx, http.MethodGet, injectAccessToken(rawURL, token), nil) + if err != nil { + return nil, err + } + var data struct { + Value []struct { + Name string `json:"name"` + Size int64 `json:"size"` + Folder any `json:"folder"` + } `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + func() { + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + err = fmt.Errorf("failed to get children details, status code: %d, body: %s", resp.StatusCode, string(body)) + return + } + err = json.NewDecoder(resp.Body).Decode(&data) + }() + if err != nil { + return nil, err + } + for _, item := range data.Value { + if item.Folder != nil { + sizes[item.Name] = item.Size + } + } + rawURL = data.NextLink + } + return sizes, nil +} + +func (d *OnedriveSharelink) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveSharelink) GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + if tool != "HttpDirect" { + return nil, errs.NotImplement + } + return d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), fileName), fileSize) +} + +func (d *OnedriveSharelink) createUploadInfo(ctx context.Context, path string, fileSize int64) (*model.HttpDirectUploadInfo, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + if fileSize >= 0 && fileSize <= simpleUploadLimit { + return &model.HttpDirectUploadInfo{ + UploadURL: injectAccessToken(d.drivePathAPIURL(path)+"/content", token), + Method: http.MethodPut, + }, nil + } + apiURL := injectAccessToken(d.drivePathAPIURL(path)+"/createUploadSession", token) + body := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to create upload session, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var data uploadSessionResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, err + } + if data.UploadURL == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: data.UploadURL, + ChunkSize: uploadSessionChunk, + Method: http.MethodPut, + }, nil +} + +func (d *OnedriveSharelink) uploadContent(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: file, + Size: file.GetSize(), + }, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader) + if err != nil { + return err + } + req.ContentLength = file.GetSize() + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to upload content, status code: %d, body: %s", resp.StatusCode, string(data)) + } + return nil +} + +func (d *OnedriveSharelink) uploadToSession(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + if file.GetSize() <= 0 { + return d.uploadSessionChunk(ctx, uploadURL, file, 0, 0, file.GetSize()) + } + ss, err := streamPkg.NewStreamSectionReader(file, uploadSessionChunk, &up) + if err != nil { + return err + } + var finish int64 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, int64(uploadSessionChunk)) + rd, err := ss.GetSectionReader(finish, byteSize) + if err != nil { + return err + } + err = retry.Do( + func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + return d.uploadSessionChunk(ctx, uploadURL, rd, finish, byteSize, file.GetSize()) + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + ss.FreeSectionReader(rd) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + } + return nil +} + +func (d *OnedriveSharelink) uploadSessionChunk(ctx context.Context, uploadURL string, reader io.Reader, start, size, total int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, driver.NewLimitedUploadStream(ctx, reader)) + if err != nil { + return err + } + req.ContentLength = size + if total > 0 { + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, start+size-1, total)) + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch { + case resp.StatusCode >= 500 && resp.StatusCode <= 504: + return fmt.Errorf("server error: %d", resp.StatusCode) + case resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted: + data, _ := io.ReadAll(resp.Body) + return errors.New(string(data)) + default: + return nil + } +} + +func (d *OnedriveSharelink) drivePathAPIURL(path string) string { + drivePath := stdpath.Join(d.driveRootPath, path) + drivePath = utils.FixAndCleanPath(drivePath) + if drivePath == "/" { + return d.DriveURL + "/root" + } + return fmt.Sprintf("%s/root:%s:", d.DriveURL, utils.EncodePath(drivePath, true)) +} + +func injectAccessToken(rawURL, token string) string { + if token == "" { + return rawURL + } + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + values := u.Query() + if strings.HasPrefix(token, "access_token=") { + values.Set("access_token", strings.TrimPrefix(token, "access_token=")) + } else { + values.Set("access_token", token) + } + u.RawQuery = values.Encode() + return u.String() +} + +func (d *OnedriveSharelink) doJSON(ctx context.Context, method, rawURL string, body any) (*http.Response, error) { + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, reader) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json;odata.metadata=minimal") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return base.HttpClient.Do(req) +} + +func (d *OnedriveSharelink) getValidDriveAccessToken(ctx context.Context) (string, error) { + d.headerMu.RLock() + token := d.DriveAccessToken + expired := time.Since(time.Unix(d.DriveTokenTime, 0)) > driveTokenTTL + d.headerMu.RUnlock() + if token != "" && !expired { + return token, nil + } + if err := d.refreshDriveContext(ctx); err != nil { + d.headerMu.RLock() + token = d.DriveAccessToken + d.headerMu.RUnlock() + if token != "" { + log.Warnf("onedrive_sharelink: use cached drive access token after refresh failure: %+v", err) + return token, nil + } + return "", err + } + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return d.DriveAccessToken, nil +} + +func (d *OnedriveSharelink) refreshDriveContext(ctx context.Context) error { + header, err := d.getValidHeaders(ctx) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.ShareLinkURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + redirectURL := resp.Header.Get("Location") + if redirectURL == "" && resp.Request != nil && resp.Request.URL != nil { + redirectURL = resp.Request.URL.String() + } + if redirectURL == "" { + return fmt.Errorf("share link did not return redirect URL") + } + return d.refreshDriveContextFromRedirect(ctx, redirectURL, header) +} + +func (d *OnedriveSharelink) refreshDriveContextFromRedirect(ctx context.Context, redirectURL string, header http.Header) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("onedrive page request failed, status code: %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + ctxInfo, err := parsePageContext(body) + if err != nil { + return err + } + rootPath, err := driveRootPathFromRedirect(redirectURL, ctxInfo.ListURL) + if err != nil { + return err + } + d.headerMu.Lock() + d.DriveURL = ctxInfo.DriveInfo.DriveURL + d.DriveAccessToken = ctxInfo.DriveInfo.DriveAccessToken + d.DriveTokenTime = time.Now().Unix() + d.driveRootPath = rootPath + d.headerMu.Unlock() + return nil +} + +func driveRootPathFromRedirect(redirectURL, listURL string) (string, error) { + u, err := url.Parse(redirectURL) + if err != nil { + return "", err + } + id := u.Query().Get("id") + if id == "" { + return "/", nil + } + if listURL == "" { + return "/", nil + } + if id == listURL { + return "/", nil + } + prefix := strings.TrimRight(listURL, "/") + "/" + if strings.HasPrefix(id, prefix) { + return utils.FixAndCleanPath(strings.TrimPrefix(id, strings.TrimRight(listURL, "/"))), nil + } + return "/", nil +} + +var pageContextRE = regexp.MustCompile(`(?s)var _spPageContextInfo=(\{.*?\});_spPageContextInfo`) + +func parsePageContext(body []byte) (*pageContextInfo, error) { + match := pageContextRE.FindSubmatch(body) + if len(match) < 2 { + return nil, fmt.Errorf("failed to find _spPageContextInfo") + } + var info pageContextInfo + if err := json.Unmarshal(match[1], &info); err != nil { + return nil, err + } + if info.DriveInfo.DriveURL == "" { + return nil, fmt.Errorf("failed to get drive URL from page context") + } + if info.DriveInfo.DriveAccessToken == "" { + return nil, fmt.Errorf("failed to get drive access token from page context") + } + return &info, nil } //func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { @@ -194,6 +663,96 @@ func cloneHeader(header http.Header) http.Header { return header.Clone() } +func (d *OnedriveSharelink) resolveDirectDownloadURL(ctx context.Context, file model.Obj, rawURL string, header http.Header) (string, error) { + var errs []error + if obj, ok := unwrapObject(file); ok { + if obj.SPItemURL != "" { + directURL, err := d.resolveSPItemDownloadURL(ctx, obj.SPItemURL, header) + if err == nil { + return directURL, nil + } + errs = append(errs, err) + } + if obj.ContentDownloadURL != "" { + return obj.ContentDownloadURL, nil + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", err + } + req.Header = cloneHeader(header) + if req.Header == nil { + req.Header = http.Header{} + } + + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + location := resp.Header.Get("Location") + if location == "" { + errs = append(errs, fmt.Errorf("download.aspx returned no redirect location, status code: %d", resp.StatusCode)) + return "", fmt.Errorf("onedrive_sharelink: direct download URL unavailable: %v", errs) + } + u, err := req.URL.Parse(location) + if err != nil { + return "", err + } + return u.String(), nil +} + +type spItemDownloadResp struct { + ContentDownloadURL string `json:"@content.downloadUrl"` +} + +func unwrapObject(obj model.Obj) (*Object, bool) { + for { + switch o := obj.(type) { + case *Object: + return o, true + case model.ObjUnwrap: + obj = o.Unwrap() + default: + return nil, false + } + } +} + +func (d *OnedriveSharelink) resolveSPItemDownloadURL(ctx context.Context, spItemURL string, header http.Header) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, spItemURL, nil) + if err != nil { + return "", err + } + req.Header = cloneHeader(header) + if req.Header == nil { + req.Header = http.Header{} + } + req.Header.Set("Accept", "application/json;odata.metadata=minimal") + + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("sp item metadata request failed, status code: %d", resp.StatusCode) + } + + var data spItemDownloadResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return "", err + } + if data.ContentDownloadURL == "" { + return "", fmt.Errorf("sp item metadata response missing @content.downloadUrl") + } + return data.ContentDownloadURL, nil +} + func (d *OnedriveSharelink) headerSnapshot() http.Header { d.headerMu.RLock() defer d.headerMu.RUnlock() diff --git a/drivers/onedrive_sharelink/meta.go b/drivers/onedrive_sharelink/meta.go index 10bea87af2..1aed1b0bad 100644 --- a/drivers/onedrive_sharelink/meta.go +++ b/drivers/onedrive_sharelink/meta.go @@ -11,16 +11,21 @@ type Addition struct { driver.RootPath ShareLinkURL string `json:"url" required:"true"` ShareLinkPassword string `json:"password"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Allow uploading directly to OneDrive without going through OpenList"` IsSharepoint bool downloadLinkPrefix string Headers http.Header HeaderTime int64 + DriveURL string + DriveAccessToken string + DriveTokenTime int64 + driveRootPath string } var config = driver.Config{ Name: "Onedrive Sharelink", - OnlyProxy: true, - NoUpload: true, + LocalSort: true, DefaultRoot: "/", } diff --git a/drivers/onedrive_sharelink/types.go b/drivers/onedrive_sharelink/types.go index 0b7d746456..4405324d8a 100644 --- a/drivers/onedrive_sharelink/types.go +++ b/drivers/onedrive_sharelink/types.go @@ -23,30 +23,55 @@ type FolderResp struct { // Item represents an individual item in the folder. type Item struct { - ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. - Name string `json:"FileLeafRef"` // Name is the name of the item. - ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. - Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. - Id string `json:"UniqueId"` // Id is the unique identifier of the item. + ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. + Name string `json:"FileLeafRef"` // Name is the name of the item. + ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. + Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. + Id string `json:"UniqueId"` // Id is the unique identifier of the item. + SPItemURL string `json:".spItemUrl"` // SPItemURL points to the SharePoint item metadata API. + ContentDownloadURL string `json:"@content.downloadUrl"` // ContentDownloadURL is a temporary cookie-free download URL. } -// fileToObj converts an Item to an ObjThumb. -func fileToObj(f Item) *model.ObjThumb { +type Object struct { + model.ObjThumb + SPItemURL string + ContentDownloadURL string +} + +type uploadSessionResp struct { + UploadURL string `json:"uploadUrl"` +} + +type pageContextInfo struct { + ListURL string `json:"listUrl"` + DriveInfo struct { + DriveURL string `json:".driveUrl"` + DriveAccessToken string `json:".driveAccessToken"` + DriveAccessTokenV2 string `json:".driveAccessTokenV21"` + } `json:"driveInfo"` +} + +// fileToObj converts an Item to an Object. +func fileToObj(f Item) *Object { // Convert Size from string to int64. size, _ := strconv.ParseInt(f.Size, 10, 64) // Convert ObjType from string to int. objtype, _ := strconv.Atoi(f.ObjType) // Create a new ObjThumb with the converted values. - file := &model.ObjThumb{ - Object: model.Object{ - Name: f.Name, - Modified: f.ModifiedTime, - Size: size, - IsFolder: objtype == 1, // Check if the item is a folder. - ID: f.Id, + file := &Object{ + ObjThumb: model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.ModifiedTime, + Size: size, + IsFolder: objtype == 1, // Check if the item is a folder. + ID: f.Id, + }, + Thumbnail: model.Thumbnail{}, }, - Thumbnail: model.Thumbnail{}, + SPItemURL: f.SPItemURL, + ContentDownloadURL: f.ContentDownloadURL, } return file } diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go index d4cd4229ac..13785939fb 100644 --- a/drivers/onedrive_sharelink/util.go +++ b/drivers/onedrive_sharelink/util.go @@ -310,38 +310,25 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, json.NewDecoder(resp.Body).Decode(&graphqlReq) log.Debugln("graphqlReq:", graphqlReq) filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row - if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" { - nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) - log.Debugln("nextHref:", nextHref) - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) + nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + if nextHref != "" { + listViewXml := strings.Replace(graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml, `"`, `\"`, -1) + renderListDataAsStreamVar := strings.Replace( + `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}`, + "REPLACEME", + listViewXml, + -1, + ) - listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml - log.Debugln("listViewXml:", listViewXml) - renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}` - listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1) - renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1) - - graphqlReqNEW := GraphQLNEWRequest{} - postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref - req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) - req.Header = tempHeader - - resp, err := client.Do(req) - if err != nil { - d.Headers, err = d.getHeaders(ctx) - if err != nil { - return nil, err - } - return d.getFiles(ctx, path) - } - defer resp.Body.Close() - json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - for graphqlReqNEW.ListData.NextHref != "" { - graphqlReqNEW = GraphQLNEWRequest{} + for nextHref != "" { + nextHref = nextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" + nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) + log.Debugln("nextHref:", nextHref) + graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) req.Header = tempHeader + resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) @@ -352,13 +339,9 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, } defer resp.Body.Close() json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) filesData = append(filesData, graphqlReqNEW.ListData.Row...) + nextHref = graphqlReqNEW.ListData.NextHref } - filesData = append(filesData, graphqlReqNEW.ListData.Row...) - } else { - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) } return filesData, nil } diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 79fc511854..b37d72a068 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -14,6 +14,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" @@ -195,6 +197,92 @@ func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + // 预计算 hash（如果不存在），使用 RangeRead 不消耗 Reader + // 这样远端驱动不需要再计算，避免 HTTP body 被重复读取 + md5Hash := s.GetHash().GetHash(utils.MD5) + sha1Hash := s.GetHash().GetHash(utils.SHA1) + sha256Hash := s.GetHash().GetHash(utils.SHA256) + sha1_128kHash := s.GetHash().GetHash(utils.SHA1_128K) + preHash := s.GetHash().GetHash(utils.PRE_HASH) + + // 计算所有缺失的 hash，确保最大兼容性 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(s, utils.MD5, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate MD5: %v", err) + md5Hash = "" + } + } + if len(sha1Hash) != utils.SHA1.Width { + var err error + sha1Hash, err = stream.StreamHashFile(s, utils.SHA1, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1: %v", err) + sha1Hash = "" + } + } + if len(sha256Hash) != utils.SHA256.Width { + var err error + sha256Hash, err = stream.StreamHashFile(s, utils.SHA256, 34, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA256: %v", err) + sha256Hash = "" + } + } + + // 计算特殊 hash（用于秒传验证） + // SHA1_128K: 前128KB的SHA1，115网盘使用 + if len(sha1_128kHash) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * 1024 // 128KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + sha1_128kHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1_128K: %v", err) + sha1_128kHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for SHA1_128K: %v", err) + } + } + + // PRE_HASH: 前1024字节的SHA1，阿里云盘使用 + if len(preHash) != utils.PRE_HASH.Width { + const PreHashSize int64 = 1024 // 1KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + preHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate PRE_HASH: %v", err) + preHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for PRE_HASH: %v", err) + } + } + + // 诊断日志：检查流的状态 + if ss, ok := s.(*stream.SeekableStream); ok { + if ss.Reader != nil { + log.Warnf("[openlist] WARNING: SeekableStream.Reader is not nil for file %s, stream may have been consumed!", s.GetName()) + } + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, @@ -206,14 +294,20 @@ func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStream req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { - req.Header.Set("X-File-Md5", md5) + if len(md5Hash) > 0 { + req.Header.Set("X-File-Md5", md5Hash) + } + if len(sha1Hash) > 0 { + req.Header.Set("X-File-Sha1", sha1Hash) + } + if len(sha256Hash) > 0 { + req.Header.Set("X-File-Sha256", sha256Hash) } - if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { - req.Header.Set("X-File-Sha1", sha1) + if len(sha1_128kHash) > 0 { + req.Header.Set("X-File-Sha1-128k", sha1_128kHash) } - if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { - req.Header.Set("X-File-Sha256", sha256) + if len(preHash) > 0 { + req.Header.Set("X-File-Pre-Hash", preHash) } req.ContentLength = s.GetSize() diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index f0b8baf097..26a4288ccf 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -8,6 +8,7 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -18,15 +19,23 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type QuarkOpen struct { model.Storage Addition - config driver.Config - conf Conf + config driver.Config + conf Conf + limiter *rate.Limiter } +// 速率限制常量：夸克开放平台限流，保守设置 +const ( + quarkRateLimit = 2.0 // 每秒2个请求，避免限流 +) + func (d *QuarkOpen) Config() driver.Config { return d.config } @@ -36,6 +45,9 @@ func (d *QuarkOpen) GetAddition() driver.Additional { } func (d *QuarkOpen) Init(ctx context.Context) error { + // 初始化速率限制器 + d.limiter = rate.NewLimiter(rate.Limit(quarkRateLimit), 1) + var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) @@ -52,11 +64,22 @@ func (d *QuarkOpen) Init(ctx context.Context) error { return err } +// waitLimit 等待速率限制 +func (d *QuarkOpen) waitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -67,6 +90,9 @@ func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } data := base.Json{ "fid": file.GetID(), } @@ -143,35 +169,116 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - var ( - md5 hash.Hash - sha1 hash.Hash - ) - writers := []io.Writer{} - if len(md5Str) != utils.MD5.Width { - md5 = utils.MD5.NewFunc() - writers = append(writers, md5) - } - if len(sha1Str) != utils.SHA1.Width { - sha1 = utils.SHA1.NewFunc() - writers = append(writers, sha1) + if err := d.waitLimit(ctx); err != nil { + return err } + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - if len(writers) > 0 { - _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) - if err != nil { - return err - } - if md5 != nil { - md5Str = hex.EncodeToString(md5.Sum(nil)) - } - if sha1 != nil { - sha1Str = hex.EncodeToString(sha1.Sum(nil)) + // 检查是否需要计算hash + needMD5 := len(md5Str) != utils.MD5.Width + needSHA1 := len(sha1Str) != utils.SHA1.Width + + if needMD5 || needSHA1 { + // 检查是否为可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 一次性计算所有hash，避免重复读取 + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + // 使用 RangeRead 分块读取文件，同时计算多个hash + multiWriter := io.MultiWriter(writers...) + size := stream.GetSize() + chunkSize := int64(10 * utils.MB) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := min(chunkSize, size-offset) + + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset) + if err != nil { + return fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + multiWriter.Write(buf[:n]) + offset += int64(n) + + // 更新进度（hash计算占用40%的进度） + up(40 * float64(offset) / float64(size)) + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } + } else { + // 不可重复读取的流（如网络流），需要缓存并计算hash + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) + if err != nil { + return err + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } } - // pre - pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + // pre - 带有 proof fail 重试逻辑 + var pre UpPreResp + var err error + err = retry.Do(func() error { + var preErr error + pre, preErr = d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + if preErr != nil { + // 检查是否为 proof fail 错误 + if strings.Contains(preErr.Error(), "proof") || strings.Contains(preErr.Error(), "43010") { + log.Warnf("[quark_open] Proof verification failed, retrying: %v", preErr) + return preErr // 返回错误触发重试 + } + // 检查是否为限流错误 + if strings.Contains(preErr.Error(), "限流") || strings.Contains(preErr.Error(), "rate") { + log.Warnf("[quark_open] Rate limited, waiting before retry: %v", preErr) + time.Sleep(2 * time.Second) // 额外等待 + return preErr + } + } + return preErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) if err != nil { return err } @@ -181,16 +288,70 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } - // get part info - partInfo := d._getPartInfo(stream, pre.Data.PartSize) - // get upload url info - upUrlInfo, err := d.upUrl(ctx, pre, partInfo) - if err != nil { + // 空文件特殊处理：跳过分片上传，直接调用 upFinish + // 由于夸克 API 对空文件处理不稳定，尝试完成上传，失败则直接成功返回 + if stream.GetSize() == 0 { + log.Infof("[quark_open] Empty file detected, attempting direct finish (task_id: %s)", pre.Data.TaskID) + err = d.upFinish(ctx, pre, []base.Json{}, []string{}) + if err != nil { + // 空文件 upFinish 失败，可能是 API 不支持，直接视为成功 + log.Warnf("[quark_open] Empty file upFinish failed: %v, treating as success", err) + } + up(100) + return nil + } + + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 + var upUrlInfo UpUrlInfo + var partInfo []base.Json + currentPartSize := pre.Data.PartSize + const maxRetries = 5 + const maxPartSize = 1024 * utils.MB // 1GB 上限 + + for attempt := 0; attempt < maxRetries; attempt++ { + // 计算分片信息 + partInfo = d._getPartInfo(stream, currentPartSize) + + // 尝试获取上传 URL + upUrlInfo, err = d.upUrl(ctx, pre, partInfo) + if err == nil { + // 成功获取上传 URL + log.Infof("[quark_open] Successfully obtained upload URLs with part size: %d MB (%d parts)", + currentPartSize/(1024*1024), len(partInfo)) + break + } + + // 检查是否为分片超限错误 + if strings.Contains(err.Error(), "exceed") { + if attempt < maxRetries-1 { + // 还有重试机会，翻倍分片大小 + newPartSize := currentPartSize * 2 + + // 检查是否超过上限 + if newPartSize > maxPartSize { + return fmt.Errorf("part list exceeded and cannot increase part size (current: %d MB, max: %d MB). File may be too large for Quark API", + currentPartSize/(1024*1024), maxPartSize/(1024*1024)) + } + + log.Warnf("[quark_open] Part list exceeded (attempt %d/%d, %d parts). Retrying with doubled part size: %d MB -> %d MB", + attempt+1, maxRetries, len(partInfo), + currentPartSize/(1024*1024), newPartSize/(1024*1024)) + + currentPartSize = newPartSize + continue // 重试 + } else { + // 已达到最大重试次数 + return fmt.Errorf("part list exceeded after %d retries. Last attempt: part size %d MB, %d parts", + maxRetries, currentPartSize/(1024*1024), len(partInfo)) + } + } + + // 其他错误，直接返回 return err } - // part up - ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + // part up - 使用调整后的 currentPartSize + ss, err := streamPkg.NewStreamSectionReader(stream, int(currentPartSize), &up) if err != nil { return err } @@ -204,30 +365,49 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } - offset := int64(i) * pre.Data.PartSize - size := min(pre.Data.PartSize, total-offset) + offset := int64(i) * currentPartSize + size := min(currentPartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } + + // 上传重试逻辑，包含URL刷新 + var etag string err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) - if err != nil { - return err + var uploadErr error + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + + // 检查是否为URL过期错误 + if uploadErr != nil && strings.Contains(uploadErr.Error(), "expire") { + log.Warnf("[quark_open] Upload URL expired for part %d, refreshing...", i) + // 刷新上传URL + newUpUrlInfo, refreshErr := d.upUrl(ctx, pre, partInfo) + if refreshErr != nil { + return fmt.Errorf("failed to refresh upload url: %w", refreshErr) + } + upUrlInfo = newUpUrlInfo + log.Infof("[quark_open] Upload URL refreshed successfully") + + // 使用新URL重试上传 + rd.Seek(0, io.SeekStart) + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) } - etags = append(etags, etag) - return nil + + return uploadErr }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } + etags = append(etags, etag) up(95 * float64(offset+size) / float64(total)) } diff --git a/drivers/quark_open/meta.go b/drivers/quark_open/meta.go index 3527b52e9c..ee1903939d 100644 --- a/drivers/quark_open/meta.go +++ b/drivers/quark_open/meta.go @@ -13,8 +13,8 @@ type Addition struct { APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` - AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` - SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` + AppID string `json:"app_id" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` + SignKey string `json:"sign_key" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` } type Conf struct { diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 788ca0e99a..1a3058375e 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -283,8 +284,15 @@ func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 - partInfo := make([]base.Json, 0) total := stream.GetSize() + + // 确保partSize合理：最小4MB，避免分片过多 + const minPartSize int64 = 4 * utils.MB + if partSize < minPartSize { + partSize = minPartSize + } + + partInfo := make([]base.Json, 0) left := total partNumber := 1 @@ -304,6 +312,7 @@ func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []ba partNumber++ } + log.Infof("[quark_open] Upload plan: file_size=%d, part_size=%d, part_count=%d", total, partSize, len(partInfo)) return partInfo } @@ -315,11 +324,17 @@ func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Js } var resp UpUrlResp + log.Infof("[quark_open] Requesting upload URLs for %d parts (task_id: %s)", len(partInfo), pre.Data.TaskID) + _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { + // 如果是分片超限错误，记录详细信息 + if strings.Contains(err.Error(), "part list exceed") { + log.Errorf("[quark_open] Part list exceeded limit! Requested %d parts. Please check Quark API documentation for actual limit.", len(partInfo)) + } return upUrlInfo, err } @@ -340,13 +355,43 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") + // ✅ 关键修复：使用更长的超时时间（10分钟） + // 慢速网络下大文件分片上传可能需要很长时间 + client := &http.Client{ + Timeout: 10 * time.Minute, + Transport: base.HttpClient.Transport, + } + // 发送请求 - resp, err := base.HttpClient.Do(req) + resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() + // 检查是否为URL过期错误（403, 410等状态码） + if resp.StatusCode == 403 || resp.StatusCode == 410 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("upload url expired (status: %d): %s", resp.StatusCode, string(body)) + } + + // ✅ 关键修复：409 PartAlreadyExist 不是错误！ + // 夸克使用Sequential模式，超时重试时如果分片已存在，说明第一次其实成功了 + if resp.StatusCode == 409 { + body, _ := io.ReadAll(resp.Body) + // 从响应体中提取已存在分片的ETag + if strings.Contains(string(body), "PartAlreadyExist") { + // 尝试从XML响应中提取ETag + if etag := extractEtagFromXML(string(body)); etag != "" { + log.Infof("[quark_open] Part %d already exists (409), using existing ETag: %s", partNumber+1, etag) + return etag, nil + } + // 如果无法提取ETag，返回错误 + log.Warnf("[quark_open] Part %d already exists but cannot extract ETag from response: %s", partNumber+1, string(body)) + return "", fmt.Errorf("part already exists but ETag not found in response") + } + } + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) @@ -355,6 +400,23 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber return resp.Header.Get("Etag"), nil } +// extractEtagFromXML 从OSS的XML错误响应中提取ETag +// 示例: "2F796AC486BB2891E3237D8BFDE020B5" +func extractEtagFromXML(xmlBody string) string { + start := strings.Index(xmlBody, "") + if start == -1 { + return "" + } + start += len("") + end := strings.Index(xmlBody[start:], "") + if end == -1 { + return "" + } + etag := xmlBody[start : start+end] + // 移除引号 + return strings.Trim(etag, "\"") +} + func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) @@ -417,25 +479,36 @@ func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey stri } func (d *QuarkOpen) refreshToken() error { - refresh, access, err := d._refreshToken() + refresh, access, appID, signKey, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, appID, signKey, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access + + // 如果在线API返回了AppID和SignKey，保存它们（不为空时才更新） + if appID != "" && appID != d.AppID { + d.AppID = appID + log.Infof("[quark_open] AppID updated from online API: %s", appID) + } + if signKey != "" && signKey != d.SignKey { + d.SignKey = signKey + log.Infof("[quark_open] SignKey updated from online API") + } + op.MustSaveDriverStorage(d) return nil } -func (d *QuarkOpen) _refreshToken() (string, string, error) { +func (d *QuarkOpen) _refreshToken() (string, string, string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp @@ -448,19 +521,20 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { }). Get(u) if err != nil { - return "", "", err + return "", "", "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { - return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + return "", "", "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } - return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") + return "", "", "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } - return resp.RefreshToken, resp.AccessToken, nil + // 返回所有字段，包括AppID和SignKey + return resp.RefreshToken, resp.AccessToken, resp.AppID, resp.SignKey, nil } // TODO 本地刷新逻辑 - return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") + return "", "", "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 00a32f001a..a57076e08e 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -48,8 +48,8 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), - Path: target, + Path: path, // Use symlink's own path, not target path } - log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) + log.Debugf("[sftp] obj: %+v, is symlink: %v, target: %s", obj, symlink, target) return obj, nil } diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go index 293df8fa14..4822089992 100644 --- a/drivers/sftp/util.go +++ b/drivers/sftp/util.go @@ -3,6 +3,7 @@ package sftp import ( "fmt" "path" + "time" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/pkg/sftp" @@ -39,6 +40,7 @@ func (d *SFTP) _initClient() error { User: d.Username, Auth: []ssh.AuthMethod{auth}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 10 * time.Second, } conn, err := ssh.Dial("tcp", d.Address, config) if err != nil { diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index d420eb4d07..ea0093b7c3 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -68,6 +68,12 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } + // An empty directory reports totalPages == 0, which would make + // pagesData a zero-length slice and panic on pagesData[0] below. + if firstResp.Meta.TotalPages < 1 { + return []model.Obj{}, nil + } + pagesData := make([][]Object, firstResp.Meta.TotalPages) pagesData[0] = firstResp.Items diff --git a/drivers/teldrive/driver_test.go b/drivers/teldrive/driver_test.go new file mode 100644 index 0000000000..a5e4de9dc8 --- /dev/null +++ b/drivers/teldrive/driver_test.go @@ -0,0 +1,38 @@ +package teldrive + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +// An empty directory makes the Teldrive API report totalPages == 0. List used +// to size pagesData by that value and then write pagesData[0], which panicked +// with "index out of range [0] with length 0" on every empty folder. +func TestListEmptyDir(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[],"meta":{"count":0,"totalPages":0,"currentPage":1}}`)) + })) + defer srv.Close() + + oldClient := base.RestyClient + base.RestyClient = resty.New() + defer func() { base.RestyClient = oldClient }() + + d := &Teldrive{} + d.Address = srv.URL + + objs, err := d.List(context.Background(), &model.Object{Path: "/"}, model.ListArgs{}) + if err != nil { + t.Fatalf("List returned error: %v", err) + } + if len(objs) != 0 { + t.Fatalf("expected no entries for an empty dir, got %d", len(objs)) + } +} diff --git a/drivers/teldrive/types.go b/drivers/teldrive/types.go index 084f967e6e..d75347b897 100644 --- a/drivers/teldrive/types.go +++ b/drivers/teldrive/types.go @@ -52,7 +52,7 @@ type chunkTask struct { fileName string chunkSize int64 reader io.ReadSeeker - ss stream.StreamSectionReaderIF + ss stream.StreamSectionReader } type CopyManager struct { diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 492b9814f5..df4113c71d 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -88,6 +88,9 @@ func (x *Thunder) Init(ctx context.Context) (err error) { // 清空 信任密钥 x.Addition.CreditKey = "" } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -521,6 +524,9 @@ func (xc *XunLeiCommon) SetCoreTokenResp(tr *CoreLoginResp) { // 携带Authorization和CaptchaToken的请求 func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), diff --git a/drivers/thunder/driver_test.go b/drivers/thunder/driver_test.go new file mode 100644 index 0000000000..5cea7f4719 --- /dev/null +++ b/drivers/thunder/driver_test.go @@ -0,0 +1,18 @@ +package thunder + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index e60522e529..da1844afa7 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -12,6 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -87,6 +88,9 @@ func (x *ThunderBrowser) Init(ctx context.Context) (err error) { // 清空 信任密钥 x.Addition.CreditKey = "" } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -639,6 +643,9 @@ func (xc *XunLeiBrowserCommon) SetSpaceTokenResp(spaceToken string) { // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiBrowserCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.GetToken(), diff --git a/drivers/thunder_browser/driver_test.go b/drivers/thunder_browser/driver_test.go new file mode 100644 index 0000000000..edf004927e --- /dev/null +++ b/drivers/thunder_browser/driver_test.go @@ -0,0 +1,18 @@ +package thunder_browser + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiBrowserCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index acbb825133..457a9c5367 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -69,13 +69,16 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) - if token.UserID != "" { + if token != nil && token.UserID != "" { x.SetUserID(token.UserID) x.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) } op.MustSaveDriverStorage(x) } } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -496,6 +499,9 @@ func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), diff --git a/drivers/thunderx/driver_test.go b/drivers/thunderx/driver_test.go new file mode 100644 index 0000000000..d3f9ef9c2c --- /dev/null +++ b/drivers/thunderx/driver_test.go @@ -0,0 +1,18 @@ +package thunderx + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiXCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 61e4a6160f..778d810b43 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -10,6 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/gowebdav" @@ -130,6 +131,9 @@ func (d *WebDav) Get(ctx context.Context, _path string) (model.Obj, error) { _path = path.Join(d.GetRootPath(), _path) info, err := d.client.Stat(_path) if err != nil { + if gowebdav.IsErrNotFound(err) { + return nil, errs.ObjectNotFound + } return nil, err } diff --git a/drivers/webdav/driver_test.go b/drivers/webdav/driver_test.go new file mode 100644 index 0000000000..6251f7ff28 --- /dev/null +++ b/drivers/webdav/driver_test.go @@ -0,0 +1,89 @@ +package webdav + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +func TestGetMapsMissingPathToObjectNotFound(t *testing.T) { + d, cleanup := newTestDriver(t, nil) + defer cleanup() + + _, err := d.Get(context.Background(), "/missing") + if !errs.IsObjectNotFound(err) { + t.Fatalf("expected object not found, got %v", err) + } +} + +func TestMakeDirAfterMissingWebDAVStat(t *testing.T) { + var mkcolCount int32 + d, cleanup := newTestDriver(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.Method == "MKCOL" && (r.URL.Path == "/new" || r.URL.Path == "/new/") { + atomic.AddInt32(&mkcolCount, 1) + w.WriteHeader(http.StatusCreated) + return true + } + return false + }) + defer cleanup() + + if err := op.MakeDir(context.Background(), d, "/new"); err != nil { + t.Fatalf("MakeDir failed: %v", err) + } + if got := atomic.LoadInt32(&mkcolCount); got != 1 { + t.Fatalf("expected one MKCOL request, got %d", got) + } +} + +func newTestDriver(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) (*WebDav, func()) { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if extra != nil && extra(w, r) { + return + } + switch r.Method { + case "PROPFIND": + if r.URL.Path == "/" { + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + _, _ = w.Write([]byte(` + + + / + + + / + + + HTTP/1.1 200 OK + + +`)) + return + } + http.NotFound(w, r) + case "MKCOL": + w.WriteHeader(http.StatusConflict) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + + d := &WebDav{Addition: Addition{Address: srv.URL, RootPath: driver.RootPath{RootFolderPath: "/"}}} + if err := d.Init(context.Background()); err != nil { + srv.Close() + t.Fatalf("init driver: %v", err) + } + return d, func() { + _ = d.Drop(context.Background()) + srv.Close() + } +} diff --git a/go.mod b/go.mod index cd86a81474..436036b5a9 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.24.0 - -toolchain go1.24.13 +go 1.25.0 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 @@ -16,7 +14,7 @@ require ( github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 github.com/ProtonMail/gopenpgp/v2 v2.9.0 - github.com/SheltonZhu/115driver v1.2.3 + github.com/SheltonZhu/115driver v1.3.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.5 github.com/antchfx/xpath v1.3.5 @@ -67,11 +65,12 @@ require ( github.com/rclone/rclone v1.70.3 github.com/shirou/gopsutil/v4 v4.25.5 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/afero v1.14.0 - github.com/spf13/cobra v1.9.1 - github.com/stretchr/testify v1.10.0 + github.com/spf13/afero v1.15.0 + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 github.com/tchap/go-patricia/v2 v2.3.3 + github.com/traefik/yaegi v0.16.1 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 @@ -139,7 +138,7 @@ require ( ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.3 + github.com/OpenListTeam/115-sdk-go v0.2.4 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect @@ -284,7 +283,7 @@ require ( github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -295,7 +294,7 @@ require ( go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/sync v0.19.0 - golang.org/x/sys v0.40.0 + golang.org/x/sys v0.44.0 golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 golang.org/x/tools v0.39.0 // indirect @@ -312,4 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.15 + +replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 7587412499..e9790a64a8 100644 --- a/go.sum +++ b/go.sum @@ -21,20 +21,26 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= -github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= +github.com/Ironboxplus/115-sdk-go v0.2.12 h1:6WB2L9FG4n+Xv5RvYXQxZBFvbhgYlWoqKYpGTR0coas= +github.com/Ironboxplus/115-sdk-go v0.2.12/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.13 h1:B815N2mOq2lF2QDyANnkZW3mpmFM7xrBJrqFeENeYVs= +github.com/Ironboxplus/115-sdk-go v0.2.13/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.14 h1:BN1FjRDbJ/UQiwWwO6mbfoU52L9/46l5oHbyEcWDhSI= +github.com/Ironboxplus/115-sdk-go v0.2.14/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.15 h1:GEFamzPCINsB2aRxjcQyvsqC0SmXuQ6H417Ga7BKUII= +github.com/Ironboxplus/115-sdk-go v0.2.15/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= -github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 h1:qLqJPr/FAsZTJiqy65JKKuFJP0V9pRVtSaIE0kqaQ8w= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= @@ -63,8 +69,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2 github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= -github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= +github.com/SheltonZhu/115driver v1.3.3 h1:Bqs86D2MziYPgIOuOJF+HzG4d7GBr71ZhSCrs/U17UU= +github.com/SheltonZhu/115driver v1.3.3/go.mod h1:OujS7azslg1/bn85sPSHnNsp4/WBI9/TiijtZL9kuSQ= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -612,12 +618,13 @@ github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBD github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -632,8 +639,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= @@ -644,6 +651,8 @@ github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8O github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/traefik/yaegi v0.16.1 h1:f1De3DVJqIDKmnasUF6MwmWv1dSEEat0wcpXhD2On3E= +github.com/traefik/yaegi v0.16.1/go.mod h1:4eVhbPb3LnD2VigQjhYbEJ69vDRFdT2HQNrXx8eEwUY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= @@ -692,6 +701,7 @@ go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -770,8 +780,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index 4f367a6319..e5d8fe20ef 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -5,6 +5,7 @@ import ( "io" "io/fs" "strings" + "unicode/utf8" "github.com/KirCute/zip" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -101,6 +102,9 @@ func decodeName(name string, efs bool) string { if efs { return name } + if utf8.ValidString(name) { + return name + } enc, err := ianaindex.IANA.Encoding(setting.GetStr(conf.NonEFSZipEncoding)) if err != nil { return name diff --git a/internal/archive/zip/utils_test.go b/internal/archive/zip/utils_test.go new file mode 100644 index 0000000000..efd2480302 --- /dev/null +++ b/internal/archive/zip/utils_test.go @@ -0,0 +1,47 @@ +package zip + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/stretchr/testify/require" +) + +func TestZipAcceptsLivpExtension(t *testing.T) { + _, ok := tool.Tools[".livp"] + require.True(t, ok) +} + +func setNonEFSZipEncoding(value string) { + op.Cache.SetSetting(conf.NonEFSZipEncoding, &model.SettingItem{ + Key: conf.NonEFSZipEncoding, + Value: value, + }) +} + +func TestDecodeNamePrefersValidUTF8WhenEFSDisabled(t *testing.T) { + setNonEFSZipEncoding("IBM437") + + name := "中文.txt" + require.Equal(t, name, decodeName(name, false)) + // Ensure the setting still exists to verify we did not bypass config due to missing setup. + require.Equal(t, "IBM437", setting.GetStr(conf.NonEFSZipEncoding)) +} + +func TestDecodeNameFallsBackToConfiguredEncoding(t *testing.T) { + setNonEFSZipEncoding("GB18030") + + name := string([]byte{0xd6, 0xd0, 0xce, 0xc4, '.', 't', 'x', 't'}) + require.Equal(t, "中文.txt", decodeName(name, false)) +} + +func TestDecodeNameRespectsEFSFlag(t *testing.T) { + setNonEFSZipEncoding("GB18030") + + name := "utf8-name.txt" + require.Equal(t, name, decodeName(name, true)) +} diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index 1dcb904d74..7adb5a988a 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -17,7 +17,7 @@ type Zip struct { } func (z *Zip) AcceptedExtensions() []string { - return []string{} + return []string{".livp"} } func (z *Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { diff --git a/internal/bootstrap/cluster.go b/internal/bootstrap/cluster.go new file mode 100644 index 0000000000..1d4fb4e106 --- /dev/null +++ b/internal/bootstrap/cluster.go @@ -0,0 +1,29 @@ +package bootstrap + +import ( + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/cluster" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// InitClusterSync loads the storage-sharing cluster engine: identity, config and +// persisted CRDT state, and registers the storage hook so local credential +// changes propagate to peers. The network loops are started later (after storages +// load) by StartClusterSync. Failure is non-fatal — the server runs fine without +// cluster sync. +func InitClusterSync() { + if _, err := cluster.Init(flags.DataDir); err != nil { + utils.Log.Warnf("[cluster] init failed, storage sync disabled: %v", err) + return + } + utils.Log.Infof("[cluster] node %s ready", cluster.Default.NodeID()) +} + +// StartClusterSync seeds records from the now-loaded local storages and starts +// the anti-entropy loop. No-op when cluster sync was not initialized. +func StartClusterSync() { + if cluster.Default == nil { + return + } + cluster.Default.Start() +} diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 74e218f8f6..8304468080 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "math" "net/url" "os" "path/filepath" @@ -96,27 +97,46 @@ func InitConfig() { confFromEnv() } - if conf.Conf.MaxConcurrency > 0 { - net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency} + if conf.Conf.MaxConcurrency > math.MaxInt32 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: math.MaxInt32} + } else if conf.Conf.MaxConcurrency > 0 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: uint32(conf.Conf.MaxConcurrency)} } - if conf.Conf.MaxBufferLimit < 0 { - m, _ := mem.VirtualMemory() - if m != nil { - conf.MaxBufferLimit = max(int(float64(m.Total)*0.05), 4*utils.MB) - conf.MaxBufferLimit -= conf.MaxBufferLimit % utils.MB + + memStat, _ := mem.VirtualMemory() + if memStat != nil { + log.Infof("total memory: %dMB, available: %dMB", memStat.Total>>20, memStat.Available>>20) + if conf.Conf.MinFreeMemory < 0 { + conf.MinFreeMemory = 0 + log.Info("disable memory cache") } else { - conf.MaxBufferLimit = 16 * utils.MB + if conf.Conf.MinFreeMemory < 16 { + t := (memStat.Total >> 20) / 10 + conf.MinFreeMemory = max(16, min(t, 1024)) << 20 + } else { + conf.MinFreeMemory = uint64(conf.Conf.MinFreeMemory) << 20 + } + log.Infof("min free memory: %dMB", conf.MinFreeMemory>>20) } + + if conf.Conf.MaxBlockLimit < 4 { + t := (memStat.Total >> 20) * 3 / 100 + conf.MaxBlockLimit = max(4, min(uint64(t), 64)) << 20 + } else { + conf.MaxBlockLimit = uint64(conf.Conf.MaxBlockLimit) << 20 + } + log.Infof("max block limit: %dMB", conf.MaxBlockLimit>>20) } else { - conf.MaxBufferLimit = conf.Conf.MaxBufferLimit * utils.MB + conf.MinFreeMemory = 0 + log.Warn("failed to get memory info, disable memory cache") } - log.Infof("max buffer limit: %dMB", conf.MaxBufferLimit/utils.MB) - if conf.Conf.MmapThreshold > 0 { - conf.MmapThreshold = conf.Conf.MmapThreshold * utils.MB + + if conf.Conf.AutoMemoryLimit > 0 { + conf.AutoMemoryLimit = uint64(conf.Conf.AutoMemoryLimit) << 20 } else { - conf.MmapThreshold = 0 + conf.AutoMemoryLimit = 0 } - log.Infof("mmap threshold: %dMB", conf.Conf.MmapThreshold) + log.Infof("auto memory limit: %dMB", conf.AutoMemoryLimit>>20) if len(conf.Conf.Log.Filter.Filters) == 0 { conf.Conf.Log.Filter.Enable = false diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index d7fd8ea478..4526aa5234 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -34,6 +34,13 @@ func initSettings() { } settingMap := map[string]*model.SettingItem{} for _, v := range settings { + if v.Key == "" { + err := db.DeleteSettingItemByKey(v.Key) + if err != nil { + utils.Log.Errorf("failed delete setting with empty key: %+v", err) + } + continue + } if !isActive(v.Key) && v.Flag != model.DEPRECATED { v.Flag = model.DEPRECATED err = op.SaveSettingItem(&v) diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 7b91769f9c..751f9b832f 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -50,7 +50,10 @@ func InitDB() { if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) { log.Fatalf("db name error.") } - dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", + // _busy_timeout makes concurrent writers wait for the WAL lock + // instead of failing with SQLITE_BUSY (e.g. parallel storage + // status persistence during startup). + dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental&_busy_timeout=5000", database.DBFile)), gormConfig) } case "mysql": diff --git a/internal/bootstrap/plugin.go b/internal/bootstrap/plugin.go new file mode 100644 index 0000000000..9a453e3b9b --- /dev/null +++ b/internal/bootstrap/plugin.go @@ -0,0 +1,81 @@ +package bootstrap + +import ( + "context" + "path/filepath" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// pluginScanInterval is how often the manager polls the plugins dir for changes +// (hot-reload). +const pluginScanInterval = 30 * time.Second + +// InitPlugins boots the plugin manager: it watches /plugins for backend +// Go-source plugins and frontend JS plugins and keeps them loaded, wires the +// plugin capability functions, and bridges the host's lifecycle hooks into the +// plugin registry so plugins receive real events. Failure is non-fatal — the +// server runs fine without plugins. +func InitPlugins() { + root := filepath.Join(flags.DataDir, "plugins") + mgr, err := plugin.NewManager(context.Background(), root, "/api/plugin/asset") + if err != nil { + utils.Log.Warnf("[plugin] init failed, plugin support disabled: %v", err) + return + } + plugin.Default = mgr + + // Capability wiring. These let plugins read/write settings without the + // plugin package importing internal/op (which would create an import cycle). + plugin.SettingGetter = func(key string) string { + return op.GetSettingsMap()[key] + } + plugin.SettingSetter = func(key, value string) error { + item, err := op.GetSettingItemByKey(key) + if err != nil { + // Unknown key → create a private setting owned by the plugin. + item = &model.SettingItem{ + Key: key, + Type: "string", + Group: model.SINGLE, + Flag: model.PRIVATE, + } + } + item.Value = value + return op.SaveSettingItem(item) + } + + // Bridge the host's existing lifecycle hooks into the plugin registry, so a + // plugin can react to storage and directory changes without us having to + // scatter Fire() calls through op. FireHook is a no-op when nobody listens. + op.RegisterStorageHook(func(typ string, storageDriver driver.Driver) { + hook := map[string]string{ + "add": plugin.HookStorageCreated, + "del": plugin.HookStorageDeleted, + "update": plugin.HookStorageUpdated, + }[typ] + if hook == "" { + return + } + st := storageDriver.GetStorage() + plugin.FireHook(hook, map[string]any{ + "mount_path": st.MountPath, + "driver": st.Driver, + }) + }) + op.RegisterObjsUpdateHook(func(_ context.Context, parent string, objs []model.Obj) { + plugin.FireHook(plugin.HookFsObjsUpdated, map[string]any{ + "path": parent, + "count": len(objs), + }) + }) + + mgr.Start(pluginScanInterval) + utils.Log.Infof("[plugin] manager started, watching %s", root) +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba657..83d282e938 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -15,6 +15,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -37,9 +39,14 @@ func Init() { InitStreamLimit() InitIndex() InitUpgradePatch() + InitPlugins() + InitClusterSync() } func Release() { + if plugin.Default != nil { + _ = plugin.Default.Close() + } db.Close() } @@ -91,6 +98,7 @@ func Start() { } InitOfflineDownloadTools() LoadStorages() + StartClusterSync() InitTaskManager() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) @@ -273,6 +281,7 @@ func Start() { func Shutdown(timeout time.Duration) { utils.Log.Println("Shutdown server...") + frontend.StopWatcher() fs.ArchiveContentUploadTaskManager.RemoveAll() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index 1389095bf5..031b2f5000 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -15,16 +16,51 @@ func LoadStorages() { if err != nil { utils.Log.Fatalf("failed get enabled storages: %+v", err) } - go func(storages []model.Storage) { - for i := range storages { - err := op.LoadStorage(context.Background(), storages[i]) - if err != nil { - utils.Log.Errorf("failed get enabled storages: %+v", err) - } else { - utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", - storages[i].MountPath, storages[i].Driver, storages[i].Order) - } + + // Seed the progress tracker so the status API reports work from the very + // first poll, before any storage has finished loading. + targets := make([]op.LoadTarget, len(storages)) + for i := range storages { + targets[i] = op.LoadTarget{MountPath: storages[i].MountPath, Driver: storages[i].Driver} + } + op.StorageLoadProgress.Begin(targets) + + // Group by driver, preserving order. Storages of the SAME driver must load + // sequentially — they often share a rate limiter, token-refresh mutex or + // login endpoint, so concurrent init of the same driver type races. Different + // driver types are independent and load in parallel. + groupOrder := make([]string, 0) + groups := make(map[string][]model.Storage) + for i := range storages { + d := storages[i].Driver + if _, ok := groups[d]; !ok { + groupOrder = append(groupOrder, d) + } + groups[d] = append(groups[d], storages[i]) + } + + go func() { + var wg sync.WaitGroup + for _, d := range groupOrder { + wg.Add(1) + go func(group []model.Storage) { + defer wg.Done() + for _, storage := range group { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadLoading, "") + err := op.LoadStorage(context.Background(), storage) + if err != nil { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadFailed, err.Error()) + utils.Log.Errorf("failed load storage: [%s], driver: [%s]: %+v", + storage.MountPath, storage.Driver, err) + } else { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadLoaded, "") + utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", + storage.MountPath, storage.Driver, storage.Order) + } + } + }(groups[d]) } + wg.Wait() conf.SendStoragesLoadedSignal() - }(storages) + }() } diff --git a/internal/cluster/config.go b/internal/cluster/config.go new file mode 100644 index 0000000000..df755da135 --- /dev/null +++ b/internal/cluster/config.go @@ -0,0 +1,150 @@ +package cluster + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" +) + +// Config is the operator-facing configuration of the credential-sharing cluster. +// It is persisted as JSON under /cluster/config.json and edited through the +// admin API (and the plugins page UI). Keeping it in its own file rather than the +// global settings table keeps the feature self-contained and pluggable. +// +// The redesign narrows sharing to CREDENTIALS only (tokens/cookies/secrets) and +// routes them through explicit, manually-defined sync groups (see groups.go). +// Peer membership is auto-discovered: any node that can open our AEAD envelope is +// admitted, public nodes advertise their address, and the rest is learned via +// peer exchange. The only manual knobs left are the shared key, an optional +// self-advertised address, and optional bootstrap seeds for the first join. +type Config struct { + // Enabled turns the whole cluster sync on/off. + Enabled bool `json:"enabled"` + // Key is the cluster pre-shared key. Every node in a cluster must share the + // exact same value; it is the sole secret that authenticates membership and + // encrypts traffic. Empty key => sync stays disabled. + Key string `json:"key"` + // Label is a human-friendly name for THIS node shown in the cluster UI of all + // nodes (e.g. "cfscan", "home-nas"). Falls back to the node id when empty. + Label string `json:"label"` + // Addr is this node's externally reachable base URL (e.g. + // "https://node1.example.com"). Advertising it lets other nodes auto-discover + // and dial this node. Leave empty for nodes behind NAT — they dial out and are + // reached through a connected public node's relay instead. + Addr string `json:"addr"` + // Seeds are optional bootstrap base URLs to dial when joining an existing + // cluster. Only needed once: after the first connection, every reachable + // node's address is learned automatically via peer exchange. + Seeds []string `json:"seeds"` + // ApplyRemote, when true (the default for a credential cluster), lets incoming + // peer credentials update local storages in a shared group. Set false to make + // a node observe/share-only. + ApplyRemote bool `json:"apply_remote"` + // AnnounceIntervalSec is how often this node re-broadcasts its inventory + + // credential digests so peers converge after any lost message. 0 => default. + AnnounceIntervalSec int `json:"announce_interval_sec"` +} + +const ( + defaultAnnounceIntervalSec = 45 +) + +func (c Config) announceInterval() int { + if c.AnnounceIntervalSec <= 0 { + return defaultAnnounceIntervalSec + } + return c.AnnounceIntervalSec +} + +// active reports whether sync should actually run: enabled with a key. Unlike the +// previous design it does NOT require a configured peer — a reachable node can run +// purely accept-only and still serve a whole cluster. +func (c Config) active() bool { + return c.Enabled && strings.TrimSpace(c.Key) != "" +} + +// seedList returns the trimmed, non-empty bootstrap URLs. +func (c Config) seedList() []string { + return cleanURLs(c.Seeds) +} + +func cleanURLs(in []string) []string { + out := make([]string, 0, len(in)) + seen := make(map[string]struct{}, len(in)) + for _, p := range in { + p = strings.TrimRight(strings.TrimSpace(p), "/") + if p == "" { + continue + } + if _, dup := seen[p]; dup { + continue + } + seen[p] = struct{}{} + out = append(out, p) + } + return out +} + +// configStore handles persistence of Config under a directory. +type configStore struct { + mu sync.RWMutex + dir string + cfg Config +} + +func newConfigStore(dir string) *configStore { + return &configStore{dir: dir} +} + +func (cs *configStore) path() string { return filepath.Join(cs.dir, "config.json") } + +// loadOrInit reads config.json, or returns a default (disabled, apply-remote-on) +// config if absent. +func (cs *configStore) loadOrInit() (Config, error) { + cs.mu.Lock() + defer cs.mu.Unlock() + b, err := os.ReadFile(cs.path()) + if err != nil { + if os.IsNotExist(err) { + cs.cfg = Config{ApplyRemote: true} + return cs.cfg, nil + } + return Config{}, err + } + var c Config + if err := json.Unmarshal(b, &c); err != nil { + return Config{}, err + } + cs.cfg = c + return c, nil +} + +func (cs *configStore) get() Config { + cs.mu.RLock() + defer cs.mu.RUnlock() + return cs.cfg +} + +// save writes config.json atomically (write temp + rename). +func (cs *configStore) save(c Config) error { + cs.mu.Lock() + defer cs.mu.Unlock() + if err := os.MkdirAll(cs.dir, 0o700); err != nil { + return err + } + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + tmp := cs.path() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + if err := os.Rename(tmp, cs.path()); err != nil { + return err + } + cs.cfg = c + return nil +} diff --git a/internal/cluster/config_test.go b/internal/cluster/config_test.go new file mode 100644 index 0000000000..2105246887 --- /dev/null +++ b/internal/cluster/config_test.go @@ -0,0 +1,42 @@ +package cluster + +import "testing" + +func TestConfigActive(t *testing.T) { + cases := []struct { + name string + cfg Config + want bool + }{ + {"disabled", Config{Enabled: false, Key: "k"}, false}, + {"no key", Config{Enabled: true}, false}, + {"blank key", Config{Enabled: true, Key: " "}, false}, + // A reachable node needs no peer/seed to be active — it can accept-only. + {"active no seed", Config{Enabled: true, Key: "k"}, true}, + {"active with seed", Config{Enabled: true, Key: "k", Seeds: []string{"http://p"}}, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.cfg.active(); got != c.want { + t.Fatalf("active() = %v, want %v", got, c.want) + } + }) + } +} + +func TestConfigSeedListNormalizes(t *testing.T) { + cfg := Config{Seeds: []string{" http://a/ ", "http://b", "", " ", "http://a"}} + got := cfg.seedList() + if len(got) != 2 || got[0] != "http://a" || got[1] != "http://b" { + t.Fatalf("seedList normalize/dedup failed: %#v", got) + } +} + +func TestConfigAnnounceIntervalDefault(t *testing.T) { + if (Config{}).announceInterval() != defaultAnnounceIntervalSec { + t.Fatal("zero interval should fall back to default") + } + if (Config{AnnounceIntervalSec: 10}).announceInterval() != 10 { + t.Fatal("explicit interval should be honored") + } +} diff --git a/internal/cluster/conn.go b/internal/cluster/conn.go new file mode 100644 index 0000000000..2a6f43282a --- /dev/null +++ b/internal/cluster/conn.go @@ -0,0 +1,333 @@ +package cluster + +import ( + "net/http" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/gorilla/websocket" +) + +// Transport model (NAT-friendly): +// +// The sync link is a STATEFUL, persistent WebSocket, never a stateless +// per-message HTTP request. A node behind NAT cannot be dialed, so it instead +// dials OUT to the reachable peers listed in its config and keeps that +// connection open; frames flow in both directions over it for the connection's +// lifetime. A reachable node also ACCEPTS inbound connections at +// /api/cluster/ws and RELAYS applied records between everyone it is connected +// to — so two NAT'd nodes that both dial the same reachable peer still converge +// through it. Every frame is the same sealed envelope used elsewhere, so the +// crypto/CRDT layers are unchanged and transport-agnostic. +const ( + writeWait = 10 * time.Second + pongWait = 60 * time.Second + pingPeriod = (pongWait * 9) / 10 + dialBackoffMin = 2 * time.Second + dialBackoffMax = 60 * time.Second + sendQueueLen = 128 + maxFrameBytes = 16 << 20 +) + +var wsUpgrader = websocket.Upgrader{ + ReadBufferSize: 4096, + WriteBufferSize: 4096, + // Auth is the cluster PSK (AEAD), not the HTTP origin — accept any origin. + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// peerConn is one authenticated, persistent connection to another node. +type peerConn struct { + mgr *Manager + ws *websocket.Conn + send chan []byte + outbound bool // true if we dialed it + addr string // base URL we dialed (outbound only); "" for inbound + since int64 // unix seconds the connection was established + nodeID string // learned from the first authenticated frame + closeOnce sync.Once + closed chan struct{} +} + +func newPeerConn(mgr *Manager, ws *websocket.Conn, outbound bool, addr string) *peerConn { + return &peerConn{ + mgr: mgr, + ws: ws, + send: make(chan []byte, sendQueueLen), + outbound: outbound, + addr: addr, + since: now(), + closed: make(chan struct{}), + } +} + +// enqueue queues a frame for sending; drops it (and closes the conn) if the peer +// is too slow, so a stuck peer can't block the whole node. +func (c *peerConn) enqueue(frame []byte) { + select { + case c.send <- frame: + case <-c.closed: + default: + utils.Log.Warnf("[cluster] peer %s send queue full, dropping connection", c.nodeID) + c.close() + } +} + +func (c *peerConn) close() { + c.closeOnce.Do(func() { + close(c.closed) + _ = c.ws.Close() + c.mgr.conns.remove(c) + }) +} + +// readPump authenticates and dispatches every incoming frame for the connection's +// lifetime. +func (c *peerConn) readPump() { + defer c.close() + c.ws.SetReadLimit(maxFrameBytes) + _ = c.ws.SetReadDeadline(time.Now().Add(pongWait)) + c.ws.SetPongHandler(func(string) error { + return c.ws.SetReadDeadline(time.Now().Add(pongWait)) + }) + for { + mt, data, err := c.ws.ReadMessage() + if err != nil { + return + } + if mt != websocket.BinaryMessage { + continue + } + c.mgr.handleFrame(c, data) + } +} + +// writePump serializes all writes for the connection and keeps it alive with +// periodic pings (essential to hold a NAT mapping open). +func (c *peerConn) writePump() { + ticker := time.NewTicker(pingPeriod) + defer func() { + ticker.Stop() + c.close() + }() + for { + select { + case <-c.closed: + return + case frame := <-c.send: + _ = c.ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.ws.WriteMessage(websocket.BinaryMessage, frame); err != nil { + return + } + case <-ticker.C: + _ = c.ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.ws.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// connRegistry tracks all live connections and indexes the most recent one per +// node id. +type connRegistry struct { + mu sync.RWMutex + conns map[*peerConn]struct{} + byNode map[string]*peerConn +} + +func newConnRegistry() *connRegistry { + return &connRegistry{ + conns: make(map[*peerConn]struct{}), + byNode: make(map[string]*peerConn), + } +} + +func (r *connRegistry) add(c *peerConn) { + r.mu.Lock() + defer r.mu.Unlock() + r.conns[c] = struct{}{} +} + +// bind records the authenticated node id for a connection (idempotent). +func (r *connRegistry) bind(c *peerConn, nodeID string) { + r.mu.Lock() + defer r.mu.Unlock() + c.nodeID = nodeID + r.byNode[nodeID] = c +} + +func (r *connRegistry) remove(c *peerConn) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.conns, c) + if c.nodeID != "" && r.byNode[c.nodeID] == c { + delete(r.byNode, c.nodeID) + } +} + +// all returns a snapshot of live connections. +func (r *connRegistry) all() []*peerConn { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*peerConn, 0, len(r.conns)) + for c := range r.conns { + out = append(out, c) + } + return out +} + +// hasNode reports whether we already have a live connection to a node id (used +// to avoid duplicate dialed+accepted links to the same peer). +func (r *connRegistry) hasNode(nodeID string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.byNode[nodeID] + return ok +} + +func (r *connRegistry) closeAll() { + for _, c := range r.all() { + c.close() + } +} + +// ---- connection establishment ---- + +// wsURL converts a peer base URL (http/https) into its cluster WebSocket URL. +func wsURL(peer string) string { + peer = strings.TrimRight(strings.TrimSpace(peer), "/") + switch { + case strings.HasPrefix(peer, "https://"): + peer = "wss://" + strings.TrimPrefix(peer, "https://") + case strings.HasPrefix(peer, "http://"): + peer = "ws://" + strings.TrimPrefix(peer, "http://") + case strings.HasPrefix(peer, "wss://"), strings.HasPrefix(peer, "ws://"): + // already a ws URL + default: + peer = "ws://" + peer + } + return peer + wsPath +} + +// startConn registers a connection, starts its pumps, and greets the peer with a +// hello (our inventory + known peer addresses for PEX + groups doc + credential +// digests) so discovery and anti-entropy begin immediately. +func (m *Manager) startConn(c *peerConn) { + m.conns.add(c) + go c.writePump() + go c.readPump() + m.sendTo(c, m.helloMessage()) +} + +// dialPeer opens an outbound persistent connection to a peer and blocks until it +// closes. Dialing OUT is what lets a NAT'd node participate without being +// reachable itself. +func (m *Manager) dialPeer(addr string) error { + ws, _, err := websocket.DefaultDialer.Dial(wsURL(addr), nil) + if err != nil { + return err + } + c := newPeerConn(m, ws, true, addr) + m.startConn(c) + <-c.closed + return nil +} + +// ServeWS upgrades an inbound HTTP request to a persistent connection. Auth is +// deferred to the first sealed frame (PSK), so the upgrade itself is open. Any +// node that can open our AEAD envelope is admitted as a peer — this is the +// "authenticated remotes auto-join" half of discovery. +func (m *Manager) ServeWS(w http.ResponseWriter, r *http.Request) { + cfg := m.cfgStore.get() + if !cfg.active() { + http.Error(w, "cluster sync disabled", http.StatusServiceUnavailable) + return + } + ws, err := wsUpgrader.Upgrade(w, r, nil) + if err != nil { + return + } + m.startConn(newPeerConn(m, ws, false, "")) +} + +// dialSupervisor keeps live outbound connections to bootstrap seeds and to every +// auto-discovered peer address we are not already connected to, re-dialing as +// connections drop. Seeds bootstrap the first join; learned addresses (from peer +// inventory exchange) keep public nodes meshed without manual config. +func (m *Manager) dialSupervisor() { + for { + if cfg := m.cfgStore.get(); cfg.active() { + for _, seed := range cfg.seedList() { + m.ensureDial(seed) + } + for _, addr := range m.discoveredDialTargets() { + m.ensureDial(addr) + } + } + select { + case <-m.stopCh: + return + case <-time.After(dialReconcile): + } + } +} + +// discoveredDialTargets returns advertised peer addresses we should dial: those +// belonging to nodes we do not already have a live connection to, and that are +// not our own advertised address. +func (m *Manager) discoveredDialTargets() []string { + self := m.cfgStore.get().Addr + self = trimURL(self) + var out []string + for _, n := range m.state.inventoryList() { + if n.NodeID == m.id.NodeID || n.Addr == "" { + continue + } + if trimURL(n.Addr) == self && self != "" { + continue + } + if m.conns.hasNode(n.NodeID) { + continue + } + out = append(out, n.Addr) + } + return out +} + +func trimURL(s string) string { + return strings.TrimRight(strings.TrimSpace(s), "/") +} + +// ensureDial starts (at most one) outbound dial loop for a peer URL. dialPeer +// blocks for the connection's lifetime, so the dialing flag also prevents a +// duplicate link while connected. +func (m *Manager) ensureDial(addr string) { + addr = trimURL(addr) + if addr == "" { + return + } + m.dialMu.Lock() + if m.dialing[addr] { + m.dialMu.Unlock() + return + } + m.dialing[addr] = true + m.dialMu.Unlock() + go func() { + defer func() { + m.dialMu.Lock() + delete(m.dialing, addr) + m.dialMu.Unlock() + }() + if err := m.dialPeer(addr); err != nil { + utils.Log.Debugf("[cluster] dial %s failed: %v", addr, err) + select { + case <-m.stopCh: + case <-time.After(dialBackoffMin): + } + } + }() +} diff --git a/internal/cluster/conn_test.go b/internal/cluster/conn_test.go new file mode 100644 index 0000000000..d966da55cf --- /dev/null +++ b/internal/cluster/conn_test.go @@ -0,0 +1,41 @@ +package cluster + +import "testing" + +func TestWSURL(t *testing.T) { + cases := map[string]string{ + "https://node.example.com": "wss://node.example.com" + wsPath, + "http://10.0.0.2:5244": "ws://10.0.0.2:5244" + wsPath, + "https://node.example.com/": "wss://node.example.com" + wsPath, + " http://h:1/ ": "ws://h:1" + wsPath, + "node.example.com": "ws://node.example.com" + wsPath, + "wss://already.example.com": "wss://already.example.com" + wsPath, + } + for in, want := range cases { + if got := wsURL(in); got != want { + t.Errorf("wsURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConnRegistry(t *testing.T) { + r := newConnRegistry() + c1 := &peerConn{closed: make(chan struct{})} + c2 := &peerConn{closed: make(chan struct{})} + r.add(c1) + r.add(c2) + if len(r.all()) != 2 { + t.Fatalf("expected 2 conns, got %d", len(r.all())) + } + r.bind(c1, "NODE1") + if !r.hasNode("NODE1") { + t.Fatal("hasNode should find a bound node") + } + r.remove(c1) + if r.hasNode("NODE1") { + t.Fatal("removed conn's node id should be gone") + } + if len(r.all()) != 1 { + t.Fatalf("expected 1 conn after remove, got %d", len(r.all())) + } +} diff --git a/internal/cluster/creds.go b/internal/cluster/creds.go new file mode 100644 index 0000000000..14fcd50262 --- /dev/null +++ b/internal/cluster/creds.go @@ -0,0 +1,180 @@ +package cluster + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "strings" +) + +// This file isolates the one thing the cluster actually replicates: a storage's +// CREDENTIAL fields. The user requirement is explicit — "除了 key 其他的都不用同步" +// (sync only the key/credential, nothing else). So instead of shipping a whole +// storage config we extract just the secret-bearing subset of a driver's Addition +// JSON, propagate that, and overlay it onto the matching storage on peer nodes, +// leaving every node-local field (mount path, root folder, cache, order, proxy…) +// untouched. +// +// OpenList drivers do not tag credential fields, so we identify them by name. The +// keyword set covers the credential fields used across drivers (refresh_token, +// access_token, cookie, password, *_secret, api_key, authorization, …). Matching +// is done on a normalized (lowercased, separator-stripped) field name so +// "refresh_token", "RefreshToken" and "refreshToken" all match. The match runs +// only over Addition keys, where non-credential fields are structural +// (root_folder_id, order_by, …) and never contain these tokens — so false +// positives are not a concern in practice. +var credKeywords = []string{ + "token", // access_token, refresh_token, token + "cookie", // cookie / cookies + "password", // password + "passwd", // passwd + "secret", // client_secret, app_secret, secret_key + "auth", // authorization, auth + "session", // session, session_id + "refresh", // refresh_token (redundant w/ token, kept for clarity) + "access", // access_token (redundant w/ token) + "credential", // credential / credentials + "apikey", // api_key, apikey + "appkey", // app_key + "privatekey", // private_key + "signkey", // sign_key + "ticket", // some drivers use a login ticket + "passport", // 123pan-style passport secret +} + +// normalizeKey lowercases and strips separators so "Refresh_Token" == "refreshtoken". +func normalizeKey(name string) string { + var b strings.Builder + for _, r := range name { + switch { + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): + b.WriteRune(r) + // drop '_', '-', spaces, etc. + } + } + return b.String() +} + +// isCredField reports whether an Addition field name denotes a credential. +func isCredField(name string) bool { + n := normalizeKey(name) + if n == "" { + return false + } + for _, k := range credKeywords { + if strings.Contains(n, k) { + return true + } + } + return false +} + +// extractCreds parses a storage Addition JSON blob and returns only its +// credential fields, in a canonical map keyed by the ORIGINAL JSON key (so they +// can be overlaid back verbatim on a peer). Returns an empty map for an empty or +// unparseable blob. +func extractCreds(additionJSON string) map[string]json.RawMessage { + out := map[string]json.RawMessage{} + if strings.TrimSpace(additionJSON) == "" { + return out + } + var all map[string]json.RawMessage + if err := json.Unmarshal([]byte(additionJSON), &all); err != nil { + return out + } + for k, v := range all { + if isCredField(k) { + out[k] = v + } + } + return out +} + +// credFieldNames returns the sorted credential field names present in a payload. +func credFieldNames(creds map[string]json.RawMessage) []string { + names := make([]string, 0, len(creds)) + for k := range creds { + names = append(names, k) + } + sort.Strings(names) + return names +} + +// canonicalCreds serializes a credential map deterministically (keys sorted) so +// the same credentials always hash identically on every node. +func canonicalCreds(creds map[string]json.RawMessage) []byte { + names := credFieldNames(creds) + var b strings.Builder + b.WriteByte('{') + for i, k := range names { + if i > 0 { + b.WriteByte(',') + } + kb, _ := json.Marshal(k) + b.Write(kb) + b.WriteByte(':') + b.Write(creds[k]) + } + b.WriteByte('}') + return []byte(b.String()) +} + +// credHash is the idempotency key: identical credentials → identical hash on +// every node. An empty credential set hashes to "" so it is never propagated +// (the engine treats "" as "nothing to share / nothing learned"). +func credHash(creds map[string]json.RawMessage) string { + if len(creds) == 0 { + return "" + } + sum := sha256.Sum256(canonicalCreds(creds)) + return hex.EncodeToString(sum[:]) +} + +// applyCreds overlays credential fields onto an existing Addition JSON blob and +// returns the updated blob. Only the keys present in creds are replaced; every +// other field of the target Addition is preserved exactly. Returns (json,true) +// when the blob actually changed, (original,false) when it was already identical +// (so the caller can skip a needless re-init / avoid churn). +func applyCreds(additionJSON string, creds map[string]json.RawMessage) (string, bool) { + if len(creds) == 0 { + return additionJSON, false + } + all := map[string]json.RawMessage{} + if strings.TrimSpace(additionJSON) != "" { + if err := json.Unmarshal([]byte(additionJSON), &all); err != nil { + // Unparseable target: rebuild from creds alone rather than corrupt it. + all = map[string]json.RawMessage{} + } + } + changed := false + for k, v := range creds { + if cur, ok := all[k]; !ok || !jsonEqual(cur, v) { + all[k] = v + changed = true + } + } + if !changed { + return additionJSON, false + } + // Marshal with sorted keys for stability (encoding/json already sorts map keys). + b, err := json.Marshal(all) + if err != nil { + return additionJSON, false + } + return string(b), true +} + +// jsonEqual compares two raw JSON values semantically (after re-normalizing) so +// whitespace differences do not count as a change. +func jsonEqual(a, b json.RawMessage) bool { + var av, bv interface{} + if json.Unmarshal(a, &av) != nil || json.Unmarshal(b, &bv) != nil { + return string(a) == string(b) + } + an, _ := json.Marshal(av) + bn, _ := json.Marshal(bv) + return string(an) == string(bn) +} diff --git a/internal/cluster/crypto.go b/internal/cluster/crypto.go new file mode 100644 index 0000000000..37d5722f05 --- /dev/null +++ b/internal/cluster/crypto.go @@ -0,0 +1,128 @@ +package cluster + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/ed25519" + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/base32" + "errors" + "fmt" + "io" +) + +// Cryptographic design (see docs/cluster-sync.md): +// +// - A cluster pre-shared key (PSK) is the root of trust. HKDF-SHA256 expands it +// into an AES-256-GCM session key. Only nodes holding the PSK can seal/open +// envelopes, which both authenticates membership and encrypts the payload — +// essential because storage configs carry secrets (cloud tokens/passwords). +// - Replay is prevented by a per-envelope random nonce plus a timestamp window +// and a recently-seen-nonce cache (enforced by the transport layer). +// - Each node owns an ed25519 identity keypair. Every synced config version is +// signed by its origin node, so a malicious member cannot forge a higher +// version for a mount it doesn't own (CRDT integrity), and nodes can be +// revoked by dropping their public key. +const ( + aeadInfo = "openlist-cluster-aead-v1" + aeadKeyN = 32 // AES-256 + nodeIDLen = 16 // bytes of the pubkey hash encoded into the node id +) + +// deriveAEADKey expands the cluster PSK into a stable AES-256 key. +func deriveAEADKey(psk []byte) ([]byte, error) { + if len(psk) == 0 { + return nil, errors.New("empty cluster key") + } + // Fixed salt: the PSK is the secret; a per-cluster constant salt is fine and + // keeps key derivation deterministic across nodes. + return hkdf.Key(sha256.New, psk, []byte("openlist-cluster"), aeadInfo, aeadKeyN) +} + +// aead builds the AES-256-GCM AEAD from a derived key. +func newAEAD(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// seal encrypts plaintext with the cluster key, binding aad. Output is +// nonce||ciphertext. aad is authenticated but not encrypted. +func seal(key, plaintext, aad []byte) ([]byte, error) { + gcm, err := newAEAD(key) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + ct := gcm.Seal(nil, nonce, plaintext, aad) + out := make([]byte, 0, len(nonce)+len(ct)) + out = append(out, nonce...) + out = append(out, ct...) + return out, nil +} + +// open reverses seal. It fails (returns error) if the key is wrong, the data was +// tampered with, or aad does not match — i.e. it both decrypts and authenticates. +func open(key, blob, aad []byte) ([]byte, error) { + gcm, err := newAEAD(key) + if err != nil { + return nil, err + } + ns := gcm.NonceSize() + if len(blob) < ns { + return nil, errors.New("ciphertext too short") + } + nonce, ct := blob[:ns], blob[ns:] + return gcm.Open(nil, nonce, ct, aad) +} + +// identity is a node's ed25519 keypair plus its derived node id. +type identity struct { + NodeID string + Pub ed25519.PublicKey + priv ed25519.PrivateKey +} + +// newIdentity generates a fresh ed25519 identity. +func newIdentity() (*identity, error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + return &identity{NodeID: nodeIDFromPub(pub), Pub: pub, priv: priv}, nil +} + +// identityFromSeed rebuilds an identity from a persisted private-key seed. +func identityFromSeed(seed []byte) (*identity, error) { + if len(seed) != ed25519.SeedSize { + return nil, fmt.Errorf("bad identity seed length %d", len(seed)) + } + priv := ed25519.NewKeyFromSeed(seed) + pub := priv.Public().(ed25519.PublicKey) + return &identity{NodeID: nodeIDFromPub(pub), Pub: pub, priv: priv}, nil +} + +func (id *identity) seed() []byte { return id.priv.Seed() } + +func (id *identity) sign(msg []byte) []byte { return ed25519.Sign(id.priv, msg) } + +// nodeIDFromPub derives a short, stable, human-friendly node id from a pubkey. +func nodeIDFromPub(pub ed25519.PublicKey) string { + sum := sha256.Sum256(pub) + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(sum[:nodeIDLen]) +} + +// verifySig checks an ed25519 signature against a known public key. +func verifySig(pub ed25519.PublicKey, msg, sig []byte) bool { + if len(pub) != ed25519.PublicKeySize { + return false + } + return ed25519.Verify(pub, msg, sig) +} diff --git a/internal/cluster/crypto_test.go b/internal/cluster/crypto_test.go new file mode 100644 index 0000000000..85bf81002e --- /dev/null +++ b/internal/cluster/crypto_test.go @@ -0,0 +1,95 @@ +package cluster + +import ( + "bytes" + "testing" +) + +func TestSealOpenRoundTrip(t *testing.T) { + key, err := deriveAEADKey([]byte("super-secret-cluster-key")) + if err != nil { + t.Fatal(err) + } + plain := []byte(`{"token":"abc123","secret":"xyz"}`) + aad := []byte("aad-context") + blob, err := seal(key, plain, aad) + if err != nil { + t.Fatal(err) + } + got, err := open(key, blob, aad) + if err != nil { + t.Fatalf("open failed: %v", err) + } + if !bytes.Equal(got, plain) { + t.Fatalf("round-trip mismatch: %q != %q", got, plain) + } +} + +func TestOpenFailsWithWrongKey(t *testing.T) { + k1, _ := deriveAEADKey([]byte("key-one")) + k2, _ := deriveAEADKey([]byte("key-two")) + blob, _ := seal(k1, []byte("hello"), nil) + if _, err := open(k2, blob, nil); err == nil { + t.Fatal("expected open to fail with a different cluster key") + } +} + +func TestOpenFailsWithTamperedAAD(t *testing.T) { + key, _ := deriveAEADKey([]byte("k")) + blob, _ := seal(key, []byte("hello"), []byte("aad-1")) + if _, err := open(key, blob, []byte("aad-2")); err == nil { + t.Fatal("expected open to fail when aad differs") + } +} + +func TestDeriveAEADKeyDeterministicAndRejectsEmpty(t *testing.T) { + a, err := deriveAEADKey([]byte("same")) + if err != nil { + t.Fatal(err) + } + b, _ := deriveAEADKey([]byte("same")) + if !bytes.Equal(a, b) { + t.Fatal("key derivation must be deterministic across nodes") + } + if len(a) != aeadKeyN { + t.Fatalf("derived key length = %d, want %d", len(a), aeadKeyN) + } + if _, err := deriveAEADKey(nil); err == nil { + t.Fatal("empty PSK must be rejected") + } +} + +func TestIdentityNodeIDBoundToPubKey(t *testing.T) { + id, err := newIdentity() + if err != nil { + t.Fatal(err) + } + // node id is the hash of the pubkey — re-deriving must match. + if nodeIDFromPub(id.Pub) != id.NodeID { + t.Fatal("node id is not bound to pubkey") + } + // persistence round-trip via seed. + id2, err := identityFromSeed(id.seed()) + if err != nil { + t.Fatal(err) + } + if id2.NodeID != id.NodeID { + t.Fatal("identity not stable across seed reload") + } +} + +func TestSignVerify(t *testing.T) { + id, _ := newIdentity() + msg := []byte("authentic message") + sig := id.sign(msg) + if !verifySig(id.Pub, msg, sig) { + t.Fatal("valid signature rejected") + } + if verifySig(id.Pub, []byte("tampered"), sig) { + t.Fatal("signature verified against the wrong message") + } + other, _ := newIdentity() + if verifySig(other.Pub, msg, sig) { + t.Fatal("signature verified under the wrong pubkey") + } +} diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go new file mode 100644 index 0000000000..5d289217d7 --- /dev/null +++ b/internal/cluster/engine.go @@ -0,0 +1,1142 @@ +package cluster + +import ( + "context" + "encoding/json" + stderrors "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + // replayWindowSec bounds clock skew + in-flight time for envelope freshness. + replayWindowSec = 300 + // wsPath is the persistent-connection endpoint peers dial. + wsPath = "/api/cluster/ws" + // dialReconcile is how often the dial supervisor re-checks connectivity. + dialReconcile = 5 * time.Second + // peerLivenessSec is how long a node is considered online after last contact + // (absent a live connection). + peerLivenessSec = 130 + // maxEvents bounds the in-memory activity log surfaced to the UI. + maxEvents = 60 +) + +// Manager is the running cluster credential-sync engine for this node. +type Manager struct { + dir string + id *identity + cfgStore *configStore + state *store + replay *replayCache + conns *connRegistry + + persistMu sync.Mutex + + recoveryMu sync.Mutex + recoveryLocks map[string]*sync.Mutex // per local mount: serialize credential adoption + recoveryQueues map[string]*recoveryQueue + + dialMu sync.Mutex + dialing map[string]bool // peer URLs with an in-flight/live outbound dial + + invMu sync.Mutex + lastSelfVer uint64 // monotonic version for our own inventory entry + + evMu sync.Mutex + events []EventView + + stopCh chan struct{} + once sync.Once +} + +// recoveryQueue is one mount's event-driven recovery mailbox. It deliberately +// contains no timer: a pair is enqueued by an observed 401, a newly received +// signed candidate, or startup reconstruction. The source of truth remains the +// durable candidate/tombstone state, so a crash can safely rebuild this queue. +type recoveryQueue struct { + running bool + pending map[string]*credRecord // credential hash -> immutable signed candidate +} + +// Default is the process-wide manager, set by Init. It is nil when cluster sync +// was never initialized. +var Default *Manager + +func now() int64 { return time.Now().Unix() } + +// Init constructs the manager: loads identity + config + persisted state and +// registers the storage hook so local credential changes propagate. It does NOT +// start the network loops — call Start once storages are loaded. +func Init(dataDir string) (*Manager, error) { + dir := filepath.Join(dataDir, "cluster") + id, err := loadOrCreateIdentity(dir) + if err != nil { + return nil, err + } + m := &Manager{ + dir: dir, + id: id, + cfgStore: newConfigStore(dir), + state: newStore(), + replay: newReplayCache(replayWindowSec), + conns: newConnRegistry(), + dialing: make(map[string]bool), + recoveryLocks: make(map[string]*sync.Mutex), + recoveryQueues: make(map[string]*recoveryQueue), + stopCh: make(chan struct{}), + } + if _, err := m.cfgStore.loadOrInit(); err != nil { + return nil, err + } + m.loadState() + + op.RegisterStorageHook(m.onStorageHook) + op.RegisterStorageCredentialHook(m.onStorageCredential) + op.RegisterStorageCredentialHealthHook(m.onStorageCredentialHealthy) + Default = m + return m, nil +} + +// NodeID returns this node's stable identity string. +func (m *Manager) NodeID() string { return m.id.NodeID } + +// ---- persistence ---- + +func (m *Manager) statePath() string { return filepath.Join(m.dir, "state.json") } + +func (m *Manager) loadState() { + b, err := os.ReadFile(m.statePath()) + if err != nil { + return + } + var ps persistedState + if err := json.Unmarshal(b, &ps); err != nil { + utils.Log.Warnf("[cluster] corrupt state file, ignoring: %v", err) + return + } + m.state.load(ps) + // resume our self-inventory version so peers never see us go backwards. + for _, n := range ps.Inventory { + if n.NodeID == m.id.NodeID && n.Version > m.lastSelfVer { + m.lastSelfVer = n.Version + } + } +} + +func (m *Manager) persist() error { + m.persistMu.Lock() + defer m.persistMu.Unlock() + ps := m.state.export() + b, err := json.MarshalIndent(ps, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(m.dir, 0o700); err != nil { + return err + } + tmp := m.statePath() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + utils.Log.Warnf("[cluster] failed to persist state: %v", err) + return err + } + if err := os.Rename(tmp, m.statePath()); err != nil { + return err + } + return nil +} + +// ---- config access ---- + +// GetConfig returns the current config (with the key redacted for display when +// redact is true). +func (m *Manager) GetConfig(redact bool) Config { + c := m.cfgStore.get() + if redact && c.Key != "" { + c.Key = "********" + } + return c +} + +// SetConfig persists a new config. A blank/"********" key keeps the existing key. +func (m *Manager) SetConfig(c Config) error { + old := m.cfgStore.get() + if c.Key == "" || c.Key == "********" { + c.Key = old.Key + } + c.Seeds = cleanURLs(c.Seeds) + c.Addr = trimURL(c.Addr) + if err := m.cfgStore.save(c); err != nil { + return err + } + // Drop all live connections so they reconnect with the new settings. + m.conns.closeAll() + go m.refreshInventory() + go m.seedLocalCreds() + return nil +} + +// GroupSpec / MemberSpec are the exported, API-facing shapes for editing groups. +type MemberSpec struct { + NodeID string `json:"node_id"` + MountPath string `json:"mount_path"` +} + +type GroupSpec struct { + ID string `json:"id"` + Name string `json:"name"` + Members []MemberSpec `json:"members"` +} + +// SetGroups replaces the cluster-shared sync-group document (admin edit) and +// propagates it. It then re-evaluates local credentials against the new groups. +func (m *Manager) SetGroups(specs []GroupSpec) error { + groups := make([]group, 0, len(specs)) + for _, s := range specs { + g := group{ID: s.ID, Name: s.Name} + for _, ms := range s.Members { + if ms.NodeID == "" || ms.MountPath == "" { + continue + } + g.Members = append(g.Members, member{NodeID: ms.NodeID, MountPath: ms.MountPath}) + } + if g.ID == "" || len(g.Members) == 0 { + continue + } + groups = append(groups, g) + } + d := m.state.setGroups(m.id, groups, now()) + m.state.pruneCreds() + if err := m.persist(); err != nil { + return fmt.Errorf("persist groups: %w", err) + } + m.recordEvent("groups", "", fmt.Sprintf("updated to %d group(s)", len(groups))) + gd := d + m.broadcast(&syncMessage{Type: "push", Groups: &gd}) + go m.seedLocalCreds() + return nil +} + +// ---- lifecycle ---- + +// Start seeds inventory + credentials from local storages and launches the +// connection dialer and anti-entropy loop. +func (m *Manager) Start() { + m.once.Do(func() { + m.state.setLocalInventory(m.selfNodeInfo()) + go m.seedLocalCreds() + go m.announceLoop() + go m.dialSupervisor() + }) +} + +// Stop halts background loops and closes all connections. +func (m *Manager) Stop() { + select { + case <-m.stopCh: + default: + close(m.stopCh) + } + m.conns.closeAll() +} + +// ---- self inventory ---- + +// selfNodeInfo builds this node's inventory entry from currently-loaded storages. +func (m *Manager) selfNodeInfo() *nodeInfo { + cfg := m.cfgStore.get() + var sts []storageInfo + for _, d := range op.GetAllStorages() { + st := d.GetStorage() + sts = append(sts, storageInfo{MountPath: st.MountPath, Driver: st.Driver, Status: st.Status}) + } + sort.Slice(sts, func(i, j int) bool { return sts[i].MountPath < sts[j].MountPath }) + + m.invMu.Lock() + v := uint64(now()) + if v <= m.lastSelfVer { + v = m.lastSelfVer + 1 + } + m.lastSelfVer = v + m.invMu.Unlock() + + return &nodeInfo{ + NodeID: m.id.NodeID, + Label: cfg.Label, + Addr: trimURL(cfg.Addr), + Storages: sts, + Version: v, + UpdatedAt: now(), + } +} + +// refreshInventory rebuilds and broadcasts our inventory entry (call after a +// storage is added/removed or its status changes). +func (m *Manager) refreshInventory() { + if !m.cfgStore.get().active() { + return + } + self := m.selfNodeInfo() + m.state.setLocalInventory(self) + m.broadcast(m.announceMessage()) +} + +// knownNodesForPEX returns inventory entries that advertise a dialable address, +// so peers can auto-discover the rest of the mesh. +func (m *Manager) knownNodesForPEX() []*nodeInfo { + var out []*nodeInfo + for _, n := range m.state.inventoryList() { + if n.Addr == "" { + continue + } + cp := n + out = append(out, &cp) + } + return out +} + +// helloMessage greets a freshly-connected peer with everything needed to +// converge: our inventory, known dialable peers (PEX), the groups doc, and our +// credential digests. +func (m *Manager) helloMessage() *syncMessage { + self := m.selfNodeInfo() + m.state.setLocalInventory(self) + gd := m.state.groupDoc() + return &syncMessage{ + Type: "hello", + Node: self, + Nodes: m.knownNodesForPEX(), + Groups: &gd, + GroupsVer: gd.Version, + Revocations: m.state.revocationSnapshot(), + CredDigests: m.offerableDigests(), + } +} + +// announceMessage is the periodic anti-entropy + inventory advert. +func (m *Manager) announceMessage() *syncMessage { + self := m.selfNodeInfo() + gd := m.state.groupDoc() + return &syncMessage{ + Type: "announce", + Node: self, + Nodes: m.knownNodesForPEX(), + Groups: &gd, + GroupsVer: gd.Version, + Revocations: m.state.revocationSnapshot(), + CredDigests: m.offerableDigests(), + } +} + +// announceLoop periodically rebroadcasts inventory + digests for convergence. +func (m *Manager) announceLoop() { + for { + cfg := m.cfgStore.get() + interval := time.Duration(cfg.announceInterval()) * time.Second + select { + case <-m.stopCh: + return + case <-time.After(interval): + } + if !m.cfgStore.get().active() { + continue + } + m.state.setLocalInventory(m.selfNodeInfo()) + m.broadcast(m.announceMessage()) + } +} + +// ---- credential seeding / hooks ---- + +// seedLocalCreds reconciles every locally failed mount after startup or a group +// change. A persisted Storage.Status=WORK is not proof that its credential pair +// is still accepted by the provider, so startup never mints a fresh candidate +// from it; however, a persisted non-WORK mount must retry durable peer +// candidates even when they arrived before this process started. +func (m *Manager) seedLocalCreds() { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + selfID := m.id.NodeID + for _, d := range op.GetAllStorages() { + st := d.GetStorage() + groups := m.state.groupsForMount(selfID, st.MountPath) + if len(groups) == 0 { + continue + } + for _, g := range groups { + if st.Status != op.WORK { + m.enqueueKnownCandidates(g.ID, st.MountPath, credHash(extractCreds(st.Addition))) + } + m.pullGroup(g.ID) + } + } +} + +// revokeOwnCandidate removes the local source candidate and returns a signed +// tombstone that every relay can retain. The tombstone's version is newer than +// the removed credential, so an old relay replay cannot re-enter state while a +// future credential rotation from this source remains valid. +func (m *Manager) revokeOwnCandidate(groupID, mount string) (*credRevocation, bool) { + return m.state.revokeOwnCred(m.id, groupID, mount, now()) +} + +// onStorageHook only tracks lifecycle/inventory changes. Authentication events +// are intentionally handled by onStorageCredential because they need the +// immutable Addition snapshot of the request that produced them. +func (m *Manager) onStorageHook(typ string, d driver.Driver) { + if !m.cfgStore.get().active() || d == nil || d.GetStorage() == nil { + return + } + if typ == "token-valid" || typ == "token-invalid" { + return + } + go m.refreshInventory() +} + +// onStorageCredential consumes a real provider result bound to the exact +// Addition used on the wire. It is the sole authority for publishing a pair or +// writing its group-wide 401 tombstone; ordinary storage update hooks are never +// authentication evidence. +func (m *Manager) onStorageCredential(typ string, event op.StorageCredentialEvent) { + if !m.cfgStore.get().active() || event.Storage == nil { + return + } + st := event.Storage.GetStorage() + if st == nil || st.Addition != event.Addition || !st.Modified.Equal(event.Modified) { + return // late result from an old client generation + } + go m.refreshInventory() + groups := m.state.groupsForMount(m.id.NodeID, st.MountPath) + if len(groups) == 0 { + return + } + + switch typ { + case "token-valid": + if st.Status != op.WORK { + for _, g := range groups { + m.pullGroup(g.ID) + } + return + } + creds := extractCreds(event.Addition) + credID := credHash(creds) + var dirty bool + var shares []*credRecord + for _, g := range groups { + if m.state.hasCredForSource(g.ID, m.id.NodeID, st.MountPath, credID) { + m.state.markCandidateHealthy(g.ID, credID, now()) + continue + } + if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { + m.state.markCandidateHealthy(g.ID, rec.CredHash, now()) + dirty = true + m.recordEvent("share", g.ID, fmt.Sprintf("%s refreshed credentials", st.MountPath)) + shares = append(shares, rec) + } + } + if dirty { + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not sharing unpersisted credential for %s: %v", st.MountPath, err) + return + } + } + if len(shares) > 0 { + m.broadcast(&syncMessage{Type: "push", Creds: shares}) + } + + case "token-invalid": + // This event is generation-bound. Any 401xxxxx invalidates the precise + // access/refresh pair used by the request even if another asynchronous + // success would otherwise leave Storage.Status as WORK. + credID := credHash(extractCreds(event.Addition)) + if credID == "" { + return + } + var revocations []*credRevocation + for _, g := range groups { + m.state.forgetCandidateHealth(g.ID, credID) + m.state.markCandidateFailed(g.ID, st.MountPath, credID, now()) + if rev, ok := m.state.revokePair(m.id, g.ID, st.MountPath, credID, now()); ok { + m.recordEvent("invalidate", g.ID, + fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) + revocations = append(revocations, rev) + } else { + m.recordEvent("invalidate", g.ID, + fmt.Sprintf("%s token invalid — quarantined failed peer candidate", st.MountPath)) + } + } + // The pair-wide 401 fact must survive a crash before a recovery worker + // can touch any alternative. This makes Revoked(P) durable-before-apply. + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not recovering after unpersisted 401 for %s: %v", st.MountPath, err) + return + } + if len(revocations) > 0 { + m.broadcast(&syncMessage{Type: "push", Revocations: revocations}) + } + for _, g := range groups { + m.enqueueKnownCandidates(g.ID, st.MountPath, credID) + m.pullGroup(g.ID) + } + } +} + +// onStorageCredentialHealthy renews in-memory proof only for the exact pair +// that received a successful provider response. It cannot accidentally mark a +// newer pair healthy after an old request returns late. +func (m *Manager) onStorageCredentialHealthy(event op.StorageCredentialEvent) { + if !m.cfgStore.get().active() || event.Storage == nil { + return + } + st := event.Storage.GetStorage() + if st == nil || st.Status != op.WORK || st.Addition != event.Addition || !st.Modified.Equal(event.Modified) { + return + } + credID := credHash(extractCreds(event.Addition)) + if credID == "" { + return + } + for _, g := range m.state.groupsForMount(m.id.NodeID, st.MountPath) { + if m.state.hasCredHash(g.ID, credID) { + m.state.markCandidateHealthy(g.ID, credID, now()) + m.state.clearCandidateFailed(g.ID, st.MountPath, credID) + } + } +} + +// enqueueKnownCandidates rebuilds a failed mount's durable recovery work. It is +// used after a local 401 and after restart; it never creates candidates or +// refreshes a token. +func (m *Manager) enqueueKnownCandidates(groupID, mount, failedHash string) { + for _, candidate := range m.state.recoveryCandidates(groupID, mount, failedHash) { + m.enqueueCandidate(candidate, mount) + } +} + +// enqueueCandidate queues one newly learned, signed candidate for one local +// mount. A queue is keyed by credential hash, so duplicate frames and relay +// paths cannot create probe storms. The worker is per (group, mount), making +// activation deterministic and serial with storage replacement. +func (m *Manager) enqueueCandidate(candidate *credRecord, mount string) { + if candidate == nil || mount == "" || !m.credentialAuthorized(candidate) { + return + } + key := candidate.GroupID + "\x00" + mount + m.recoveryMu.Lock() + if m.recoveryQueues == nil { + m.recoveryQueues = make(map[string]*recoveryQueue) + } + queue := m.recoveryQueues[key] + if queue == nil { + queue = &recoveryQueue{pending: make(map[string]*credRecord)} + m.recoveryQueues[key] = queue + } + if _, exists := queue.pending[candidate.CredHash]; !exists { + cp := *candidate + queue.pending[candidate.CredHash] = &cp + } + if queue.running { + m.recoveryMu.Unlock() + return + } + queue.running = true + m.recoveryMu.Unlock() + go m.runCandidateRecovery(key, candidate.GroupID, mount) +} + +func (m *Manager) takeCandidate(key string) *credRecord { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + queue := m.recoveryQueues[key] + if queue == nil { + return nil + } + if len(queue.pending) == 0 { + queue.running = false + delete(m.recoveryQueues, key) + return nil + } + hashes := make([]string, 0, len(queue.pending)) + for hash := range queue.pending { + hashes = append(hashes, hash) + } + sort.Strings(hashes) + candidate := queue.pending[hashes[0]] + delete(queue.pending, hashes[0]) + return candidate +} + +func (m *Manager) finishCandidateRecovery(key string) { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + delete(m.recoveryQueues, key) +} + +// runCandidateRecovery is the only place a received candidate may become the +// mount's active storage. A successful probe stops the queue; a provider 401 +// emits a durable group-wide tombstone before the next candidate is attempted. +func (m *Manager) runCandidateRecovery(key, groupID, mount string) { + for { + candidate := m.takeCandidate(key) + if candidate == nil { + return + } + if !m.credentialAuthorized(candidate) { + continue + } + attempted, success, terminal := m.applyCredRecordToMount(candidate, mount) + if success { + m.finishCandidateRecovery(key) + return + } + if !attempted || !terminal { + continue + } + if rev, revoked := m.state.revokePair(m.id, groupID, mount, candidate.CredHash, now()); revoked { + m.recordEvent("invalidate", groupID, + fmt.Sprintf("%s rejected candidate %s with provider 401", mount, shortHash(candidate.CredHash))) + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted candidate rejection for %s: %v", mount, err) + return + } + m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) + } + } +} + +// pullGroup asks peers for the latest credential of a group. +func (m *Manager) pullGroup(groupID string) { + if !m.cfgStore.get().active() { + return + } + m.broadcast(&syncMessage{Type: "pull", Wants: []string{groupID}}) +} + +// hasHealthyLocalCandidate reports whether this node is currently using this +// exact credential successfully. A signed record received from a peer is not +// evidence that it works on this machine. +func (m *Manager) hasHealthyLocalCandidate(r *credRecord) bool { + if r == nil || !m.state.candidateHealthy(r.GroupID, r.CredHash, now()) { + return false + } + g, ok := m.state.groupByID(r.GroupID) + if !ok { + return false + } + for _, mp := range g.mountsForNode(m.id.NodeID) { + d, err := op.GetStorageByMountPath(mp) + if err != nil { + continue + } + st := d.GetStorage() + if st.Status != op.WORK { + continue + } + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + continue + } + if credHash(extractCreds(st.Addition)) == r.CredHash { + return true + } + } + return false +} + +// canOfferCred reports whether this node may relay a credential candidate. A +// valid member signature is the authority for transport: a relay does not need +// to prove the pair locally before forwarding it. Local proof only controls +// whether this node activates a pair on one of its mounts. Keeping those two +// facts separate is essential for NAT/accept-only topologies, where a healthy +// source and an invalid target may never have a direct connection. +func (m *Manager) canOfferCred(r *credRecord) bool { + return m.credentialAuthorized(r) +} + +// offerableDigests is the signed, non-revoked candidate inventory that may be +// relayed during anti-entropy. It is deliberately independent of local mount +// health; otherwise a hub that did not itself mount 115 would black-hole a +// healthy source's credential record. +func (m *Manager) offerableDigests() []credDigest { + var out []credDigest + for _, r := range m.state.credSnapshot() { + if m.canOfferCred(r) { + out = append(out, digestOfCred(r)) + } + } + return out +} + +func (m *Manager) recoveryLock(mount string) *sync.Mutex { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + if m.recoveryLocks == nil { + m.recoveryLocks = make(map[string]*sync.Mutex) + } + lock := m.recoveryLocks[mount] + if lock == nil { + lock = &sync.Mutex{} + m.recoveryLocks[mount] = lock + } + return lock +} + +// applyCredRecord overlays a group's credential onto this node's member mounts. +func (m *Manager) applyCredRecord(r *credRecord) { + if r == nil { + return + } + cfg := m.cfgStore.get() + if !cfg.ApplyRemote { + return + } + g, ok := m.state.groupByID(r.GroupID) + if !ok { + return + } + for _, mp := range g.mountsForNode(m.id.NodeID) { + m.applyCredRecordToMount(r, mp) + } +} + +// applyCredRecordToMount adopts one candidate on one local mount. It returns +// whether a probe was attempted, whether activation completed, and whether the +// probe received an exact provider-auth 401. Candidate payloads are verified by +// a temporary real driver before production storage is changed. +func (m *Manager) applyCredRecordToMount(r *credRecord, mp string) (attempted, success, terminal bool) { + if r == nil || !m.cfgStore.get().ApplyRemote || m.state.candidateFailed(r.GroupID, mp, r.CredHash) { + return false, false, false + } + ctx := context.Background() + { + lock := m.recoveryLock(mp) + lock.Lock() + defer lock.Unlock() + d, err := op.GetStorageByMountPath(mp) + if err != nil { + return false, false, false + } + before := *d.GetStorage() // preserve a local LKG for rollback + st := before + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", + r.GroupID, mp, st.Driver, r.OriginDriver) + return false, false, false + } + if st.Status == op.WORK { + // A peer's newer Lamport value is a recovery candidate, not authority + // to overwrite credentials this mount has already proved locally. + return false, false, false + } + newAdd, changed := applyCreds(st.Addition, r.Payload) + if !changed { + return false, false, false // already has these credentials — no churn, no re-init + } + if err := probeCredential(ctx, st, newAdd); err != nil { + utils.Log.Warnf("[cluster] probe candidate %s for %s failed: %v", shortHash(r.CredHash), mp, err) + return true, false, isProvider401(err) + } + st.Addition = newAdd + if err := op.UpdateStorage(ctx, st); err != nil { + utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) + // UpdateStorage persists before it initializes. Restore the complete + // previous storage record rather than leaving an unproven pair behind. + if rollbackErr := op.UpdateStorage(ctx, before); rollbackErr != nil { + utils.Log.Errorf("[cluster] rollback %s after failed candidate commit: %v", mp, rollbackErr) + } + return true, false, false + } + m.state.markCandidateHealthy(r.GroupID, r.CredHash, now()) + m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) + utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) + return true, true, false + } + return false, false, false +} + +// probeCredential constructs the registered production driver in isolation. It +// runs that driver's actual Init path (115 Open performs UserInfo) without +// registering it in op's storage map or writing SQLite. A failed candidate thus +// never replaces the mount's active/LKG pair. +func probeCredential(ctx context.Context, storage model.Storage, addition string) error { + constructor, err := op.GetDriver(storage.Driver) + if err != nil { + return err + } + temporary := constructor() + probe := newProbeStorage(storage, addition) + temporary.SetStorage(probe) + if err := utils.Json.UnmarshalFromString(probe.Addition, temporary.GetAddition()); err != nil { + return err + } + probeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return temporary.Init(probeCtx) +} + +// newProbeStorage gives a temporary driver the candidate fields but no ownership +// of the persisted storage row. Some production drivers report an authenticated +// Init through lifecycle hooks; ID=0 prevents such a callback from changing the +// live row, and WORK prevents a successful probe from emitting token-valid. +func newProbeStorage(live model.Storage, addition string) model.Storage { + live.ID = 0 + live.Status = op.WORK + live.Addition = addition + return live +} + +func isProvider401(err error) bool { + var authErr *sdk.Error + return stderrors.As(err, &authErr) && sdk.Is401Started(authErr.Code) +} + +// ---- absorb (merge) helpers ---- + +func (m *Manager) absorbInventory(nodes []*nodeInfo) []*nodeInfo { + var merged []*nodeInfo + for _, n := range nodes { + if n == nil || n.NodeID == "" || n.NodeID == m.id.NodeID { + continue + } + if m.state.mergeInventory(n, now()) { + merged = append(merged, n) + } + } + return merged +} + +func (m *Manager) absorbGroups(d *groupDoc) bool { + if d == nil || !d.verify() { + return false + } + if m.state.mergeGroups(d) { + m.state.pruneCreds() + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted groups: %v", err) + return false + } + m.recordEvent("groups", "", fmt.Sprintf("received %d group(s) from %s", len(d.Groups), shortNode(d.Origin))) + go m.seedLocalCreds() + return true + } + return false +} + +func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { + var merged []*credRecord + for _, r := range recs { + if r == nil || !r.verify() || !m.credentialAuthorized(r) { + continue + } + if m.state.mergeCred(r) { + merged = append(merged, r) + } + } + if len(merged) > 0 { + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not activating unpersisted received credentials: %v", err) + return nil + } + for _, r := range merged { + // A record newly arriving after a local 401 must wake recovery even if + // the earlier empty-candidate scan already completed. The queue itself + // deduplicates the pair and serializes the real driver probe. + g, ok := m.state.groupByID(r.GroupID) + if !ok { + continue + } + for _, mount := range g.mountsForNode(m.id.NodeID) { + m.enqueueCandidate(r, mount) + } + } + } + return merged +} + +func (m *Manager) absorbRevocations(revs []*credRevocation) []*credRevocation { + var merged []*credRevocation + for _, rev := range revs { + if !credentialRevocationAuthorizedByGroups(m.state.groupList(), rev) { + continue + } + if m.state.mergeRevocation(rev) { + merged = append(merged, rev) + m.recordEvent("revoke", rev.GroupID, + fmt.Sprintf("%s revoked candidate from %s", rev.OriginMount, shortNode(rev.Origin))) + } + } + if len(merged) > 0 { + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted revocations: %v", err) + return nil + } + } + return merged +} + +// credentialAuthorized verifies that a record signer is an explicit member of +// the signed group at the exact mount it claims. The signature proves who wrote +// the record; the group document proves that writer may supply this group. +func (m *Manager) credentialAuthorized(r *credRecord) bool { + return credentialAuthorizedByGroups(m.state.groupList(), r) +} + +// ---- anti-entropy reply ---- + +// buildReply answers a peer's digests / wants, telling it what we hold that it +// lacks and requesting what it holds that we lack. +func (m *Manager) buildReply(in *syncMessage) *syncMessage { + reply := &syncMessage{Type: "reply"} + + // Groups: if ours is newer, offer it. + local := m.state.groupDoc() + if local.Version > in.GroupsVer { + reply.Groups = &local + } + + // Credential anti-entropy is per (group, origin, mount) candidate. A group-level + // LWW exchange would silently promote one member into a primary and discard + // the independent recovery paths held by the other members. + peerHas := make(map[string]credDigest, len(in.CredDigests)) + wanted := make(map[string]struct{}) + offered := make(map[string]struct{}) + appendWant := func(groupID string) { + if groupID == "" { + return + } + if _, exists := wanted[groupID]; exists { + return + } + wanted[groupID] = struct{}{} + reply.Wants = append(reply.Wants, groupID) + } + appendCred := func(r *credRecord) { + if r == nil || !m.canOfferCred(r) { + return + } + key := credDigestKey(digestOfCred(r)) + if _, exists := offered[key]; exists { + return + } + offered[key] = struct{}{} + reply.Creds = append(reply.Creds, r) + } + for _, dg := range in.CredDigests { + if dg.GroupID == "" || dg.Origin == "" { + continue + } + key := credDigestKey(dg) + peerHas[key] = dg + cur, ok := m.state.getCredForSource(dg.GroupID, dg.Origin, dg.OriginMount) + if !ok { + appendWant(dg.GroupID) + continue + } + mine := digestOfCred(cur) + switch { + case credDigestDominates(mine, dg): + if m.canOfferCred(cur) { + appendCred(cur) + } else { + appendWant(dg.GroupID) + } + case credDigestDominates(dg, mine): + appendWant(dg.GroupID) + } + } + // Records we hold the peer never mentioned. + for _, cur := range m.state.credSnapshot() { + if _, seen := peerHas[credDigestKey(digestOfCred(cur))]; !seen { + appendCred(cur) + } + } + + // An explicit pull requests every locally-proven candidate in the group. + for _, gid := range in.Wants { + for _, cur := range m.state.credsForGroup(gid) { + appendCred(cur) + } + } + reply.Revocations = m.state.revocationSnapshot() + + if reply.Groups == nil && len(reply.Creds) == 0 && len(reply.Revocations) == 0 && len(reply.Wants) == 0 { + return nil + } + return reply +} + +// ---- networking ---- + +// seal wraps a message in an encrypted envelope addressed from this node. +func (m *Manager) seal(msg *syncMessage) ([]byte, error) { + key, err := deriveAEADKey([]byte(m.cfgStore.get().Key)) + if err != nil { + return nil, err + } + return sealEnvelope(key, m.id, msg, now()) +} + +// sendTo enqueues a sealed message on a single connection. +func (m *Manager) sendTo(c *peerConn, msg *syncMessage) { + frame, err := m.seal(msg) + if err != nil { + return + } + c.enqueue(frame) +} + +// broadcast enqueues a sealed message on every live connection. +func (m *Manager) broadcast(msg *syncMessage) { + conns := m.conns.all() + if len(conns) == 0 { + return + } + frame, err := m.seal(msg) + if err != nil { + return + } + for _, c := range conns { + c.enqueue(frame) + } +} + +// relayMsg forwards newly-merged info to every connection except the one it +// arrived on — this is what lets two NAT'd nodes converge through a common +// reachable peer. Signed records/groups stay authentic across the relay. +func (m *Manager) relayMsg(msg *syncMessage, except *peerConn) { + conns := m.conns.all() + if len(conns) <= 1 { + return + } + frame, err := m.seal(msg) + if err != nil { + return + } + for _, c := range conns { + if c == except { + continue + } + c.enqueue(frame) + } +} + +// handleFrame authenticates and processes one inbound frame. A frame that fails +// to open (wrong key / replay / stale) drops the connection. +func (m *Manager) handleFrame(c *peerConn, data []byte) { + cfg := m.cfgStore.get() + if !cfg.active() { + c.close() + return + } + key, err := deriveAEADKey([]byte(cfg.Key)) + if err != nil { + c.close() + return + } + env, msg, err := openEnvelope(key, data, now(), m.replay, replayWindowSec) + if err != nil { + utils.Log.Debugf("[cluster] frame rejected from %s: %v", c.nodeID, err) + c.close() + return + } + if env.Sender == m.id.NodeID { + c.close() // connected to ourselves + return + } + if c.nodeID == "" { + m.conns.bind(c, env.Sender) + } + + relay := &syncMessage{Type: "push"} + relayHas := false + + // Inventory + PEX. + var invs []*nodeInfo + if msg.Node != nil { + invs = append(invs, msg.Node) + } + invs = append(invs, msg.Nodes...) + if merged := m.absorbInventory(invs); len(merged) > 0 { + relay.Nodes = merged + relayHas = true + } + // Groups. + if m.absorbGroups(msg.Groups) { + gd := m.state.groupDoc() + relay.Groups = &gd + relayHas = true + } + // Candidate tombstones must converge before credentials: otherwise a relay + // can briefly resurrect an old signed token while a revocation is in flight. + if revoked := m.absorbRevocations(msg.Revocations); len(revoked) > 0 { + relay.Revocations = revoked + relayHas = true + } + // Credentials. + if merged := m.absorbCreds(msg.Creds); len(merged) > 0 { + // A relay must not amplify a peer credential merely because its signature + // was valid. Only candidates that this node has actually proven locally + // may leave this node again. + for _, r := range merged { + if m.canOfferCred(r) { + relay.Creds = append(relay.Creds, r) + } + } + if len(relay.Creds) > 0 { + relayHas = true + } + } + if relayHas { + m.relayMsg(relay, c) + } + + // Anti-entropy reply only for digest/want-bearing messages (avoids echo). + switch msg.Type { + case "hello", "announce", "pull": + if reply := m.buildReply(msg); reply != nil { + m.sendTo(c, reply) + } + } +} + +// ---- events ---- + +func (m *Manager) recordEvent(kind, groupID, detail string) { + m.evMu.Lock() + defer m.evMu.Unlock() + m.events = append(m.events, EventView{Time: now(), Kind: kind, GroupID: groupID, Detail: detail}) + if len(m.events) > maxEvents { + m.events = m.events[len(m.events)-maxEvents:] + } +} + +func (m *Manager) eventList() []EventView { + m.evMu.Lock() + defer m.evMu.Unlock() + out := make([]EventView, len(m.events)) + copy(out, m.events) + // newest first + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out +} + +func shortNode(id string) string { + if len(id) <= 8 { + return id + } + return id[:8] +} diff --git a/internal/cluster/identity.go b/internal/cluster/identity.go new file mode 100644 index 0000000000..add3f4aa8c --- /dev/null +++ b/internal/cluster/identity.go @@ -0,0 +1,30 @@ +package cluster + +import ( + "crypto/ed25519" + "os" + "path/filepath" +) + +// loadOrCreateIdentity returns this node's stable ed25519 identity, generating +// and persisting a fresh one (the 32-byte seed) on first run. The seed file is +// 0600 — it is the node's private key and the basis of its node id, which other +// nodes pin (a node id is the hash of the public key). +func loadOrCreateIdentity(dir string) (*identity, error) { + path := filepath.Join(dir, "identity.key") + seed, err := os.ReadFile(path) + if err == nil && len(seed) == ed25519.SeedSize { + return identityFromSeed(seed) + } + id, err := newIdentity() + if err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(path, id.seed(), 0o600); err != nil { + return nil, err + } + return id, nil +} diff --git a/internal/cluster/state.go b/internal/cluster/state.go new file mode 100644 index 0000000000..becaa150ae --- /dev/null +++ b/internal/cluster/state.go @@ -0,0 +1,1169 @@ +package cluster + +import ( + "crypto/sha256" + "encoding/json" + "sort" + "sync" +) + +// ---------------------------------------------------------------------------- +// Sync groups (the multipartite mapping) +// +// A group links storages ACROSS nodes that should share one credential — e.g. +// the same 115 account mounted on cfscan, tx and home-nas. The set of groups is +// cluster-wide shared state, replicated as a single last-writer-wins document +// (admin edits are infrequent, so whole-doc LWW is the simplest convergent +// choice). Each group is a connected component of the overall k-partite graph. +// ---------------------------------------------------------------------------- + +// member is one storage on one node. +type member struct { + NodeID string `json:"node_id"` + MountPath string `json:"mount_path"` +} + +type group struct { + ID string `json:"id"` + Name string `json:"name"` + Members []member `json:"members"` +} + +// mountsForNode returns the mount paths this node contributes to the group. +func (g group) mountsForNode(nodeID string) []string { + var out []string + for _, m := range g.Members { + if m.NodeID == nodeID { + out = append(out, m.MountPath) + } + } + return out +} + +// groupDoc is the replicated set of groups plus LWW version metadata. +type groupDoc struct { + Groups []group `json:"groups"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + OriginPub []byte `json:"origin_pub"` + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` +} + +// canonicalGroups returns the groups sorted deterministically so signing and +// comparison are stable regardless of insertion order. +func canonicalGroups(groups []group) []group { + out := make([]group, len(groups)) + copy(out, groups) + for i := range out { + ms := make([]member, len(out[i].Members)) + copy(ms, out[i].Members) + sort.Slice(ms, func(a, b int) bool { + if ms[a].NodeID != ms[b].NodeID { + return ms[a].NodeID < ms[b].NodeID + } + return ms[a].MountPath < ms[b].MountPath + }) + out[i].Members = ms + } + sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) + return out +} + +func (d *groupDoc) signingBytes() []byte { + canon := struct { + Groups []group `json:"groups"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + }{canonicalGroups(d.Groups), d.Version, d.Origin} + b, _ := json.Marshal(canon) + sum := sha256.Sum256(b) + return sum[:] +} + +func (d *groupDoc) verify() bool { + if d == nil { + return false + } + if d.Version == 0 { + // Version zero is reserved for the exact local bootstrap document. A + // non-empty v0 document has no authenticated ordering and must not win a + // tie-break against a fresh node's empty state. + return len(d.Groups) == 0 && d.Origin == "" && len(d.OriginPub) == 0 && len(d.Sig) == 0 && d.UpdatedAt == 0 + } + if nodeIDFromPub(d.OriginPub) != d.Origin { + return false + } + return verifySig(d.OriginPub, d.signingBytes(), d.Sig) +} + +func (d *groupDoc) dominates(other *groupDoc) bool { + if d.Version != other.Version { + return d.Version > other.Version + } + return d.Origin > other.Origin +} + +// ---------------------------------------------------------------------------- +// Credential records (the thing actually replicated per group) +// ---------------------------------------------------------------------------- + +// credRecord carries a group's current credential payload. The payload holds the +// secret fields and only ever travels inside the AEAD-sealed envelope. Records +// are signed by their origin so they stay authentic across relays. +type credRecord struct { + GroupID string `json:"group_id"` + OriginDriver string `json:"origin_driver"` // apply only to same-driver mounts + Fields []string `json:"fields"` // credential field names included + CredHash string `json:"cred_hash"` // idempotency key + Payload map[string]json.RawMessage `json:"payload"` // field -> value (secret) + Version uint64 `json:"version"` // Lamport clock + Origin string `json:"origin"` // authoring node id + OriginPub []byte `json:"origin_pub"` // origin ed25519 pubkey + OriginMount string `json:"origin_mount"` // authoring mount path + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` // v1-compatible record signature + MountSig []byte `json:"mount_sig,omitempty"` // binds OriginMount for upgraded peers +} + +// signingBytes deliberately preserves the original credential signature format. +// Older nodes ignore MountSig and can therefore keep verifying records from an +// upgraded node during a rolling deployment. +func (r *credRecord) signingBytes() []byte { + return r.credentialSigningBytes(false) +} + +// v2SigningBytes was briefly emitted by 5a61c4b5. It is accepted on read so a +// state.json written by that revision is not silently discarded. +func (r *credRecord) v2SigningBytes() []byte { + return r.credentialSigningBytes(true) +} + +func (r *credRecord) credentialSigningBytes(bindMount bool) []byte { + var b []byte + b = append(b, r.GroupID...) + b = append(b, 0) + b = append(b, r.OriginDriver...) + b = append(b, 0) + if bindMount { + b = append(b, r.OriginMount...) + b = append(b, 0) + } + b = append(b, r.CredHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + return b +} + +func (r *credRecord) mountSigningBytes() []byte { + b := append([]byte("openlist/cluster/credential-mount/v1\x00"), r.signingBytes()...) + b = append(b, 0) + b = append(b, r.OriginMount...) + return b +} + +type credentialSignature uint8 + +const ( + credentialSignatureInvalid credentialSignature = iota + credentialSignatureLegacy + credentialSignatureMountBound + credentialSignatureV2 +) + +func (r *credRecord) signatureKind() credentialSignature { + if r == nil || nodeIDFromPub(r.OriginPub) != r.Origin || credHash(r.Payload) != r.CredHash { + return credentialSignatureInvalid + } + if verifySig(r.OriginPub, r.signingBytes(), r.Sig) { + if len(r.MountSig) == 0 { + return credentialSignatureLegacy + } + if verifySig(r.OriginPub, r.mountSigningBytes(), r.MountSig) { + return credentialSignatureMountBound + } + return credentialSignatureInvalid + } + if len(r.MountSig) == 0 && verifySig(r.OriginPub, r.v2SigningBytes(), r.Sig) { + return credentialSignatureV2 + } + return credentialSignatureInvalid +} + +func (r *credRecord) verify() bool { + return r.signatureKind() != credentialSignatureInvalid +} + +func (r *credRecord) dominates(other *credRecord) bool { + if r.Version != other.Version { + return r.Version > other.Version + } + if r.Origin != other.Origin { + return r.Origin > other.Origin + } + return r.CredHash > other.CredHash +} + +// credRevocation is a signed tombstone for one source mount's credential +// candidate. It is intentionally separate from a credential record: a relay +// must retain and forward the tombstone after it has discarded the secret, or +// an old signed credential can be replayed back to its source. +type credRevocation struct { + GroupID string `json:"group_id"` + Origin string `json:"origin"` + OriginPub []byte `json:"origin_pub"` + OriginMount string `json:"origin_mount"` + CredHash string `json:"cred_hash"` + Version uint64 `json:"version"` + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` +} + +func (r *credRevocation) signingBytes() []byte { + var b []byte + b = append(b, "openlist/cluster/credential-revocation/v1\x00"...) + b = append(b, r.GroupID...) + b = append(b, 0) + b = append(b, r.OriginMount...) + b = append(b, 0) + b = append(b, r.CredHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + return b +} + +func (r *credRevocation) verify() bool { + return r != nil && r.GroupID != "" && r.OriginMount != "" && r.CredHash != "" && + r.Version > 0 && nodeIDFromPub(r.OriginPub) == r.Origin && + verifySig(r.OriginPub, r.signingBytes(), r.Sig) +} + +func (r *credRevocation) dominates(other *credRevocation) bool { + if other == nil || r.Version != other.Version { + return other == nil || r.Version > other.Version + } + return r.CredHash > other.CredHash +} + +// credDigest is the compact (no-secret) advert of a held credential record. +type credDigest struct { + GroupID string `json:"group_id"` + CredHash string `json:"cred_hash"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + OriginMount string `json:"origin_mount,omitempty"` +} + +func digestOfCred(r *credRecord) credDigest { + return credDigest{GroupID: r.GroupID, CredHash: r.CredHash, Version: r.Version, Origin: r.Origin, OriginMount: r.OriginMount} +} + +func candidateKey(origin, mount string) string { return origin + "\x00" + mount } + +// credDigestKey identifies one mount-level candidate within a group. +func credDigestKey(d credDigest) string { + return d.GroupID + "\x00" + candidateKey(d.Origin, d.OriginMount) +} + +func credDigestDominates(a, b credDigest) bool { + if a.Version != b.Version { + return a.Version > b.Version + } + if a.Origin != b.Origin { + return a.Origin > b.Origin + } + if a.OriginMount != b.OriginMount { + return a.OriginMount > b.OriginMount + } + return a.CredHash > b.CredHash +} + +// ---------------------------------------------------------------------------- +// Node inventory (soft state powering the UI + group editing) +// ---------------------------------------------------------------------------- + +type storageInfo struct { + MountPath string `json:"mount_path"` + Driver string `json:"driver"` + Status string `json:"status"` +} + +type nodeInfo struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Storages []storageInfo `json:"storages"` + Version uint64 `json:"version"` + UpdatedAt int64 `json:"updated_at"` + // seenAt is set locally on receipt for liveness; not part of the wire form's + // trust (it is overwritten each time we hear from/about the node). + seenAt int64 `json:"-"` +} + +// ---------------------------------------------------------------------------- +// store: the in-memory cluster state (groups + creds + inventory) +// ---------------------------------------------------------------------------- + +type store struct { + mu sync.RWMutex + groups groupDoc + // One group is a peer set, not a primary/replica pair. Each member keeps its + // own current candidate, keyed by the signing origin and mount; Lamport + // ordering only resolves successive credentials from the same source mount. + creds map[string]map[string]*credRecord // group id -> (origin, mount) -> candidate + // revocations are source-signed tombstones. They prevent a relay from + // reintroducing a credential after that source proved it invalid. + revocations map[string]map[string]*credRevocation // group id -> (origin, mount) -> tombstone + // failedCandidates is node-local only: a candidate may be valid on its origin + // yet unusable from this node. Persisting the quarantine avoids a restart + // repeatedly refreshing the exact token that just failed here. + failedCandidates map[string]map[string]map[string]int64 // group id -> local mount -> credential hash -> failed unix second + // candidateHealth is an in-memory proof that this node has actually used a + // credential successfully. It is deliberately not persisted: a restart must + // prove the credential again instead of trusting a stale WORK status. + candidateHealth map[string]map[string]int64 // group id -> credential hash -> last proven unix second + inventory map[string]*nodeInfo // keyed by node id + lamport uint64 +} + +const candidateLeaseSec int64 = 15 * 60 + +func newStore() *store { + return &store{ + creds: make(map[string]map[string]*credRecord), + revocations: make(map[string]map[string]*credRevocation), + failedCandidates: make(map[string]map[string]map[string]int64), + candidateHealth: make(map[string]map[string]int64), + inventory: make(map[string]*nodeInfo), + } +} + +func (s *store) observe(v uint64) { + if v > s.lamport { + s.lamport = v + } +} + +// ---- groups ---- + +func (s *store) groupDoc() groupDoc { + s.mu.RLock() + defer s.mu.RUnlock() + return s.groups +} + +func (s *store) groupList() []group { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]group, len(s.groups.Groups)) + copy(out, s.groups.Groups) + return out +} + +// setGroups authors a new groups document locally (admin edit). Returns the +// signed doc to broadcast. +func (s *store) setGroups(id *identity, groups []group, now int64) groupDoc { + s.mu.Lock() + defer s.mu.Unlock() + s.lamport++ + d := groupDoc{ + Groups: canonicalGroups(groups), + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + UpdatedAt: now, + } + d.Sig = id.sign(d.signingBytes()) + s.groups = d + return d +} + +// mergeGroups integrates a peer's groups document under LWW. Returns true if it +// replaced ours. +func (s *store) mergeGroups(d *groupDoc) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.observe(d.Version) + if d.dominates(&s.groups) { + s.groups = *d + return true + } + return false +} + +// groupsForMount returns the groups that include a (this-node) mount path. +func (s *store) groupsForMount(nodeID, mountPath string) []group { + s.mu.RLock() + defer s.mu.RUnlock() + var out []group + for _, g := range s.groups.Groups { + for _, m := range g.Members { + if m.NodeID == nodeID && m.MountPath == mountPath { + out = append(out, g) + break + } + } + } + return out +} + +func (s *store) groupByID(id string) (group, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, g := range s.groups.Groups { + if g.ID == id { + return g, true + } + } + return group{}, false +} + +// credentialAuthorizedByGroups binds a signed credential to a configured source +// mount. Legacy signatures did not bind OriginMount, so they are safe only when +// that node has exactly one member mount in the group; upgraded records carry a +// separate MountSig and support multiple mounts from the same node. +func credentialAuthorizedByGroups(groups []group, r *credRecord) bool { + if r == nil || r.GroupID == "" || r.Origin == "" || r.OriginMount == "" || !r.verify() { + return false + } + var membersForOrigin int + var exactMount bool + for _, g := range groups { + if g.ID != r.GroupID { + continue + } + for _, member := range g.Members { + if member.NodeID != r.Origin { + continue + } + membersForOrigin++ + if member.MountPath == r.OriginMount { + exactMount = true + } + } + } + if !exactMount { + return false + } + return r.signatureKind() != credentialSignatureLegacy || membersForOrigin == 1 +} + +func credentialRevocationAuthorizedByGroups(groups []group, rev *credRevocation) bool { + if !rev.verify() { + return false + } + for _, g := range groups { + if g.ID != rev.GroupID { + continue + } + for _, member := range g.Members { + if member.NodeID == rev.Origin && member.MountPath == rev.OriginMount { + return true + } + } + } + return false +} + +// ---- creds ---- + +func (s *store) getCred(groupID string) (*credRecord, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + var newest *credRecord + for _, r := range s.creds[groupID] { + if newest == nil || r.dominates(newest) { + newest = r + } + } + return newest, newest != nil +} + +func (s *store) getCredForSource(groupID, origin, mount string) (*credRecord, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.creds[groupID][candidateKey(origin, mount)] + return r, ok +} + +func (s *store) credsForGroup(groupID string) []*credRecord { + s.mu.RLock() + defer s.mu.RUnlock() + byOrigin := s.creds[groupID] + out := make([]*credRecord, 0, len(byOrigin)) + for _, r := range byOrigin { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) + return out +} + +func (s *store) hasCredHash(groupID, hash string) bool { + if hash == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + for _, r := range s.creds[groupID] { + if r.CredHash == hash { + return true + } + } + return false +} + +func (s *store) hasCredForSource(groupID, origin, mount, hash string) bool { + if hash == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.creds[groupID][candidateKey(origin, mount)] + return ok && r.CredHash == hash +} + +func (s *store) credSnapshot() []*credRecord { + s.mu.RLock() + defer s.mu.RUnlock() + var out []*credRecord + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + out = append(out, r) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID + } + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) + return out +} + +func (s *store) credDigests() []credDigest { + s.mu.RLock() + defer s.mu.RUnlock() + var out []credDigest + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + out = append(out, digestOfCred(r)) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID + } + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) + return out +} + +// localCredChange records a credential a node observed on one of its own mounts. +// Returns (nil,false) when the credential is unchanged (idempotent: "其他节点看到 +// 一样的 token 就不用变") or empty. +func (s *store) localCredChange(id *identity, groupID, driver, mount string, payload map[string]json.RawMessage, now int64) (*credRecord, bool) { + h := credHash(payload) + if h == "" { + return nil, false + } + s.mu.Lock() + defer s.mu.Unlock() + byOrigin := s.creds[groupID] + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[groupID] = byOrigin + } + if s.pairRevokedLocked(groupID, h) { + return nil, false + } + key := candidateKey(id.NodeID, mount) + // Pair identity is group-wide. A node that adopted an existing peer pair + // must keep that pair's original provenance; re-signing it under another + // origin would let one 401 tombstone leave duplicate live copies behind. + for existingKey, existing := range byOrigin { + if existingKey == key && existing.CredHash == h && existing.signatureKind() != credentialSignatureMountBound { + // Preserve the rolling-compatible upgrade path for a record that this + // exact source had already authored under a legacy signature. + continue + } + if existing.CredHash == h { + return nil, false + } + } + if cur, ok := byOrigin[key]; ok && cur.CredHash == h && cur.signatureKind() == credentialSignatureMountBound { + return nil, false // this source mount has not changed + } + s.lamport++ + r := &credRecord{ + GroupID: groupID, + OriginDriver: driver, + Fields: credFieldNames(payload), + CredHash: h, + Payload: payload, + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + UpdatedAt: now, + } + r.Sig = id.sign(r.signingBytes()) + r.MountSig = id.sign(r.mountSigningBytes()) + byOrigin[key] = r + return r, true +} + +// mergeCred integrates a peer's credential record under LWW. The caller must have +// verified the signature. Returns true if it replaced/added ours. +func (s *store) mergeCred(r *credRecord) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.observe(r.Version) + if s.pairRevokedLocked(r.GroupID, r.CredHash) { + return false // a provider-auth tombstone is terminal for this complete pair + } + byOrigin := s.creds[r.GroupID] + for _, existing := range byOrigin { + if existing.CredHash == r.CredHash { + // A signature proves who observed the pair, not that this observer is + // entitled to mint a second authority record for it. + return false + } + } + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[r.GroupID] = byOrigin + } + key := candidateKey(r.Origin, r.OriginMount) + cur, ok := byOrigin[key] + if ok { + if cur.CredHash == r.CredHash { + return false // idempotent: same credential + } + if !r.dominates(cur) { + return false + } + } + cp := *r + byOrigin[key] = &cp + return true +} + +// dropOwnCred removes this source mount's local candidate without revoking it. +// It is used for uncertain startup failures (for example a transient TLS +// timeout), where this node must stop advertising the candidate but must not +// globally invalidate a credential that may still work on its origin. +func (s *store) dropOwnCred(groupID, selfID, mount string) bool { + s.mu.Lock() + defer s.mu.Unlock() + byOrigin := s.creds[groupID] + if byOrigin == nil { + return false + } + key := candidateKey(selfID, mount) + if _, ok := byOrigin[key]; !ok { + return false + } + delete(byOrigin, key) + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + return true +} + +// revokeOwnCred removes a source candidate only after the provider has +// explicitly declared it invalid. Unlike dropOwnCred, this emits a signed +// tombstone so relays cannot replay the candidate back to its origin. +func (s *store) revokeOwnCred(id *identity, groupID, mount string, at int64) (*credRevocation, bool) { + if id == nil || at <= 0 { + return nil, false + } + s.mu.Lock() + defer s.mu.Unlock() + byOrigin := s.creds[groupID] + if byOrigin == nil { + return nil, false + } + key := candidateKey(id.NodeID, mount) + current := byOrigin[key] + if current == nil { + return nil, false + } + s.lamport++ + rev := &credRevocation{ + GroupID: groupID, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + CredHash: current.CredHash, + Version: s.lamport, + UpdatedAt: at, + } + rev.Sig = id.sign(rev.signingBytes()) + byRevocation := s.revocations[groupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[groupID] = byRevocation + } + byRevocation[rev.CredHash] = rev + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == rev.CredHash { + delete(byOrigin, sourceKey) + } + } + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + cp := *rev + return &cp, true +} + +// revokePair records a provider-auth rejection for a complete credential pair. +// Unlike revokeOwnCred, the reporter need not be the pair's original publisher: +// a 401 is about the pair itself, and must suppress every re-authored or relayed +// copy in the group. +func (s *store) revokePair(id *identity, groupID, mount, hash string, at int64) (*credRevocation, bool) { + if id == nil || groupID == "" || mount == "" || hash == "" || at <= 0 { + return nil, false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.pairRevokedLocked(groupID, hash) { + return nil, false + } + s.lamport++ + rev := &credRevocation{ + GroupID: groupID, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + CredHash: hash, + Version: s.lamport, + UpdatedAt: at, + } + rev.Sig = id.sign(rev.signingBytes()) + byRevocation := s.revocations[groupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[groupID] = byRevocation + } + byRevocation[hash] = rev + if byOrigin := s.creds[groupID]; byOrigin != nil { + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == hash { + delete(byOrigin, sourceKey) + } + } + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + } + cp := *rev + return &cp, true +} + +func (s *store) pairRevokedLocked(groupID, hash string) bool { + if hash == "" { + return false + } + _, ok := s.revocations[groupID][hash] + return ok +} + +func (s *store) revocationSnapshot() []*credRevocation { + s.mu.RLock() + defer s.mu.RUnlock() + var out []*credRevocation + for _, bySource := range s.revocations { + for _, rev := range bySource { + if !rev.verify() { + continue + } + cp := *rev + out = append(out, &cp) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID + } + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) + return out +} + +// mergeRevocation applies a verified provider-auth tombstone. A tombstone is +// keyed by complete pair hash, not by its reporting source: the same pair may +// have been observed by several nodes, but one 401 makes every copy unusable. +func (s *store) mergeRevocation(rev *credRevocation) bool { + if rev == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + s.observe(rev.Version) + byRevocation := s.revocations[rev.GroupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[rev.GroupID] = byRevocation + } + if current := byRevocation[rev.CredHash]; current != nil && !rev.dominates(current) { + return false + } + cp := *rev + byRevocation[rev.CredHash] = &cp + if byOrigin := s.creds[rev.GroupID]; byOrigin != nil { + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == rev.CredHash { + delete(byOrigin, sourceKey) + } + } + if len(byOrigin) == 0 { + delete(s.creds, rev.GroupID) + } + } + return true +} + +// pruneCreds drops credential records for groups that no longer exist. +func (s *store) pruneCreds() { + s.mu.Lock() + defer s.mu.Unlock() + live := make(map[string]struct{}, len(s.groups.Groups)) + for _, g := range s.groups.Groups { + live[g.ID] = struct{}{} + } + for id := range s.creds { + if _, ok := live[id]; !ok { + delete(s.creds, id) + delete(s.candidateHealth, id) + } + } + for id := range s.revocations { + if _, ok := live[id]; !ok { + delete(s.revocations, id) + } + } + for id := range s.failedCandidates { + if _, ok := live[id]; !ok { + delete(s.failedCandidates, id) + } + } +} + +func (s *store) markCandidateHealthy(groupID, hash string, at int64) { + if groupID == "" || hash == "" || at <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byHash := s.candidateHealth[groupID] + if byHash == nil { + byHash = make(map[string]int64) + s.candidateHealth[groupID] = byHash + } + byHash[hash] = at +} + +func (s *store) forgetCandidateHealth(groupID, hash string) { + if groupID == "" || hash == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byHash := s.candidateHealth[groupID] + delete(byHash, hash) + if len(byHash) == 0 { + delete(s.candidateHealth, groupID) + } +} + +func (s *store) candidateHealthy(groupID, hash string, at int64) bool { + if groupID == "" || hash == "" || at <= 0 { + return false + } + s.mu.RLock() + provenAt := s.candidateHealth[groupID][hash] + s.mu.RUnlock() + return provenAt > 0 && at >= provenAt && at-provenAt <= candidateLeaseSec +} + +func (s *store) markCandidateFailed(groupID, mount, hash string, at int64) { + if groupID == "" || mount == "" || hash == "" || at <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byMount := s.failedCandidates[groupID] + if byMount == nil { + byMount = make(map[string]map[string]int64) + s.failedCandidates[groupID] = byMount + } + byHash := byMount[mount] + if byHash == nil { + byHash = make(map[string]int64) + byMount[mount] = byHash + } + byHash[hash] = at +} + +func (s *store) clearCandidateFailed(groupID, mount, hash string) { + if groupID == "" || mount == "" || hash == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byMount := s.failedCandidates[groupID] + byHash := byMount[mount] + delete(byHash, hash) + if len(byHash) == 0 { + delete(byMount, mount) + } + if len(byMount) == 0 { + delete(s.failedCandidates, groupID) + } +} + +func (s *store) candidateFailed(groupID, mount, hash string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.failedCandidates[groupID][mount][hash] > 0 +} + +// recoveryCandidates returns persisted alternatives for one local mount. The +// current failed credential and any locally quarantined candidate are excluded; +// the order is deterministic so a recovery attempt is bounded and reproducible. +func (s *store) recoveryCandidates(groupID, mount, failedHash string) []*credRecord { + candidates := s.credsForGroup(groupID) + out := make([]*credRecord, 0, len(candidates)) + for _, candidate := range candidates { + if candidate.CredHash == failedHash || s.candidateFailed(groupID, mount, candidate.CredHash) { + continue + } + out = append(out, candidate) + } + return out +} + +// ---- inventory ---- + +func (s *store) setLocalInventory(info *nodeInfo) { + s.mu.Lock() + defer s.mu.Unlock() + info.seenAt = info.UpdatedAt + s.inventory[info.NodeID] = info +} + +// mergeInventory integrates a peer's inventory entry (LWW by version). seenAt is +// always refreshed so liveness reflects the latest contact. +func (s *store) mergeInventory(info *nodeInfo, now int64) bool { + s.mu.Lock() + defer s.mu.Unlock() + cur, ok := s.inventory[info.NodeID] + if ok && info.Version < cur.Version { + cur.seenAt = now // still heard about it; keep liveness fresh + return false + } + cp := *info + cp.seenAt = now + s.inventory[info.NodeID] = &cp + return true +} + +func (s *store) inventoryList() []nodeInfo { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]nodeInfo, 0, len(s.inventory)) + for _, n := range s.inventory { + cp := *n + cp.seenAt = n.seenAt + out = append(out, cp) + } + sort.Slice(out, func(i, j int) bool { return out[i].NodeID < out[j].NodeID }) + return out +} + +// dialableAddrs returns advertised peer addresses (excluding our own node id) for +// auto-discovery dialing. +func (s *store) dialableAddrs(selfID string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + var out []string + for id, n := range s.inventory { + if id == selfID || n.Addr == "" { + continue + } + out = append(out, n.Addr) + } + return out +} + +// ---- persistence ---- + +type persistedState struct { + Lamport uint64 `json:"lamport"` + Groups groupDoc `json:"groups"` + Creds []*credRecord `json:"creds"` + Revocations []*credRevocation `json:"revocations,omitempty"` + FailedCandidates []candidateFailure `json:"failed_candidates,omitempty"` + Inventory []*nodeInfo `json:"inventory"` +} + +// candidateFailure is local-only recovery metadata; it deliberately contains no +// credential payload, only the already-public credential hash. +type candidateFailure struct { + GroupID string `json:"group_id"` + Mount string `json:"mount"` + CredHash string `json:"cred_hash"` + FailedAt int64 `json:"failed_at"` +} + +func (s *store) export() persistedState { + s.mu.RLock() + defer s.mu.RUnlock() + ps := persistedState{Lamport: s.lamport, Groups: s.groups} + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + ps.Creds = append(ps.Creds, r) + } + } + for _, bySource := range s.revocations { + for _, rev := range bySource { + if rev.verify() { + ps.Revocations = append(ps.Revocations, rev) + } + } + } + for groupID, byMount := range s.failedCandidates { + for mount, byHash := range byMount { + for hash, failedAt := range byHash { + ps.FailedCandidates = append(ps.FailedCandidates, candidateFailure{ + GroupID: groupID, Mount: mount, CredHash: hash, FailedAt: failedAt, + }) + } + } + } + for _, n := range s.inventory { + ps.Inventory = append(ps.Inventory, n) + } + return ps +} + +func (s *store) load(ps persistedState) { + s.mu.Lock() + defer s.mu.Unlock() + s.lamport = ps.Lamport + if ps.Groups.verify() { + s.groups = ps.Groups + } else { + s.groups = groupDoc{} + } + s.creds = make(map[string]map[string]*credRecord) + s.revocations = make(map[string]map[string]*credRevocation) + s.candidateHealth = make(map[string]map[string]int64) + s.failedCandidates = make(map[string]map[string]map[string]int64) + for _, rev := range ps.Revocations { + if !credentialRevocationAuthorizedByGroups(s.groups.Groups, rev) { + continue + } + bySource := s.revocations[rev.GroupID] + if bySource == nil { + bySource = make(map[string]*credRevocation) + s.revocations[rev.GroupID] = bySource + } + key := candidateKey(rev.Origin, rev.OriginMount) + if current := bySource[key]; current == nil || rev.dominates(current) { + cp := *rev + bySource[key] = &cp + } + if rev.Version > s.lamport { + s.lamport = rev.Version + } + } + for _, r := range ps.Creds { + if !credentialAuthorizedByGroups(s.groups.Groups, r) { + continue + } + key := candidateKey(r.Origin, r.OriginMount) + if rev := s.revocations[r.GroupID][key]; rev != nil && rev.Version >= r.Version { + continue + } + byOrigin := s.creds[r.GroupID] + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[r.GroupID] = byOrigin + } + if cur, ok := byOrigin[key]; !ok || r.dominates(cur) { + cp := *r + byOrigin[key] = &cp + } + if r.Version > s.lamport { + s.lamport = r.Version + } + } + for _, failed := range ps.FailedCandidates { + if failed.GroupID == "" || failed.Mount == "" || failed.CredHash == "" || failed.FailedAt <= 0 { + continue + } + byMount := s.failedCandidates[failed.GroupID] + if byMount == nil { + byMount = make(map[string]map[string]int64) + s.failedCandidates[failed.GroupID] = byMount + } + byHash := byMount[failed.Mount] + if byHash == nil { + byHash = make(map[string]int64) + byMount[failed.Mount] = byHash + } + if failed.FailedAt > byHash[failed.CredHash] { + byHash[failed.CredHash] = failed.FailedAt + } + } + s.inventory = make(map[string]*nodeInfo, len(ps.Inventory)) + for _, n := range ps.Inventory { + if n == nil || n.NodeID == "" { + continue + } + cp := *n + s.inventory[n.NodeID] = &cp + } +} + +func (s *store) lamportNow() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.lamport +} + +// shortHash returns a short prefix of a hash for display. +func shortHash(h string) string { + if len(h) <= 12 { + return h + } + return h[:12] +} diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go new file mode 100644 index 0000000000..d475fa1871 --- /dev/null +++ b/internal/cluster/state_test.go @@ -0,0 +1,893 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + open115 "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +// ---- credential extraction ---- + +func TestIsCredField(t *testing.T) { + creds := []string{"refresh_token", "RefreshToken", "access_token", "cookie", "password", "client_secret", "api_key", "authorization", "session_id"} + for _, c := range creds { + if !isCredField(c) { + t.Errorf("%q should be detected as a credential field", c) + } + } + nonCreds := []string{"root_folder_id", "root_folder_path", "order_by", "region", "endpoint", "chunk_size", "device_id"} + for _, n := range nonCreds { + if isCredField(n) { + t.Errorf("%q should NOT be detected as a credential field", n) + } + } +} + +func TestExtractAndApplyCreds(t *testing.T) { + addition := `{"root_folder_id":"123","refresh_token":"OLD","access_token":"AOLD","order_by":"name"}` + creds := extractCreds(addition) + if len(creds) != 2 { + t.Fatalf("expected 2 credential fields, got %d (%v)", len(creds), creds) + } + if _, ok := creds["root_folder_id"]; ok { + t.Fatal("structural field leaked into credential set") + } + + // Apply new creds onto a peer Addition that has a DIFFERENT root folder. + peer := `{"root_folder_id":"999","refresh_token":"OLDER","access_token":"AOLDER","order_by":"size"}` + newCreds := map[string]json.RawMessage{ + "refresh_token": json.RawMessage(`"NEW"`), + "access_token": json.RawMessage(`"ANEW"`), + } + out, changed := applyCreds(peer, newCreds) + if !changed { + t.Fatal("applyCreds should report a change") + } + var m map[string]string + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + if m["refresh_token"] != "NEW" || m["access_token"] != "ANEW" { + t.Fatalf("credentials not overlaid: %v", m) + } + // node-local fields preserved. + if m["root_folder_id"] != "999" || m["order_by"] != "size" { + t.Fatalf("node-local fields must be preserved: %v", m) + } + + // Re-applying identical creds is a no-op (idempotent — no churn). + if _, changed := applyCreds(out, newCreds); changed { + t.Fatal("re-applying identical creds must not report a change") + } +} + +// A complete credential pair has one group-wide identity. Seeing the same pair +// on another member must not turn it into a newer, independently revocable +// candidate, and a provider-auth rejection must suppress every replay of it. +func TestPairIdentityPreventsReauthoringAndBlocksReplayAfterRejection(t *testing.T) { + a, _ := newIdentity() + b, _ := newIdentity() + s := newStore() + pair := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"access-A"`), + "refresh_token": json.RawMessage(`"refresh-A"`), + } + + first, ok := s.localCredChange(a, "g1", "115 Open", "/storage/115", pair, 1) + if !ok { + t.Fatal("first locally proven pair should be recorded") + } + if _, ok := s.localCredChange(b, "g1", "115 Open", "/storage/115", pair, 2); ok { + t.Fatal("an imported pair must retain its original provenance, not be re-authored by another node") + } + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("same pair must have one catalog entry, got %d", got) + } + + rev, ok := s.revokePair(a, "g1", "/storage/115", first.CredHash, 3) + if !ok { + t.Fatal("provider-auth rejection must create a pair-level tombstone") + } + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("pair tombstone must remove every copy, got %d", got) + } + if s.mergeCred(first) { + t.Fatal("a rejected pair must not be replayed by its original signer") + } + if _, ok := s.localCredChange(b, "g1", "115 Open", "/storage/115", pair, 4); ok { + t.Fatal("a rejected pair must not re-enter under another node identity") + } + if !s.mergeRevocation(rev) { + // A locally created tombstone is already present; repeated delivery must + // be a harmless no-op rather than changing catalog state. + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("replayed tombstone changed catalog to %d entries", got) + } + } +} + +// A legacy peer may already have re-authored a pair before v2 reaches every +// node. Receiving that second signature must not recreate duplicate authority. +func TestMergeCredRejectsPeerReauthoringOfExistingPair(t *testing.T) { + a, _ := newIdentity() + b, _ := newIdentity() + s := newStore() + pair := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"access-A"`), + "refresh_token": json.RawMessage(`"refresh-A"`), + } + first, ok := s.localCredChange(a, "g1", "115 Open", "/storage/115", pair, 1) + if !ok { + t.Fatal("first pair should be recorded") + } + reauthored := &credRecord{ + GroupID: first.GroupID, + OriginDriver: first.OriginDriver, + Fields: first.Fields, + CredHash: first.CredHash, + Payload: first.Payload, + Version: first.Version + 100, + Origin: b.NodeID, + OriginPub: b.Pub, + OriginMount: "/storage/115", + UpdatedAt: 2, + } + reauthored.Sig = b.sign(reauthored.signingBytes()) + reauthored.MountSig = b.sign(reauthored.mountSigningBytes()) + if s.mergeCred(reauthored) { + t.Fatal("same pair from another origin must not be accepted as a newer candidate") + } + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("re-authored pair created %d catalog records, want 1", got) + } +} + +// Storage updates are not credential-validation events. In particular, +// UpdateStorage after receiving an imported pair must not re-sign and publish it +// from the receiving node. This uses the real 115 driver type and real manager +// state; no test driver or callback mock is involved. +func TestOnlyTokenValidMayPublishStoragePair(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.setGroups(id, []group{{ + ID: "g1", + Members: []member{{NodeID: id.NodeID, MountPath: "/storage/115"}}, + }}, 1) + m := &Manager{ + dir: t.TempDir(), + id: id, + state: s, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + d := &open115.Open115{Storage: model.Storage{ + MountPath: "/storage/115", + Driver: "115 Open", + Status: op.WORK, + Addition: `{"access_token":"access-A","refresh_token":"refresh-A"}`, + }} + + m.onStorageHook("update", d) + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("ordinary update published %d credential record(s), want 0", got) + } + + m.onStorageCredential("token-valid", op.StorageCredentialEvent{Storage: d, Addition: d.GetStorage().Addition}) + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("token-valid published %d credential record(s), want 1", got) + } +} + +// An authentication result belongs to the pair sent with that request, not to +// whatever happens to be mounted when its goroutine finally runs. This uses the +// real 115 driver type and exercises the manager's generation guard without +// fabricating a driver implementation. +func TestLateCredentialEventCannotRevokeReplacementPair(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.setGroups(id, []group{{ + ID: "g1", + Members: []member{{NodeID: id.NodeID, MountPath: "/storage/115"}}, + }}, 1) + newPair := `{"access_token":"new-access","refresh_token":"new-refresh"}` + d := &open115.Open115{Storage: model.Storage{ + MountPath: "/storage/115", + Driver: "115 Open", + Status: op.WORK, + Addition: newPair, + }} + m := &Manager{ + dir: t.TempDir(), + id: id, + state: s, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + oldPair := `{"access_token":"old-access","refresh_token":"old-refresh"}` + m.onStorageCredential("token-invalid", op.StorageCredentialEvent{Storage: d, Addition: oldPair}) + if d.GetStorage().Status != op.WORK { + t.Fatalf("late old-pair 401 changed replacement status to %q", d.GetStorage().Status) + } + if got := len(s.revocationSnapshot()); got != 0 { + t.Fatalf("late old-pair 401 wrote %d revocations", got) + } +} + +// Candidate verification must be hermetic with respect to the live storage +// record. A temporary driver's successful Init can emit lifecycle callbacks, so +// its storage identity must never point at the persisted mount being recovered. +func TestProbeStorageIsEphemeralAndKeepsCandidateAddition(t *testing.T) { + live := model.Storage{ + ID: 42, + MountPath: "/storage/115", + Driver: "115 Open", + Status: "token invalid", + Addition: `{"access_token":"old","refresh_token":"old"}`, + } + candidate := `{"access_token":"new","refresh_token":"new"}` + + probe := newProbeStorage(live, candidate) + if probe.ID != 0 { + t.Fatalf("probe storage ID = %d, want 0 so callbacks cannot write the live row", probe.ID) + } + if probe.Status != op.WORK { + t.Fatalf("probe storage status = %q, want %q to suppress token-valid lifecycle writes", probe.Status, op.WORK) + } + if probe.Addition != candidate { + t.Fatalf("probe addition = %q, want candidate payload", probe.Addition) + } + if probe.MountPath != live.MountPath || probe.Driver != live.Driver { + t.Fatalf("probe changed mount identity: %#v", probe) + } +} + +func TestCredHashIdempotent(t *testing.T) { + a := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`), "access_token": json.RawMessage(`"y"`)} + b := map[string]json.RawMessage{"access_token": json.RawMessage(`"y"`), "refresh_token": json.RawMessage(`"x"`)} + if credHash(a) != credHash(b) { + t.Fatal("credHash must be order-independent") + } + if credHash(map[string]json.RawMessage{}) != "" { + t.Fatal("empty creds must hash to empty string") + } + c := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"z"`)} + if credHash(a) == credHash(c) { + t.Fatal("different creds must hash differently") + } +} + +// ---- groups doc LWW ---- + +func TestGroupDocSignAndMerge(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() + s := newStore() + + g := []group{{ID: "g1", Name: "115", Members: []member{{NodeID: id1.NodeID, MountPath: "/115"}}}} + d1 := s.setGroups(id1, g, 100) + if !d1.verify() { + t.Fatal("authored groups doc must verify") + } + + // A peer's newer doc dominates. + d2 := groupDoc{ + Groups: append(g, group{ID: "g2", Name: "ali", Members: []member{{NodeID: id2.NodeID, MountPath: "/ali"}}}), + Version: d1.Version + 5, + Origin: id2.NodeID, + OriginPub: id2.Pub, + UpdatedAt: 200, + } + d2.Sig = id2.sign(d2.signingBytes()) + if !s.mergeGroups(&d2) { + t.Fatal("newer groups doc should be adopted") + } + if len(s.groupList()) != 2 { + t.Fatalf("expected 2 groups after merge, got %d", len(s.groupList())) + } + // An older doc is ignored. + if s.mergeGroups(&d1) { + t.Fatal("older groups doc must be ignored") + } + // Tampered doc rejected. + d2.Groups[0].Name = "tampered" + if d2.verify() { + t.Fatal("tampered groups doc must fail verification") + } +} + +func TestGroupDocRejectsNonCanonicalVersionZero(t *testing.T) { + if !(&groupDoc{}).verify() { + t.Fatal("the empty bootstrap document must remain valid") + } + id, _ := newIdentity() + forged := &groupDoc{ + Groups: []group{{ID: "g1", Members: []member{{NodeID: id.NodeID, MountPath: "/115"}}}}, + Origin: id.NodeID, + OriginPub: id.Pub, + UpdatedAt: 1, + } + forged.Sig = id.sign(forged.signingBytes()) + if forged.verify() { + t.Fatal("a non-empty version-zero group document must be rejected") + } +} + +func TestGroupsForMount(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.setGroups(id, []group{ + {ID: "g1", Members: []member{{NodeID: "N1", MountPath: "/a"}, {NodeID: "N2", MountPath: "/a"}}}, + {ID: "g2", Members: []member{{NodeID: "N1", MountPath: "/b"}}}, + }, 1) + if got := s.groupsForMount("N1", "/a"); len(got) != 1 || got[0].ID != "g1" { + t.Fatalf("groupsForMount mismatch: %#v", got) + } + if got := s.groupsForMount("N3", "/a"); len(got) != 0 { + t.Fatal("unknown node should match no groups") + } +} + +// ---- credential record LWW + idempotency ---- + +func TestCredRecordMergeLWW(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() + s := newStore() + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + r1, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 1) + if !ok || !r1.verify() { + t.Fatal("first credential change should be recorded and verify") + } + // idempotent: same payload again -> no-op. + if _, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 2); ok { + t.Fatal("identical credential must be a no-op") + } + + // peer record with higher version dominates. + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + r2 := &credRecord{GroupID: "g1", OriginDriver: "115", Fields: []string{"refresh_token"}, Payload: payloadB, Version: r1.Version + 10, Origin: id2.NodeID, OriginPub: id2.Pub, CredHash: credHash(payloadB)} + r2.Sig = id2.sign(r2.signingBytes()) + if !r2.verify() { + t.Fatal("peer record should verify") + } + if !s.mergeCred(r2) { + t.Fatal("newer peer credential should win") + } + cur, _ := s.getCred("g1") + if cur.CredHash != credHash(payloadB) { + t.Fatal("store should hold the newer credential") + } + // older loses. + if s.mergeCred(r1) { + t.Fatal("older credential must be ignored") + } + // idempotent merge of identical hash. + if s.mergeCred(r2) { + t.Fatal("merging identical credential must be a no-op") + } +} + +// A group is a peer set, not a primary/replica pair. A newer credential from +// one healthy member must never erase a different member's candidate: each +// origin owns its current candidate and nodes choose what to activate locally. +func TestCredRecordMergePreservesCandidatesFromIndependentOrigins(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() + s := newStore() + + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + if _, ok := s.localCredChange(id1, "g1", "115", "/115-a", payloadA, 1); !ok { + t.Fatal("local candidate should be recorded") + } + + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + peer := &credRecord{ + GroupID: "g1", + OriginDriver: "115", + Fields: []string{"refresh_token"}, + Payload: payloadB, + Version: 99, // Must not make this origin a group-wide primary. + Origin: id2.NodeID, + OriginPub: id2.Pub, + OriginMount: "/115-b", + CredHash: credHash(payloadB), + } + peer.Sig = id2.sign(peer.signingBytes()) + if !s.mergeCred(peer) { + t.Fatal("peer candidate should merge") + } + + got := s.credSnapshot() + if len(got) != 2 { + t.Fatalf("candidate count = %d, want 2 independent origins; snapshot=%#v", len(got), got) + } + seen := map[string]string{} + for _, candidate := range got { + seen[candidate.Origin] = candidate.CredHash + } + if seen[id1.NodeID] != credHash(payloadA) || seen[id2.NodeID] != credHash(payloadB) { + t.Fatalf("candidate origins were not preserved: %#v", seen) + } +} + +func TestCandidateHealthLeaseExpires(t *testing.T) { + s := newStore() + const groupID = "g1" + const hash = "candidate-hash" + const provenAt int64 = 1_000 + + s.markCandidateHealthy(groupID, hash, provenAt) + if !s.candidateHealthy(groupID, hash, provenAt+candidateLeaseSec) { + t.Fatal("candidate must remain offerable through its health lease") + } + if s.candidateHealthy(groupID, hash, provenAt+candidateLeaseSec+1) { + t.Fatal("idle candidate must stop being offerable after its health lease") + } + + s.markCandidateHealthy(groupID, hash, provenAt) + s.forgetCandidateHealth(groupID, hash) + if s.candidateHealthy(groupID, hash, provenAt) { + t.Fatal("token-invalid must revoke the health lease immediately") + } +} + +func TestCredRecordVerifyRejectsForgery(t *testing.T) { + id, _ := newIdentity() + other, _ := newIdentity() + payload := map[string]json.RawMessage{"token": json.RawMessage(`"x"`)} + r := &credRecord{GroupID: "g", Payload: payload, CredHash: credHash(payload), Version: 1, Origin: id.NodeID, OriginPub: id.Pub} + r.Sig = id.sign(r.signingBytes()) + r.MountSig = id.sign(r.mountSigningBytes()) + // claim someone else's origin without their key. + r.Origin = other.NodeID + if r.verify() { + t.Fatal("a record claiming a foreign origin must fail verification") + } +} + +func TestCredRecordVerifyBindsOriginMount(t *testing.T) { + id, _ := newIdentity() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`)} + r := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115-a", + } + r.Sig = id.sign(r.signingBytes()) + r.MountSig = id.sign(r.mountSigningBytes()) + if !r.verify() { + t.Fatal("freshly signed record must verify") + } + r.OriginMount = "/115-b" + if r.verify() { + t.Fatal("changing origin mount must invalidate the credential signature") + } +} + +// legacyCredSigningBytes is the credential wire-signature format used before +// OriginMount was bound separately. Keeping this fixture lets an upgrade prove +// it can still read state.json written by the deployed 53bc1db protocol. +func legacyCredSigningBytes(r *credRecord) []byte { + var b []byte + b = append(b, r.GroupID...) + b = append(b, 0) + b = append(b, r.OriginDriver...) + b = append(b, 0) + b = append(b, r.CredHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + return b +} + +func TestCredRecordVerifiesLegacySignature(t *testing.T) { + id, _ := newIdentity() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"legacy"`)} + r := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 7, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115", + } + r.Sig = id.sign(legacyCredSigningBytes(r)) + if !r.verify() { + t.Fatal("a 53bc1db credential must remain verifiable after upgrade") + } +} + +func TestLocalCredChangeMigratesLegacySourceWithoutTokenChange(t *testing.T) { + id, _ := newIdentity() + s := newStore() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"legacy"`)} + legacy := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115", + } + legacy.Sig = id.sign(legacyCredSigningBytes(legacy)) + if !s.mergeCred(legacy) { + t.Fatal("legacy credential should be accepted into local state") + } + migrated, ok := s.localCredChange(id, "g", "115 Open", "/115", payload, 2) + if !ok || migrated == nil || len(migrated.MountSig) == 0 { + t.Fatal("a healthy local source must rewrite its legacy record with MountSig") + } + if migrated.signatureKind() != credentialSignatureMountBound { + t.Fatal("migrated record must use the rolling-compatible mount-bound format") + } +} + +func TestSameOriginMultipleMountCandidatesArePreserved(t *testing.T) { + id, _ := newIdentity() + s := newStore() + first := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"first"`)} + second := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"second"`)} + if _, ok := s.localCredChange(id, "g", "115 Open", "/115-a", first, 1); !ok { + t.Fatal("first local mount candidate was not recorded") + } + if _, ok := s.localCredChange(id, "g", "115 Open", "/115-b", second, 2); !ok { + t.Fatal("second local mount candidate was not recorded") + } + got := s.credsForGroup("g") + if len(got) != 2 { + t.Fatalf("candidate count = %d, want separate candidates for both mounts: %#v", len(got), got) + } + if !s.dropOwnCred("g", id.NodeID, "/115-a") { + t.Fatal("invalidating one source mount must drop that source candidate") + } + remaining := s.credsForGroup("g") + if len(remaining) != 1 || remaining[0].OriginMount != "/115-b" { + t.Fatalf("invalidating /115-a removed the wrong candidate: %#v", remaining) + } +} + +// dropOwnCred must remove only our own-authored record so a dead/stale local +// token can never out-rank (by Lamport version) a peer's genuinely-valid one and +// block recovery. A peer-authored record must survive. +func TestDropOwnCred(t *testing.T) { + id1, _ := newIdentity() // self + id2, _ := newIdentity() // peer + s := newStore() + + // nothing recorded yet -> nothing to drop. + if s.dropOwnCred("g1", id1.NodeID, "/115") { + t.Fatal("dropping a non-existent record must return false") + } + + // our own credential -> dropped, store no longer holds it. + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + if _, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 1); !ok { + t.Fatal("own credential should be recorded") + } + if !s.dropOwnCred("g1", id1.NodeID, "/115") { + t.Fatal("own credential should be dropped") + } + if _, ok := s.getCred("g1"); ok { + t.Fatal("store must not retain a dropped own credential") + } + + // a peer's credential must NOT be dropped by us. + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + r := &credRecord{GroupID: "g2", OriginDriver: "115", Fields: []string{"refresh_token"}, Payload: payloadB, Version: 5, Origin: id2.NodeID, OriginPub: id2.Pub, CredHash: credHash(payloadB)} + r.Sig = id2.sign(r.signingBytes()) + if !s.mergeCred(r) { + t.Fatal("peer credential should merge") + } + if s.dropOwnCred("g2", id1.NodeID, "/115") { + t.Fatal("a peer-authored credential must never be dropped as own") + } + if _, ok := s.getCred("g2"); !ok { + t.Fatal("peer credential must survive dropOwnCred") + } +} + +// A source that has proved one of its credentials invalid must be able to +// prevent an old signed copy held by a relay from entering its state again. +// Without this, the source drops its own candidate, sends a pull, and a hub +// immediately returns the exact same stale candidate as a peer record. +func TestDroppedOwnCredentialRejectsStaleRelayReplay(t *testing.T) { + source, _ := newIdentity() + s := newStore() + payload := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"dead-access"`), + "refresh_token": json.RawMessage(`"dead-refresh"`), + } + record, ok := s.localCredChange(source, "g1", "115 Open", "/storage/115", payload, 1) + if !ok { + t.Fatal("source candidate was not recorded") + } + if rev, ok := s.revokeOwnCred(source, "g1", "/storage/115", 2); !ok || !rev.verify() { + t.Fatal("source candidate was not revoked after invalidation") + } + + if s.mergeCred(record) { + t.Fatal("stale signed candidate from a relay was accepted after source invalidation") + } + if _, ok := s.getCredForSource("g1", source.NodeID, "/storage/115"); ok { + t.Fatal("revoked source candidate re-entered state") + } +} + +func TestRecoveryCandidatesSkipCurrentAndQuarantinedCredentials(t *testing.T) { + idA, _ := newIdentity() + idB, _ := newIdentity() + idC, _ := newIdentity() + s := newStore() + for _, tc := range []struct { + id *identity + mount string + payload map[string]json.RawMessage + }{ + {idA, "/a", map[string]json.RawMessage{"access_token": json.RawMessage(`"A"`)}}, + {idB, "/b", map[string]json.RawMessage{"access_token": json.RawMessage(`"B"`)}}, + {idC, "/c", map[string]json.RawMessage{"access_token": json.RawMessage(`"C"`)}}, + } { + if _, ok := s.localCredChange(tc.id, "g1", "115 Open", tc.mount, tc.payload, 1); !ok { + t.Fatalf("candidate %s was not recorded", tc.mount) + } + } + failedHash := credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"A"`)}) + currentHash := credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"B"`)}) + s.markCandidateFailed("g1", "/local", failedHash, 2) + + candidates := s.recoveryCandidates("g1", "/local", currentHash) + if len(candidates) != 1 || candidates[0].CredHash != credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"C"`)}) { + t.Fatalf("recovery candidates = %#v, want only remaining C candidate", candidates) + } +} + +func TestSignedRevocationRemovesRelayCopyButAllowsNewerRotation(t *testing.T) { + source, _ := newIdentity() + relay := newStore() + sourceState := newStore() + payloadA := map[string]json.RawMessage{"access_token": json.RawMessage(`"old"`)} + old, ok := sourceState.localCredChange(source, "g1", "115 Open", "/storage/115", payloadA, 1) + if !ok { + t.Fatal("source candidate was not recorded") + } + if !relay.mergeCred(old) { + t.Fatal("relay did not accept initial source candidate") + } + rev, ok := sourceState.revokeOwnCred(source, "g1", "/storage/115", 2) + if !ok || !rev.verify() { + t.Fatal("source revocation was not signed") + } + if !relay.mergeRevocation(rev) { + t.Fatal("relay did not accept source revocation") + } + if _, ok := relay.getCredForSource("g1", source.NodeID, "/storage/115"); ok { + t.Fatal("relay retained revoked candidate") + } + if relay.mergeCred(old) { + t.Fatal("relay accepted revoked candidate replay") + } + + payloadB := map[string]json.RawMessage{"access_token": json.RawMessage(`"rotated"`)} + newer, ok := sourceState.localCredChange(source, "g1", "115 Open", "/storage/115", payloadB, 3) + if !ok || newer.Version <= rev.Version { + t.Fatalf("source rotation did not advance beyond revocation: %#v", newer) + } + if !relay.mergeCred(newer) { + t.Fatal("relay rejected newer credential rotation after revocation") + } +} + +// A signed candidate is relayable even when the relay has no local health proof. +// Otherwise a hub between two NAT'd nodes black-holes a good source's pair and +// recovery cannot converge. Local health remains required only for activation. +func TestCanOfferCred(t *testing.T) { + id1, _ := newIdentity() // self + id2, _ := newIdentity() // peer + admin, _ := newIdentity() + s := newStore() + s.setGroups(admin, []group{{ + ID: "g1", + Members: []member{{NodeID: id1.NodeID, MountPath: "/local"}, {NodeID: id2.NodeID, MountPath: "/peer"}}, + }}, 1) + m := &Manager{id: id1, state: s} + + if m.canOfferCred(nil) { + t.Fatal("a nil record is never offerable") + } + + // A peer-authored signed candidate must traverse this node unchanged. + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"peer"`)} + peer := &credRecord{ + GroupID: "g1", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id2.NodeID, + OriginPub: id2.Pub, + OriginMount: "/peer", + } + peer.Sig = id2.sign(peer.signingBytes()) + if !m.canOfferCred(peer) { + t.Fatal("an authorized peer candidate must be relayed without local proof") + } + + // our own record, but no group/healthy mount -> not offerable. + own := &credRecord{GroupID: "g1", Origin: id1.NodeID} + if m.canOfferCred(own) { + t.Fatal("our own record must NOT be offered when the token is not proven healthy") + } +} + +// This is an end-to-end backend regression for the exact missed transition: +// a peer candidate may arrive after the local mount is already invalid. It uses +// the registered production Local driver, real op storage persistence, and the +// same isolated probe + UpdateStorage path used in production -- not a mock +// driver or a mocked callback. The 115-specific three-node acceptance remains +// a separate live test because it must use a real operator-provided pair. +func TestReceivedCandidateRecoversInvalidMountWithRealDriver(t *testing.T) { + dbName := fmt.Sprintf("file:cluster-recovery-%d?mode=memory&cache=shared", time.Now().UnixNano()) + database, err := gorm.Open(sqlite.Open(dbName), &gorm.Config{}) + if err != nil { + t.Fatalf("open test database: %v", err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + + oldRoot := t.TempDir() + mount := fmt.Sprintf("/cluster-recovery-%d", time.Now().UnixNano()) + storage := model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: fmt.Sprintf(`{"root_folder_path":%q}`, oldRoot), + } + id, err := op.CreateStorage(context.Background(), storage) + if err != nil { + t.Fatalf("create real local storage: %v", err) + } + t.Cleanup(func() { _ = op.DeleteStorageById(context.Background(), id) }) + + local, _ := newIdentity() + peer, _ := newIdentity() + admin, _ := newIdentity() + state := newStore() + state.setGroups(admin, []group{{ + ID: "g1", + Members: []member{ + {NodeID: local.NodeID, MountPath: mount}, + {NodeID: peer.NodeID, MountPath: "/peer"}, + }, + }}, 1) + manager := &Manager{ + dir: t.TempDir(), + id: local, + state: state, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + + d, err := op.GetStorageByMountPath(mount) + if err != nil { + t.Fatalf("get created storage: %v", err) + } + // Model the persisted outcome of a real 401 before a peer has a replacement. + d.GetStorage().SetStatus("token invalid") + payload := map[string]json.RawMessage{"access_token": json.RawMessage(`"peer-pair"`)} + candidate := &credRecord{ + GroupID: "g1", + OriginDriver: "Local", + Fields: []string{"access_token"}, + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: peer.NodeID, + OriginPub: peer.Pub, + OriginMount: "/peer", + UpdatedAt: now(), + } + candidate.Sig = peer.sign(candidate.signingBytes()) + candidate.MountSig = peer.sign(candidate.mountSigningBytes()) + if merged := manager.absorbCreds([]*credRecord{candidate}); len(merged) != 1 { + t.Fatalf("candidate merge = %#v, want one accepted record", merged) + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + current, getErr := op.GetStorageByMountPath(mount) + if getErr == nil && current.GetStorage().Status == op.WORK && state.candidateHealthy("g1", candidate.CredHash, now()) { + for _, event := range manager.eventList() { + if event.Kind == "apply" && event.GroupID == "g1" { + return + } + } + } + time.Sleep(20 * time.Millisecond) + } + current, _ := op.GetStorageByMountPath(mount) + t.Fatalf("received candidate did not activate invalid mount: %#v", current.GetStorage()) +} + +func TestAbsorbCredsRejectsUnauthorizedOriginMount(t *testing.T) { + admin, _ := newIdentity() + memberID, _ := newIdentity() + localID, _ := newIdentity() + s := newStore() + s.setGroups(admin, []group{{ + ID: "g1", + Members: []member{{NodeID: memberID.NodeID, MountPath: "/115"}}, + }}, 1) + m := &Manager{dir: t.TempDir(), id: localID, state: s, cfgStore: &configStore{cfg: Config{ApplyRemote: false}}} + + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"candidate"`)} + bad := &credRecord{ + GroupID: "g1", + OriginDriver: "115 Open", + Fields: []string{"refresh_token"}, + Payload: payload, + CredHash: credHash(payload), + Version: 2, + Origin: memberID.NodeID, + OriginPub: memberID.Pub, + OriginMount: "/not-a-group-member", + } + bad.Sig = memberID.sign(bad.signingBytes()) + if got := m.absorbCreds([]*credRecord{bad}); len(got) != 0 { + t.Fatalf("unauthorized origin mount was accepted: %#v", got) + } + if _, ok := s.getCredForSource("g1", memberID.NodeID, "/115"); ok { + t.Fatal("unauthorized credential must not enter replicated state") + } + + good := *bad + good.OriginMount = "/115" + good.Sig = memberID.sign(good.signingBytes()) + if got := m.absorbCreds([]*credRecord{&good}); len(got) != 1 { + t.Fatalf("authorized group member credential rejected: %#v", got) + } +} + +// ---- inventory ---- + +func TestInventoryMergeAndDialTargets(t *testing.T) { + s := newStore() + n1 := &nodeInfo{NodeID: "N1", Addr: "https://n1", Version: 1} + s.mergeInventory(n1, 1000) + // newer version replaces. + s.mergeInventory(&nodeInfo{NodeID: "N1", Addr: "https://n1b", Version: 2}, 1001) + // older ignored (but liveness refreshed). + s.mergeInventory(&nodeInfo{NodeID: "N1", Addr: "https://old", Version: 1}, 1002) + list := s.inventoryList() + if len(list) != 1 || list[0].Addr != "https://n1b" { + t.Fatalf("inventory LWW failed: %#v", list) + } + s.mergeInventory(&nodeInfo{NodeID: "SELF", Addr: "https://self", Version: 1}, 1003) + addrs := s.dialableAddrs("SELF") + if len(addrs) != 1 || addrs[0] != "https://n1b" { + t.Fatalf("dialableAddrs should exclude self: %#v", addrs) + } +} diff --git a/internal/cluster/status.go b/internal/cluster/status.go new file mode 100644 index 0000000000..ff8a5c607f --- /dev/null +++ b/internal/cluster/status.go @@ -0,0 +1,222 @@ +package cluster + +// This file builds the redaction-safe overview the admin UI renders: the full +// peer roster with liveness, every sync group with its members and current +// credential state, live connections, recent activity, and rollup stats. No +// secrets (credential payloads) are ever included. + +// MemberView is one storage participating in a group, annotated for display. +type MemberView struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + MountPath string `json:"mount_path"` + Online bool `json:"online"` + Present bool `json:"present"` // the node currently advertises this mount + IsOrigin bool `json:"is_origin"` // authored the group's current credential + Self bool `json:"self"` +} + +// GroupView is a sync group plus its current (no-secret) credential state. +type GroupView struct { + ID string `json:"id"` + Name string `json:"name"` + Members []MemberView `json:"members"` + Fields []string `json:"fields"` // credential field names being synced + CredHash string `json:"cred_hash"` // short, for "same/different" comparison + Version uint64 `json:"version"` // credential record version + Origin string `json:"origin"` // node id that authored the credential + UpdatedAt int64 `json:"updated_at"` // last credential change + HasCred bool `json:"has_cred"` // a credential has been observed/shared +} + +// NodeView is one cluster node with its advertised storages and liveness. +type NodeView struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Self bool `json:"self"` + Online bool `json:"online"` + LastSeen int64 `json:"last_seen"` + Storages []storageInfo `json:"storages"` +} + +// ConnView is one live peer connection. +type ConnView struct { + NodeID string `json:"node_id"` + Addr string `json:"addr"` + Outbound bool `json:"outbound"` + Since int64 `json:"since"` +} + +// EventView is one entry in the activity log. +type EventView struct { + Time int64 `json:"time"` + Kind string `json:"kind"` // share | apply | groups | error + GroupID string `json:"group_id"` + Detail string `json:"detail"` +} + +// StatsView is the rollup shown at the top of the cluster panel. +type StatsView struct { + NodesTotal int `json:"nodes_total"` + NodesOnline int `json:"nodes_online"` + GroupsTotal int `json:"groups_total"` + CredsTotal int `json:"creds_total"` + Connections int `json:"connections"` +} + +// Status is the complete admin overview of the cluster. +type Status struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Enabled bool `json:"enabled"` + Active bool `json:"active"` + Nodes []NodeView `json:"nodes"` + Groups []GroupView `json:"groups"` + Connections []ConnView `json:"connections"` + Events []EventView `json:"events"` + Stats StatsView `json:"stats"` +} + +// isOnline reports whether a node id has a live connection or was heard from +// recently. +func (m *Manager) isOnline(nodeID string, seenAt int64) bool { + if nodeID == m.id.NodeID { + return true + } + if m.conns.hasNode(nodeID) { + return true + } + return seenAt > 0 && now()-seenAt < peerLivenessSec +} + +// Status returns a redaction-safe snapshot for the admin UI. +func (m *Manager) Status() Status { + cfg := m.cfgStore.get() + + // Roster: inventory + a freshly-computed self entry. + self := m.selfNodeInfo() + byNode := map[string]nodeInfo{} + for _, n := range m.state.inventoryList() { + byNode[n.NodeID] = n + } + selfEntry := *self + selfEntry.seenAt = now() + byNode[self.NodeID] = selfEntry + + mountPresent := func(nodeID, mount string) bool { + n, ok := byNode[nodeID] + if !ok { + return false + } + for _, s := range n.Storages { + if s.MountPath == mount { + return true + } + } + return false + } + labelOf := func(nodeID string) string { + if n, ok := byNode[nodeID]; ok && n.Label != "" { + return n.Label + } + return shortNode(nodeID) + } + + nodes := make([]NodeView, 0, len(byNode)) + online := 0 + for id, n := range byNode { + isSelf := id == m.id.NodeID + on := m.isOnline(id, n.seenAt) + if on { + online++ + } + lbl := n.Label + if lbl == "" { + lbl = shortNode(id) + } + nodes = append(nodes, NodeView{ + NodeID: id, + Label: lbl, + Addr: n.Addr, + Self: isSelf, + Online: on, + LastSeen: n.seenAt, + Storages: n.Storages, + }) + } + sortNodeViews(nodes) + + // Groups + credential state. + groups := make([]GroupView, 0) + for _, g := range m.state.groupList() { + gv := GroupView{ID: g.ID, Name: g.Name} + if cr, ok := m.state.getCred(g.ID); ok { + gv.HasCred = true + gv.Fields = cr.Fields + gv.CredHash = shortHash(cr.CredHash) + gv.Version = cr.Version + gv.Origin = cr.Origin + gv.UpdatedAt = cr.UpdatedAt + } + for _, mem := range g.Members { + n := byNode[mem.NodeID] + gv.Members = append(gv.Members, MemberView{ + NodeID: mem.NodeID, + Label: labelOf(mem.NodeID), + MountPath: mem.MountPath, + Online: m.isOnline(mem.NodeID, n.seenAt), + Present: mountPresent(mem.NodeID, mem.MountPath), + IsOrigin: gv.HasCred && mem.NodeID == gv.Origin, + Self: mem.NodeID == m.id.NodeID, + }) + } + groups = append(groups, gv) + } + + // Connections. + var conns []ConnView + for _, c := range m.conns.all() { + conns = append(conns, ConnView{NodeID: c.nodeID, Addr: c.addr, Outbound: c.outbound, Since: c.since}) + } + + return Status{ + NodeID: m.id.NodeID, + Label: cfg.Label, + Addr: trimURL(cfg.Addr), + Enabled: cfg.Enabled, + Active: cfg.active(), + Nodes: nodes, + Groups: groups, + Connections: conns, + Events: m.eventList(), + Stats: StatsView{ + NodesTotal: len(nodes), + NodesOnline: online, + GroupsTotal: len(groups), + CredsTotal: len(m.state.credSnapshot()), + Connections: len(conns), + }, + } +} + +func sortNodeViews(ns []NodeView) { + // self first, then by label/id. + for i := 1; i < len(ns); i++ { + for j := i; j > 0; j-- { + a, b := ns[j-1], ns[j] + less := false + if b.Self && !a.Self { + less = true + } else if a.Self == b.Self { + less = b.Label < a.Label + } + if less { + ns[j-1], ns[j] = ns[j], ns[j-1] + } else { + break + } + } + } +} diff --git a/internal/cluster/transport.go b/internal/cluster/transport.go new file mode 100644 index 0000000000..3c6a695ef4 --- /dev/null +++ b/internal/cluster/transport.go @@ -0,0 +1,154 @@ +package cluster + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sync" +) + +// syncMessage is the inner (encrypted) protocol payload exchanged between nodes. +// A single struct carries every interaction (hello, inventory/PEX, group-doc +// sync, credential push/pull, and anti-entropy) so one frame format covers them +// all. Unused fields are omitted on the wire. +// +// - Node: the sender's own inventory entry (who am I, my storages, addr). +// - Nodes: relayed inventory entries + advertised peer addresses (PEX). +// - Groups: the shared sync-group document (LWW). +// - GroupsVer: the sender's groups version, for cheap anti-entropy. +// - Creds: credential records being offered (push, or reply to a Want). +// - Revocations: source-signed tombstones for invalidated credential records. +// - CredDigests: compact, no-secret advert of the credential records the sender +// holds, so the receiver can detect what it is missing. +// - Wants: group ids the sender wants the receiver to send creds for. +type syncMessage struct { + Type string `json:"type"` // "hello" | "push" | "announce" | "pull" | "reply" + Node *nodeInfo `json:"node,omitempty"` + Nodes []*nodeInfo `json:"nodes,omitempty"` + Groups *groupDoc `json:"groups,omitempty"` + GroupsVer uint64 `json:"groups_ver,omitempty"` + Creds []*credRecord `json:"creds,omitempty"` + Revocations []*credRevocation `json:"revocations,omitempty"` + CredDigests []credDigest `json:"cred_digests,omitempty"` + Wants []string `json:"wants,omitempty"` +} + +// envelope is the outer, on-the-wire structure. Its Cipher is the syncMessage +// sealed with the cluster key; only PSK holders can open it (membership auth + +// confidentiality for the secrets inside). Sender/PubKey/Timestamp/Nonce are +// authenticated as AEAD additional data so they cannot be tampered with. +type envelope struct { + Sender string `json:"sender"` // sender node id (== hash(PubKey)) + PubKey []byte `json:"pubkey"` // sender ed25519 pubkey + Timestamp int64 `json:"ts"` // unix seconds (replay window) + Nonce string `json:"nonce"` // base64 random, replay cache key + Cipher []byte `json:"cipher"` // sealed syncMessage +} + +// envelopeAAD binds the cleartext header fields to the ciphertext. +func envelopeAAD(sender string, pub []byte, ts int64, nonce string) []byte { + aad := make([]byte, 0, len(sender)+len(pub)+len(nonce)+16) + aad = append(aad, sender...) + aad = append(aad, 0) + aad = append(aad, pub...) + aad = append(aad, 0) + aad = append(aad, nonce...) + aad = append(aad, 0) + t := make([]byte, 8) + for i := 0; i < 8; i++ { + t[i] = byte(ts >> (8 * uint(i))) + } + aad = append(aad, t...) + return aad +} + +// sealEnvelope builds an encrypted envelope around a syncMessage. +func sealEnvelope(key []byte, id *identity, msg *syncMessage, now int64) ([]byte, error) { + plain, err := json.Marshal(msg) + if err != nil { + return nil, err + } + nb := make([]byte, 18) + if _, err := rand.Read(nb); err != nil { + return nil, err + } + nonce := base64.RawStdEncoding.EncodeToString(nb) + aad := envelopeAAD(id.NodeID, id.Pub, now, nonce) + cipher, err := seal(key, plain, aad) + if err != nil { + return nil, err + } + return json.Marshal(envelope{ + Sender: id.NodeID, + PubKey: id.Pub, + Timestamp: now, + Nonce: nonce, + Cipher: cipher, + }) +} + +// openEnvelope authenticates and decrypts an envelope. It enforces: +// - sender node id == hash(pubkey) (self-consistent identity), +// - timestamp within replayWindow of now, +// - nonce not seen before (replay cache), +// - AEAD open with the cluster key (membership + integrity + confidentiality). +func openEnvelope(key []byte, raw []byte, now int64, rc *replayCache, replayWindow int64) (*envelope, *syncMessage, error) { + var env envelope + if err := json.Unmarshal(raw, &env); err != nil { + return nil, nil, err + } + if nodeIDFromPub(env.PubKey) != env.Sender { + return nil, nil, errors.New("sender id does not match pubkey") + } + if d := now - env.Timestamp; d > replayWindow || d < -replayWindow { + return nil, nil, fmt.Errorf("timestamp outside replay window (%ds)", d) + } + if !rc.checkAndAdd(env.Nonce, now) { + return nil, nil, errors.New("replayed nonce") + } + aad := envelopeAAD(env.Sender, env.PubKey, env.Timestamp, env.Nonce) + plain, err := open(key, env.Cipher, aad) + if err != nil { + return nil, nil, fmt.Errorf("decrypt failed (wrong cluster key or tampered): %w", err) + } + var msg syncMessage + if err := json.Unmarshal(plain, &msg); err != nil { + return nil, nil, err + } + return &env, &msg, nil +} + +// replayCache remembers recently-seen nonces to reject replays. Entries expire +// after the replay window so memory stays bounded. +type replayCache struct { + mu sync.Mutex + seen map[string]int64 + window int64 +} + +func newReplayCache(window int64) *replayCache { + return &replayCache{seen: make(map[string]int64), window: window} +} + +// checkAndAdd returns false if the nonce was already seen (a replay); otherwise +// it records the nonce and returns true. It opportunistically evicts expired +// entries. +func (rc *replayCache) checkAndAdd(nonce string, now int64) bool { + rc.mu.Lock() + defer rc.mu.Unlock() + if _, ok := rc.seen[nonce]; ok { + return false + } + // Evict stale entries (cheap amortized sweep). + if len(rc.seen) > 0 { + for k, t := range rc.seen { + if now-t > rc.window { + delete(rc.seen, k) + } + } + } + rc.seen[nonce] = now + return true +} diff --git a/internal/cluster/transport_test.go b/internal/cluster/transport_test.go new file mode 100644 index 0000000000..2ebb7c13df --- /dev/null +++ b/internal/cluster/transport_test.go @@ -0,0 +1,62 @@ +package cluster + +import "testing" + +func TestEnvelopeSealOpenRoundTrip(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + msg := &syncMessage{Type: "push", Wants: []string{"/a", "/b"}} + + raw, err := sealEnvelope(key, id, msg, 1000) + if err != nil { + t.Fatal(err) + } + env, got, err := openEnvelope(key, raw, 1000, rc, replayWindowSec) + if err != nil { + t.Fatalf("open envelope failed: %v", err) + } + if env.Sender != id.NodeID { + t.Fatal("sender id mismatch") + } + if got.Type != "push" || len(got.Wants) != 2 { + t.Fatalf("payload not preserved: %+v", got) + } +} + +func TestEnvelopeReplayRejected(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(key, id, &syncMessage{Type: "announce"}, 1000) + + if _, _, err := openEnvelope(key, raw, 1000, rc, replayWindowSec); err != nil { + t.Fatalf("first open should succeed: %v", err) + } + if _, _, err := openEnvelope(key, raw, 1001, rc, replayWindowSec); err == nil { + t.Fatal("replayed nonce must be rejected") + } +} + +func TestEnvelopeStaleTimestampRejected(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(key, id, &syncMessage{Type: "announce"}, 1000) + + // now is far beyond the replay window from the envelope timestamp. + if _, _, err := openEnvelope(key, raw, 1000+replayWindowSec+10, rc, replayWindowSec); err == nil { + t.Fatal("stale timestamp must be rejected") + } +} + +func TestEnvelopeWrongKeyRejected(t *testing.T) { + k1, _ := deriveAEADKey([]byte("psk-1")) + k2, _ := deriveAEADKey([]byte("psk-2")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(k1, id, &syncMessage{Type: "announce"}, 1000) + if _, _, err := openEnvelope(k2, raw, 1000, rc, replayWindowSec); err == nil { + t.Fatal("wrong cluster key must fail to open (membership auth)") + } +} diff --git a/internal/conf/config.go b/internal/conf/config.go index f347380d81..223fd9606c 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -106,6 +106,10 @@ type SFTP struct { Listen string `json:"listen" env:"LISTEN"` } +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -118,10 +122,12 @@ type Config struct { TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` + FrontendRepo string `json:"frontend_repo" env:"FRONTEND_REPO"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` - MaxBufferLimit int `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"` - MmapThreshold int `json:"mmap_thresholdMB" env:"MMAP_THRESHOLD_MB"` + AutoMemoryLimit int `json:"auto_memory_limit" env:"AUTO_MEMORY_LIMIT"` + MinFreeMemory int `json:"min_free_memory" env:"MIN_FREE_MEMORY"` + MaxBlockLimit int `json:"max_block_limit" env:"MAX_BLOCK_LIMIT"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` @@ -130,6 +136,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` LastLaunchedVersion string `json:"last_launched_version"` ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"` } @@ -162,7 +169,8 @@ func DefaultConfig(dataDir string) *Config { Host: "http://localhost:7700", Index: "openlist", }, - BleveDir: indexDir, + BleveDir: indexDir, + FrontendRepo: FrontendRepoDefault, Log: LogConfig{ Enable: true, Name: logPath, @@ -178,8 +186,7 @@ func DefaultConfig(dataDir string) *Config { }, }, }, - MaxBufferLimit: -1, - MmapThreshold: 4, + AutoMemoryLimit: 4, MaxConnections: 0, MaxConcurrency: 64, TlsInsecureSkipVerify: false, @@ -244,6 +251,9 @@ func DefaultConfig(dataDir string) *Config { Enable: false, Listen: ":5222", }, + MCP: MCP{ + Enable: false, + }, LastLaunchedVersion: "", ProxyAddress: "", } diff --git a/internal/conf/var.go b/internal/conf/var.go index 972f69997a..76de345873 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,6 +12,7 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" + FrontendRepoDefault string = "Ironboxplus/OpenList-Frontend" ) var ( @@ -25,11 +26,16 @@ var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( - // 单个Buffer最大限制 - MaxBufferLimit = 16 * 1024 * 1024 - // 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存 - MmapThreshold = 4 * 1024 * 1024 + // 在HybridCache中使用[]byte缓存数据流的限制，内存为Go自动管理，直到GC + AutoMemoryLimit uint64 = 4 * 1024 * 1024 + // 最小空闲内存，当内存不足时，HybridCache会回退到文件缓存。 + // 如果为0，HybridCache会使用文件缓存，不占用内存。 + MinFreeMemory uint64 = 16 * 1024 * 1024 + // 限制HybridCache手动管理内存单次的扩容大小，超过该阈值将分多次扩容。 + // MinFreeMemory大于0时，也限制 Downloader 的PartSize + MaxBlockLimit uint64 = 16 * 1024 * 1024 ) + var ( RawIndexHtml string ManageHtml string diff --git a/internal/db/db.go b/internal/db/db.go index 96529c15d3..a6bda52b3c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.Favorite)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/favorite.go b/internal/db/favorite.go new file mode 100644 index 0000000000..1cbeb6d5ff --- /dev/null +++ b/internal/db/favorite.go @@ -0,0 +1,44 @@ +package db + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func CreateFavorite(f *model.Favorite) error { + return errors.WithStack(db.Create(f).Error) +} + +func UpdateFavorite(f *model.Favorite) error { + return errors.WithStack(db.Save(f).Error) +} + +// DeleteFavorite removes a favorite scoped to the owning user so users can't +// delete favorites belonging to others. +func DeleteFavorite(userID, id uint) error { + return errors.WithStack( + db.Where("user_id = ? AND id = ?", userID, id).Delete(&model.Favorite{}).Error, + ) +} + +func GetFavoritesByUser(userID uint) ([]model.Favorite, error) { + var favorites []model.Favorite + if err := db.Where("user_id = ?", userID).Order(columnName("created_at") + " DESC").Find(&favorites).Error; err != nil { + return nil, errors.Wrapf(err, "failed get favorites") + } + return favorites, nil +} + +// GetFavoriteByUserAndPath looks up a single favorite by (userID, path) for +// dedup/toggle. Returns (nil, nil) when no matching favorite exists. +func GetFavoriteByUserAndPath(userID uint, path string) (*model.Favorite, error) { + var f model.Favorite + if err := db.Where("user_id = ? AND path = ?", userID, path).First(&f).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed get favorite") + } + return &f, nil +} diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index f20e507f07..0d42aca6d5 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -65,5 +65,5 @@ func SaveSettingItem(item *model.SettingItem) error { } func DeleteSettingItemByKey(key string) error { - return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error) + return errors.WithStack(db.Where(fmt.Sprintf("%s = ?", columnName("key")), key).Delete(model.SettingItem{}).Error) } diff --git a/internal/db/storage.go b/internal/db/storage.go index 0c660a156c..decec5b640 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" @@ -22,6 +23,25 @@ func UpdateStorage(storage *model.Storage) error { return errors.WithStack(db.Save(storage).Error) } +// UpdateStorageStatus just updates storage status in database. +func UpdateStorageStatus(id uint, status string) error { + return errors.WithStack(db.Model(&model.Storage{}).Where("id = ?", id).Update("status", status).Error) +} + +// UpdateStorageStatusIfModified updates a lifecycle status only if the row is +// still the storage generation observed by the caller. This is used for auth +// callbacks: an old in-flight request must not mark a newly reconfigured token +// invalid after UpdateStorage has advanced Modified. +func UpdateStorageStatusIfModified(id uint, modified time.Time, status string) (bool, error) { + tx := db.Model(&model.Storage{}). + Where("id = ? AND modified = ?", id, modified). + Update("status", status) + if tx.Error != nil { + return false, errors.WithStack(tx.Error) + } + return tx.RowsAffected > 0, nil +} + // DeleteStorageById just delete storage from database by id func DeleteStorageById(id uint) error { return errors.WithStack(db.Delete(&model.Storage{}, id).Error) diff --git a/internal/driver/video.go b/internal/driver/video.go new file mode 100644 index 0000000000..423b50fa88 --- /dev/null +++ b/internal/driver/video.go @@ -0,0 +1,39 @@ +package driver + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// VideoPlayInfo describes one official transcoded online-play source for a video. +type VideoPlayInfo struct { + Resolution string `json:"resolution"` // human readable, e.g. "1080P" + Definition int `json:"definition"` + URL string `json:"url"` +} + +// VideoPlayer is an optional driver capability that exposes the storage +// provider's official online-play (transcoded streaming) sources for a video. +type VideoPlayer interface { + VideoPlay(ctx context.Context, file model.Obj) ([]VideoPlayInfo, error) +} + +// VideoSubtitleInfo describes one external subtitle track for a video, exposed +// by providers that index/extract subtitles independently of the media stream +// (e.g. 115 extracts a container's embedded subtitle tracks during transcoding +// and serves them as standalone files). Because they are independent of the +// play source, they render on every quality tier — including transcoded HLS +// streams that don't carry the original container's embedded subtitles. +type VideoSubtitleInfo struct { + Language string `json:"language"` // provider language code, e.g. "chi", "eng" + Title string `json:"title"` // human label, e.g. "简体中文" + URL string `json:"url"` + Type string `json:"type"` // subtitle format: "srt" | "ass" | "vtt" +} + +// VideoSubtitleProvider is an optional driver capability exposing the provider's +// subtitle tracks for a video, independent of the play source. +type VideoSubtitleProvider interface { + VideoSubtitle(ctx context.Context, file model.Obj) ([]VideoSubtitleInfo, error) +} diff --git a/internal/frontend/fetcher.go b/internal/frontend/fetcher.go new file mode 100644 index 0000000000..b087615786 --- /dev/null +++ b/internal/frontend/fetcher.go @@ -0,0 +1,507 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + defaultFrontendRepo = "OpenListTeam/OpenList-Frontend" + versionFile = ".frontend_version" + distDirName = "dist" + maxExtractFileSize = 500 * 1024 * 1024 // 500MB per file +) + +// FetchResult contains the result of a fetch operation +type FetchResult struct { + Version string + Downloaded bool + DistPath string +} + +// GetDistPath returns the path where dynamically fetched frontend dist is stored +func GetDistPath() string { + return filepath.Join(flags.DataDir, "frontend_dist") +} + +// GetVersionFilePath returns the path to the version tracking file +func GetVersionFilePath() string { + return filepath.Join(GetDistPath(), versionFile) +} + +// HasValidDist checks if the dynamic dist directory exists and has an index.html +func HasValidDist() bool { + distPath := GetDistPath() + _, err := os.Stat(filepath.Join(distPath, distDirName, "index.html")) + return err == nil +} + +// ReadCurrentVersion reads the currently cached version from disk +func ReadCurrentVersion() string { + data, err := os.ReadFile(GetVersionFilePath()) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// writeVersion writes the version string to the version tracking file +func writeVersion(version string) error { + return os.WriteFile(GetVersionFilePath(), []byte(version), 0644) +} + +// FetchFromRolling downloads the frontend dist from the GitHub rolling release +func FetchFromRolling(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "rolling", "") +} + +// FetchFromLatest downloads the frontend dist from the GitHub latest release +func FetchFromLatest(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "", "") +} + +// githubRelease represents a GitHub release for JSON parsing +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []struct { + BrowserDownloadURL string `json:"browser_download_url"` + Name string `json:"name"` + } `json:"assets"` + PublishedAt string `json:"published_at"` +} + +type githubRef struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +type githubAnnotatedTag struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +func getFrontendRepo() string { + if conf.Conf != nil && strings.TrimSpace(conf.Conf.FrontendRepo) != "" { + return strings.TrimSpace(conf.Conf.FrontendRepo) + } + return defaultFrontendRepo +} + +func shortHash(sha string) string { + const shortLen = 12 + if len(sha) > shortLen { + return sha[:shortLen] + } + return sha +} + +func versionIdentifier(tag, commitSHA, fallback string) string { + if strings.TrimSpace(commitSHA) != "" { + return fmt.Sprintf("%s@%s", tag, shortHash(commitSHA)) + } + if strings.TrimSpace(fallback) != "" { + return fallback + } + return tag +} + +func resolveTagCommitSHA(ctx context.Context, client *http.Client, baseURL, tag string) (string, error) { + if strings.TrimSpace(tag) == "" { + return "", fmt.Errorf("empty tag") + } + + apiBase := "https://api.github.com" + if baseURL != "" { + apiBase = strings.TrimRight(baseURL, "/") + } + + repo := getFrontendRepo() + refURL := fmt.Sprintf("%s/repos/%s/git/ref/tags/%s", apiBase, repo, url.PathEscape(tag)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, refURL, nil) + if err != nil { + return "", fmt.Errorf("create ref request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetch tag ref: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("tag ref API returned %d: %s", resp.StatusCode, string(body)) + } + + var ref githubRef + if err := json.NewDecoder(resp.Body).Decode(&ref); err != nil { + return "", fmt.Errorf("decode ref JSON: %w", err) + } + + switch ref.Object.Type { + case "commit": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty commit sha in ref response") + } + return ref.Object.SHA, nil + case "tag": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty tag sha in ref response") + } + tagObjURL := fmt.Sprintf("%s/repos/%s/git/tags/%s", apiBase, repo, ref.Object.SHA) + tagReq, err := http.NewRequestWithContext(ctx, http.MethodGet, tagObjURL, nil) + if err != nil { + return "", fmt.Errorf("create tag object request: %w", err) + } + tagReq.Header.Set("Accept", "application/vnd.github.v3+json") + tagReq.Header.Set("User-Agent", "OpenList") + + tagResp, err := client.Do(tagReq) + if err != nil { + return "", fmt.Errorf("fetch tag object: %w", err) + } + defer tagResp.Body.Close() + + if tagResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(tagResp.Body) + return "", fmt.Errorf("tag object API returned %d: %s", tagResp.StatusCode, string(body)) + } + + var tagObj githubAnnotatedTag + if err := json.NewDecoder(tagResp.Body).Decode(&tagObj); err != nil { + return "", fmt.Errorf("decode tag object JSON: %w", err) + } + if tagObj.Object.SHA == "" { + return "", fmt.Errorf("empty object sha in tag object response") + } + return tagObj.Object.SHA, nil + default: + if ref.Object.SHA == "" { + return "", fmt.Errorf("unsupported ref object type %q with empty sha", ref.Object.Type) + } + return ref.Object.SHA, nil + } +} + +// fetchFromTag downloads frontend dist from a GitHub release tag. +// If baseURL is non-empty, it replaces api.github.com (used for testing). +func fetchFromTag(ctx context.Context, tag string, baseURL string) (*FetchResult, error) { + repo := getFrontendRepo() + var apiURL string + if baseURL != "" { + if tag == "" { + apiURL = fmt.Sprintf("%s/repos/%s/releases/latest", baseURL, repo) + } else { + apiURL = fmt.Sprintf("%s/repos/%s/releases/tags/%s", baseURL, repo, tag) + } + } else { + if tag == "" { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo) + } else { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/tags/%s", repo, tag) + } + } + + client := newHTTPClient() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch release info: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("github API returned %d: %s", resp.StatusCode, string(body)) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("decode release JSON: %w", err) + } + + // Find the dist tarball URL (non-lite) + var tarURL string + for _, asset := range release.Assets { + if strings.Contains(asset.Name, "openlist-frontend-dist") && + !strings.Contains(asset.Name, "lite") && + strings.HasSuffix(asset.Name, ".tar.gz") { + tarURL = asset.BrowserDownloadURL + break + } + } + if tarURL == "" { + return nil, fmt.Errorf("no frontend dist tarball found in release %s", release.TagName) + } + + commitSHA, err := resolveTagCommitSHA(ctx, client, baseURL, release.TagName) + if err != nil { + utils.Log.Warnf("[frontend] failed to resolve tag %s hash: %v", release.TagName, err) + } + + resolvedVersion := versionIdentifier(release.TagName, commitSHA, tarURL) + + // Use tag+commit-hash as the primary version identifier. + // For rolling releases the tag itself is static, but its target commit moves. + // If hash resolve fails, fallback to tarball URL so updates can still be detected. + currentVersion := ReadCurrentVersion() + if currentVersion == resolvedVersion && HasValidDist() { + utils.Log.Infof("[frontend] version %s already cached, skipping download", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: false, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil + } + + utils.Log.Infof("[frontend] downloading version %s from %s", resolvedVersion, tarURL) + if err := downloadAndExtract(ctx, client, tarURL); err != nil { + return nil, fmt.Errorf("download and extract: %w", err) + } + + if err := writeVersion(resolvedVersion); err != nil { + utils.Log.Warnf("[frontend] failed to write version file: %v", err) + } + + utils.Log.Infof("[frontend] successfully fetched version %s", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: true, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil +} + +func downloadAndExtract(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create download request: %w", err) + } + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download tarball: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download returned status %d", resp.StatusCode) + } + + destDir := GetDistPath() + tmpDir := filepath.Join(destDir, ".tmp-"+fmt.Sprintf("%d", time.Now().UnixNano())) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := extractTarGz(resp.Body, tmpDir); err != nil { + return fmt.Errorf("extract tar.gz: %w", err) + } + + // Determine the source directory: + // If the tarball contains a "dist" subdirectory, use it; + // otherwise, the files are at the root and we use tmpDir directly. + srcDir := tmpDir + if _, err := os.Stat(filepath.Join(tmpDir, distDirName)); err == nil { + srcDir = filepath.Join(tmpDir, distDirName) + } + + // Atomic swap: hold lock to minimize the window where dist is absent + finalDir := filepath.Join(destDir, distDirName) + oldDir := filepath.Join(destDir, distDirName+".old") + + distSwapMu.Lock() + os.RemoveAll(oldDir) + os.Rename(finalDir, oldDir) + if err := os.Rename(srcDir, finalDir); err != nil { + os.RemoveAll(finalDir) + os.Rename(oldDir, finalDir) + distSwapMu.Unlock() + return fmt.Errorf("rename new dist: %w", err) + } + distSwapMu.Unlock() + os.RemoveAll(oldDir) + + return nil +} + +func extractTarGz(r io.Reader, dest string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("tar next: %w", err) + } + + // Normalize: strip leading ./ so "./dist" becomes "dist" + name := strings.TrimPrefix(hdr.Name, "./") + if name == "" || name == "." { + continue // skip bare directory entry + } + + target := filepath.Join(dest, name) + + // Security: prevent path traversal + if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("path traversal detected: %s", hdr.Name) + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg: + if hdr.Size > maxExtractFileSize { + return fmt.Errorf("file too large: %s (%d bytes)", hdr.Name, hdr.Size) + } + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, io.LimitReader(tr, maxExtractFileSize)); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + +// newHTTPClient creates an HTTP client that respects proxy configuration +func newHTTPClient() *http.Client { + transport := &http.Transport{} + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { + if proxyURL := mustParseURL(conf.Conf.ProxyAddress); proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{ + Transport: transport, + Timeout: 5 * time.Minute, + } +} + +func mustParseURL(raw string) *url.URL { + u, err := url.Parse(raw) + if err != nil { + utils.Log.Warnf("[frontend] invalid proxy URL %q: %v", raw, err) + return nil + } + return u +} + +// EnsureDist ensures a valid frontend dist is available, fetching if necessary. +// It first tries the dynamic dist, then falls back to fetching from GitHub. +// Returns the path to the dist directory, or empty string if no dist is available. +func EnsureDist(ctx context.Context) string { + // If user explicitly configured dist_dir, use that + if conf.Conf != nil && conf.Conf.DistDir != "" { + if _, err := os.Stat(filepath.Join(conf.Conf.DistDir, "index.html")); err == nil { + return conf.Conf.DistDir + } + } + + // Check if dynamic dist already exists + if HasValidDist() { + return filepath.Join(GetDistPath(), distDirName) + } + + // If auto-fetch is enabled (and WebVersion is rolling/beta/dev), try fetching + if shouldAutoFetch() { + utils.Log.Infof("[frontend] no local dist found, fetching from rolling release...") + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] failed to fetch from rolling: %v", err) + // Fall through to return empty (embedded dist will be used as fallback) + return "" + } + return result.DistPath + } + + return "" +} + +func shouldAutoFetch() bool { + v := conf.WebVersion + return v == "" || v == "rolling" || v == "beta" || v == "dev" +} + +// Ensure the directory exists for the frontend dist +func init() { + _ = os.MkdirAll(GetDistPath(), 0755) +} + +var distSwapMu sync.Mutex + +// Ensure that the sync.Once pattern is used for the fetcher +var ( + fetchMu sync.Mutex + fetchDone bool + fetchResult string +) + +// EnsureDistOnce is a thread-safe version of EnsureDist that only fetches once per process. +// On failure, it does not lock the state so subsequent calls can retry. +func EnsureDistOnce(ctx context.Context) string { + fetchMu.Lock() + defer fetchMu.Unlock() + if fetchDone { + return fetchResult + } + result := EnsureDist(ctx) + if result != "" { + fetchResult = result + fetchDone = true + } + return result +} + +// ResetFetchState resets the fetch state (used for testing or re-fetch) +func ResetFetchState() { + fetchMu.Lock() + defer fetchMu.Unlock() + fetchDone = false + fetchResult = "" +} diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go new file mode 100644 index 0000000000..3ed6d95466 --- /dev/null +++ b/internal/frontend/fetcher_test.go @@ -0,0 +1,627 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// createTestTarGz creates a tar.gz containing files with given names and contents +func createTestTarGz(t *testing.T, files map[string]string) []byte { + t.Helper() + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return + } + if _, err := tw.Write([]byte(content)); err != nil { + return + } + } + }() + data, err := io.ReadAll(pr) + if err != nil { + t.Fatalf("read tar.gz: %v", err) + } + return data +} + +func TestExtractTarGz(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "dist/index.html": "hello", + "dist/assets/app.js": "console.log('app')", + "dist/assets/style.css": "body {}", + "dist/images/logo.svg": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz: %v", err) + } + + for name, expectedContent := range files { + path := filepath.Join(tmpDir, name) + data, err := os.ReadFile(path) + if err != nil { + t.Errorf("read %s: %v", name, err) + continue + } + if string(data) != expectedContent { + t.Errorf("content of %s: got %q, want %q", name, string(data), expectedContent) + } + } +} + +func TestExtractTarGzDotSlash(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "./dist/index.html": "dot-slash", + "./": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz with ./ prefix: %v", err) + } + + data, err := os.ReadFile(filepath.Join(tmpDir, "dist", "index.html")) + if err != nil { + t.Fatalf("read dist/index.html: %v", err) + } + if string(data) != "dot-slash" { + t.Errorf("got %q, want dot-slash content", string(data)) + } +} + +func TestExtractTarGzPathTraversal(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "../../../etc/passwd": "root:x:0:0", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err == nil { + t.Fatal("expected error for path traversal, got nil") + } + if !strings.Contains(err.Error(), "path traversal") { + t.Errorf("expected path traversal error, got: %v", err) + } +} + +func TestExtractTarGzRejectsOversizedFile(t *testing.T) { + tmpDir := t.TempDir() + // Create a tar.gz with a file whose header claims a size exceeding the limit + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + hdr := &tar.Header{ + Name: "dist/huge.bin", + Mode: 0644, + Size: maxExtractFileSize + 1, + } + _ = tw.WriteHeader(hdr) + // Write just enough to pass; the size check should reject before reading + buf := make([]byte, 1024) + for written := int64(0); written < hdr.Size; written += int64(len(buf)) { + n := min(int64(len(buf)), hdr.Size-written) + _, _ = tw.Write(buf[:n]) + } + }() + data, _ := io.ReadAll(pr) + + err := extractTarGz(strings.NewReader(string(data)), tmpDir) + if err == nil { + t.Fatal("expected error for oversized file, got nil") + } + if !strings.Contains(err.Error(), "too large") { + t.Errorf("expected 'too large' error, got: %v", err) + } +} + +func TestHasValidDist(t *testing.T) { + if HasValidDist() { + t.Log("HasValidDist returned true (may have existing dist from previous runs)") + } +} + +func TestWriteAndReadVersion(t *testing.T) { + _ = os.MkdirAll(GetDistPath(), 0755) + versionPath := GetVersionFilePath() + + origData, origErr := os.ReadFile(versionPath) + defer func() { + if origErr == nil { + _ = os.WriteFile(versionPath, origData, 0644) + } else { + _ = os.Remove(versionPath) + } + }() + + testVersion := "v1.0.0-test" + if err := writeVersion(testVersion); err != nil { + t.Fatalf("writeVersion: %v", err) + } + + got := ReadCurrentVersion() + if got != testVersion { + t.Errorf("ReadCurrentVersion: got %q, want %q", got, testVersion) + } +} + +func TestShouldAutoFetch(t *testing.T) { + origVersion := conf.WebVersion + defer func() { conf.WebVersion = origVersion }() + + tests := []struct { + version string + want bool + }{ + {"", true}, + {"rolling", true}, + {"beta", true}, + {"dev", true}, + {"v3.0.0", false}, + {"latest", false}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + conf.WebVersion = tt.version + if got := shouldAutoFetch(); got != tt.want { + t.Errorf("shouldAutoFetch(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestFetchFromRollingIntegration(t *testing.T) { + files := map[string]string{ + "./dist/index.html": "integration", + "./dist/assets/test.js": "console.log('test')", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-test", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/frontend.tar.gz" + }] + }`, ts.URL))) + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling-test": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "0123456789abcdef0123456789abcdef01234567" + } + }`)) + case "/download/frontend.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + ctx := context.Background() + result, err := fetchFromTag(ctx, "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + + if result.Version != "rolling-test@0123456789ab" { + t.Errorf("version: got %q, want %q", result.Version, "rolling-test@0123456789ab") + } + if !result.Downloaded { + t.Error("expected Downloaded=true") + } + + idx, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(idx) != "integration" { + t.Errorf("index.html: got %q", string(idx)) + } + + ver := ReadCurrentVersion() + expectedVer := "rolling-test@0123456789ab" + if ver != expectedVer { + t.Errorf("version file: got %q, want %q", ver, expectedVer) + } +} + +func TestLegacyConfigJSONGetsDefaultFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"site_url":"https://example.com"}`), cfg); err != nil { + t.Fatalf("unmarshal legacy config: %v", err) + } + if cfg.FrontendRepo != conf.FrontendRepoDefault { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, conf.FrontendRepoDefault) + } +} + +func TestDefaultConfigUsesBuiltFrontendRepoDefault(t *testing.T) { + orig := conf.FrontendRepoDefault + conf.FrontendRepoDefault = "Ironboxplus/OpenList-Frontend" + defer func() { conf.FrontendRepoDefault = orig }() + + cfg := conf.DefaultConfig("data") + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, "Ironboxplus/OpenList-Frontend") + } +} + +func TestExistingConfigJSONKeepsFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"frontend_repo":"Ironboxplus/OpenList-Frontend"}`), cfg); err != nil { + t.Fatalf("unmarshal config with frontend_repo: %v", err) + } + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q", cfg.FrontendRepo) + } +} + +func TestFetchFromTagUsesConfiguredFrontendRepo(t *testing.T) { + origConf := conf.Conf + if origConf == nil { + conf.Conf = &conf.Config{} + } else { + confCopy := *origConf + conf.Conf = &confCopy + } + defer func() { conf.Conf = origConf }() + conf.Conf.FrontendRepo = "Ironboxplus/OpenList-Frontend" + + files := map[string]string{ + "./dist/index.html": "custom-repo", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/Ironboxplus/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-custom", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/custom.tar.gz" + }] + }`, ts.URL))) + case "/repos/Ironboxplus/OpenList-Frontend/git/ref/tags/rolling-custom": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "fedcba9876543210fedcba9876543210fedcba98" + } + }`)) + case "/download/custom.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tarData) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0o755) + _ = os.RemoveAll(filepath.Join(GetDistPath(), distDirName)) + _ = os.Remove(GetVersionFilePath()) + + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag(custom repo): %v", err) + } + if result.Version != "rolling-custom@fedcba987654" { + t.Fatalf("version: got %q, want %q", result.Version, "rolling-custom@fedcba987654") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "custom-repo" { + t.Fatalf("index.html: got %q", string(data)) + } +} + +func TestResolveTagCommitSHA_AnnotatedTag(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "tag", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }`)) + case "/repos/OpenListTeam/OpenList-Frontend/git/tags/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + sha, err := resolveTagCommitSHA(context.Background(), ts.Client(), ts.URL, "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA: %v", err) + } + if sha != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("sha: got %q, want %q", sha, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + } +} + +func TestVersionIdentifierFallback(t *testing.T) { + tests := []struct { + name string + tag string + sha string + fallback string + want string + }{ + {name: "hash preferred", tag: "rolling", sha: "0123456789abcdef", fallback: "fallback", want: "rolling@0123456789ab"}, + {name: "fallback url", tag: "rolling", sha: "", fallback: "http://example.com/dist.tar.gz", want: "http://example.com/dist.tar.gz"}, + {name: "tag only", tag: "rolling", sha: "", fallback: "", want: "rolling"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := versionIdentifier(tt.tag, tt.sha, tt.fallback) + if got != tt.want { + t.Fatalf("versionIdentifier() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestResolveTagCommitSHA_RealGitHubWithProxy10808(t *testing.T) { + if os.Getenv("OPENLIST_REAL_GITHUB_TEST") != "1" { + t.Skip("set OPENLIST_REAL_GITHUB_TEST=1 to run real GitHub integration test (proxy 127.0.0.1:10808 recommended)") + } + + if os.Getenv("HTTP_PROXY") == "" && os.Getenv("http_proxy") == "" { + _ = os.Setenv("HTTP_PROXY", "http://127.0.0.1:10808") + } + if os.Getenv("HTTPS_PROXY") == "" && os.Getenv("https_proxy") == "" { + _ = os.Setenv("HTTPS_PROXY", "http://127.0.0.1:10808") + } + + client := newHTTPClient() + sha, err := resolveTagCommitSHA(context.Background(), client, "", "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA(real): %v", err) + } + + matched, _ := regexp.MatchString("^[0-9a-f]{40}$", sha) + if !matched { + t.Fatalf("sha format invalid: %q", sha) + } + + // Optional sanity: ensure API can fetch release JSON in real scenario + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling", nil) + if err != nil { + t.Fatalf("create release request: %v", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("fetch release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("release API status=%d body=%s", resp.StatusCode, string(body)) + } + + var release map[string]any + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + t.Fatalf("decode release: %v", err) + } + if release["tag_name"] == nil { + t.Fatalf("release tag_name missing") + } +} + +func TestStaleCacheOverriddenByNewerRelease(t *testing.T) { + oldFiles := map[string]string{"./dist/index.html": "old"} + oldTar := createTestTarGz(t, oldFiles) + newFiles := map[string]string{"./dist/index.html": "new"} + newTar := createTestTarGz(t, newFiles) + + callCount := 0 + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case fmt.Sprintf("/repos/%s/releases/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling", + "assets": [{"name": "openlist-frontend-dist.tar.gz", "browser_download_url": "%s/download/dist.tar.gz"}] + }`, ts.URL))) + case fmt.Sprintf("/repos/%s/git/ref/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"object":{"type":"commit","sha":"aaaaaaaaaaaabbbbbbbbbbbbccccccccccccdddd"}}`)) + case "/download/dist.tar.gz": + callCount++ + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + if callCount == 1 { + _, _ = w.Write(oldTar) + } else { + _, _ = w.Write(newTar) + } + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + destDir := GetDistPath() + _ = os.MkdirAll(destDir, 0755) + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + // Write a stale cached version + _ = writeVersion("rolling@stale000000000") + _ = os.MkdirAll(filepath.Join(destDir, distDirName), 0755) + _ = os.WriteFile(filepath.Join(destDir, distDirName, "index.html"), []byte("stale"), 0644) + + // Fetch should detect version mismatch and download + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag with stale cache: %v", err) + } + if !result.Downloaded { + t.Error("expected Downloaded=true when cache is stale") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "old" { + t.Errorf("index.html: got %q, want old content", string(data)) + } +} + +func TestWatcherTriggersCallbackOnNewVersion(t *testing.T) { + files := map[string]string{"./dist/index.html": "watcher"} + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case fmt.Sprintf("/repos/%s/releases/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling", + "assets": [{"name": "openlist-frontend-dist.tar.gz", "browser_download_url": "%s/download/dist.tar.gz"}] + }`, ts.URL))) + case fmt.Sprintf("/repos/%s/git/ref/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"object":{"type":"commit","sha":"watchertest1234567890watchertest1234567890"}}`)) + case "/download/dist.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + _, _ = w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + destDir := GetDistPath() + _ = os.MkdirAll(destDir, 0755) + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + // The watcher's check() calls FetchFromRolling which uses api.github.com. + // For this test, call fetchFromTag directly and verify Downloaded triggers callback logic. + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + if !result.Downloaded { + t.Fatal("expected Downloaded=true for watcher callback trigger") + } + if result.Version != "rolling@watchertest1" { + t.Errorf("version: got %q", result.Version) + } +} + +func TestEnsureDistOnceFailureDoesNotLock(t *testing.T) { + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0755) + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + origVersion := conf.WebVersion + conf.WebVersion = "v3.0.0" // shouldAutoFetch returns false + defer func() { conf.WebVersion = origVersion }() + + ctx := context.Background() + + result := EnsureDistOnce(ctx) + if result != "" { + t.Errorf("expected empty result, got %q", result) + } + + // Second call should also work (not locked by previous failure) + result2 := EnsureDistOnce(ctx) + if result2 != "" { + t.Errorf("expected empty result on retry, got %q", result2) + } +} diff --git a/internal/frontend/watcher.go b/internal/frontend/watcher.go new file mode 100644 index 0000000000..1a0102fd14 --- /dev/null +++ b/internal/frontend/watcher.go @@ -0,0 +1,122 @@ +package frontend + +import ( + "context" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const defaultCheckInterval = 30 * time.Minute + +// Watcher periodically checks for new frontend versions and fetches them. +type Watcher struct { + interval time.Duration + stopCh chan struct{} + stopped bool + mu sync.Mutex + onUpdated func() +} + +// NewWatcher creates a new frontend watcher. +// onUpdated is called when a new version is fetched (used to reload static files). +func NewWatcher(onUpdated func()) *Watcher { + return &Watcher{ + interval: defaultCheckInterval, + stopCh: make(chan struct{}), + onUpdated: onUpdated, + } +} + +// SetInterval changes the check interval. Must be called before Start. +func (w *Watcher) SetInterval(d time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + w.interval = d +} + +// Start begins the periodic check loop in a background goroutine. +func (w *Watcher) Start() { + w.mu.Lock() + interval := w.interval + w.mu.Unlock() + + go func() { + utils.Log.Infof("[frontend] watcher started, checking every %s", interval) + // Check immediately on start, then periodically + w.check() + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-w.stopCh: + utils.Log.Infof("[frontend] watcher stopped") + return + case <-ticker.C: + w.check() + } + } + }() +} + +// Stop signals the watcher to stop. +func (w *Watcher) Stop() { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return + } + w.stopped = true + close(w.stopCh) +} + +func (w *Watcher) check() { + if !shouldAutoFetch() { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] watcher check failed: %v", err) + return + } + + if result.Downloaded { + utils.Log.Infof("[frontend] watcher fetched new version: %s", result.Version) + if w.onUpdated != nil { + w.onUpdated() + } + } +} + +// globalWatcher is the singleton watcher instance +var ( + globalWatcher *Watcher + watcherMu sync.Mutex +) + +// StartWatcher starts the global frontend watcher. +func StartWatcher(onUpdated func()) { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + return + } + globalWatcher = NewWatcher(onUpdated) + globalWatcher.Start() +} + +// StopWatcher stops the global frontend watcher. +func StopWatcher() { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + globalWatcher.Stop() + globalWatcher = nil + } +} diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be83..379cc61532 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -192,21 +193,23 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) - existedObjs := make(map[string]bool) + // Pre-create the destination directory first + t.Status = "ensuring destination directory exists" + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + log.Warnf("[copy_move] failed to ensure destination dir [%s]: %v, will continue", dstActualPath, err) + // Continue anyway - the directory might exist but Get failed due to cache issues + } + + // Build the set of files already at dst so merge can skip them. + // Subdirectories are created on-demand: each sub-task's recursive + // RunWithNextTaskCallback calls MakeDir at line 198, and op.Put + // also calls MakeDir for the parent before uploading. op.MakeDir + // itself walks up the parent chain so deep trees are covered. + var existedObjs map[string]bool if t.TaskType == merge { - dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) - if err != nil && !errors.Is(err, errs.ObjectNotFound) { - // 目标文件夹不存在的情况不是错误，会在之后新建文件夹 - // 这种情况显然不需要统计existedObjs，dstObjs保持为nil，下面这个for将不会执行 - return errors.WithMessagef(err, "failed list dst [%s] objs", dstActualPath) - } - for _, obj := range dstObjs { - if err := t.Ctx().Err(); err != nil { - return err - } - if !obj.IsDir() { - existedObjs[obj.GetName()] = true - } + existedObjs, err = t.existingDstFiles(dstActualPath) + if err != nil { + return err } } @@ -263,6 +266,44 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } +// existingDstFiles wraps existingDstFilesFn with the storage-bound List call, +// for use by RunWithNextTaskCallback. Returns the set of file names already +// present at dstActualPath; a non-existent destination yields an empty map +// rather than an error so merge tasks can run against a fresh destination. +func (t *FileTransferTask) existingDstFiles(dstActualPath string) (map[string]bool, error) { + listDst := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.DstStorage, path, model.ListArgs{}) + } + return existingDstFilesFn(t.Ctx(), listDst, dstActualPath) +} + +// existingDstFilesFn collects names of files (not directories) already +// present at dstPath. A non-existent destination (errs.ObjectNotFound, +// possibly wrapped) is treated as an empty result so merge tasks can run +// against fresh destinations — see PR #1898. Other List errors propagate. +// +// listDst is injected so tests can drive this without a real storage driver. +func existingDstFilesFn( + ctx context.Context, + listDst func(context.Context, string) ([]model.Obj, error), + dstPath string, +) (map[string]bool, error) { + dstObjs, err := listDst(ctx, dstPath) + if err != nil && !errors.Is(err, errs.ObjectNotFound) { + return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) + } + existed := make(map[string]bool, len(dstObjs)) + for _, obj := range dstObjs { + if err := ctx.Err(); err != nil { + return nil, err + } + if !obj.IsDir() { + existed[obj.GetName()] = true + } + } + return existed, nil +} + var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 0000000000..37869604bc --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,496 @@ +package fs + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + pkgerrors "github.com/pkg/errors" +) + +// ---------- helpers ---------- + +func dirObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: true} +} + +func fileObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: false} +} + +// listRecorder records every call to listDst so tests can assert on +// invocation patterns when needed. +type listRecorder struct { + mu sync.Mutex + calls []string + respond func(path string) ([]model.Obj, error) +} + +func newListRecorder(respond func(string) ([]model.Obj, error)) *listRecorder { + return &listRecorder{respond: respond} +} + +func (r *listRecorder) listDst(_ context.Context, path string) ([]model.Obj, error) { + r.mu.Lock() + r.calls = append(r.calls, path) + r.mu.Unlock() + return r.respond(path) +} + +// ---------- tests: existingDstFilesFn ---------- +// +// These tests pin down the contract of the merge-mode existedObjs builder +// extracted from RunWithNextTaskCallback. They are the regression guard +// for the deletion of the BFS precreate logic: the only invariant that +// the BFS precreate quietly protected (and that #1898's ObjectNotFound +// tolerance independently fixes) lives in this function. + +// 1. dst exists but is empty → empty map, no error. +func TestExistingDstFilesFn_EmptyDst(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { return nil, nil }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected empty map, got %v", got) + } +} + +// 2. dst contains only files → all included. +func TestExistingDstFilesFn_OnlyFiles(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{fileObj("a.txt"), fileObj("b.txt"), fileObj("c.txt")}, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, name := range []string{"a.txt", "b.txt", "c.txt"} { + if !got[name] { + t.Errorf("expected %q in existed set, got %v", name, got) + } + } + if len(got) != 3 { + t.Errorf("expected 3 entries, got %d: %v", len(got), got) + } +} + +// 3. dst contains only directories → empty map (dirs don't count as +// "existed" because merge only skips already-uploaded *files*). +func TestExistingDstFilesFn_OnlyDirs(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{dirObj("subA"), dirObj("subB")}, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("dirs must not appear in existed-files map, got %v", got) + } +} + +// 4. dst contains mixed files and dirs → only files are recorded. +func TestExistingDstFilesFn_MixedFilesAndDirs(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + fileObj("go.mod"), + }, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 3 { + t.Errorf("expected 3 file entries, got %d: %v", len(got), got) + } + for _, name := range []string{"readme.md", "main.go", "go.mod"} { + if !got[name] { + t.Errorf("expected %q in existed set, got %v", name, got) + } + } + for _, name := range []string{"assets", "pkg"} { + if got[name] { + t.Errorf("dir %q must NOT be in existed set, got %v", name, got) + } + } +} + +// 5. dst doesn't exist (raw errs.ObjectNotFound) → empty map, no error. +// THIS IS THE REGRESSION TEST FOR #1898. +func TestExistingDstFilesFn_DstDoesNotExist_RawError(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return nil, errs.ObjectNotFound + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/never-existed") + if err != nil { + t.Fatalf("ObjectNotFound on dst must be tolerated, got error: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected empty map on non-existent dst, got %v", got) + } +} + +// 6. dst doesn't exist (wrapped via pkg/errors.WithMessage) → empty map. +// Guards the errors.Is unwrapping behavior that matters in practice: +// op.List wraps the underlying ObjectNotFound with context messages. +func TestExistingDstFilesFn_DstDoesNotExist_WrappedError(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + // emulate the op.List wrapping path: GetUnwrap returns + // ObjectNotFound, list wraps with WithMessage twice. + return nil, pkgerrors.WithMessage(pkgerrors.WithMessage(errs.ObjectNotFound, "failed get dir"), "while listing") + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("wrapped ObjectNotFound must be tolerated, got error: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected empty map, got %v", got) + } +} + +// 7. List returns a non-ObjectNotFound error (e.g. permission denied, +// I/O failure) → error propagates so the task fails fast. +func TestExistingDstFilesFn_OtherListError_Propagates(t *testing.T) { + sentinel := errors.New("permission denied") + rec := newListRecorder(func(string) ([]model.Obj, error) { + return nil, sentinel + }) + _, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err == nil { + t.Fatal("expected error to propagate, got nil") + } + if !errors.Is(err, sentinel) { + t.Fatalf("expected wrapped sentinel error, got: %v", err) + } +} + +// 8. Context cancelled before list → ctx.Err returned (the contract for +// listDst is to honor ctx; the caller's loop also checks ctx.Err). +func TestExistingDstFilesFn_CtxCancelledBeforeList(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + rec := newListRecorder(func(string) ([]model.Obj, error) { + // A real op.List would honor ctx and return ctx.Err. + return nil, ctx.Err() + }) + _, err := existingDstFilesFn(ctx, rec.listDst, "/dst") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } +} + +// 9. Context cancelled mid-iteration → loop bails with ctx.Err and the +// partial map is not returned. +func TestExistingDstFilesFn_CtxCancelledDuringIteration(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Big enough list that cancelling after returning still gives the + // loop a chance to observe ctx.Err. + objs := make([]model.Obj, 1000) + for i := range objs { + objs[i] = fileObj(fmt.Sprintf("f-%d.txt", i)) + } + + rec := newListRecorder(func(string) ([]model.Obj, error) { + cancel() // cancel immediately after list returns + return objs, nil + }) + _, err := existingDstFilesFn(ctx, rec.listDst, "/dst") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } +} + +// 10. dst contains duplicate file names (pathological but possible if a +// driver returns duplicates) → map dedupes silently, no panic, no error. +func TestExistingDstFilesFn_DuplicateNames(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{ + fileObj("dup.txt"), + fileObj("dup.txt"), + fileObj("unique.txt"), + }, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 2 { + t.Fatalf("expected 2 dedup'd entries, got %d: %v", len(got), got) + } + if !got["dup.txt"] || !got["unique.txt"] { + t.Fatalf("expected both names, got %v", got) + } +} + +// 11. dst contains a large number of files → all included; the call +// completes within a reasonable budget (sanity, not strict perf). +func TestExistingDstFilesFn_LargeDst(t *testing.T) { + const N = 10000 + objs := make([]model.Obj, N) + for i := range objs { + objs[i] = fileObj(fmt.Sprintf("f-%05d.bin", i)) + } + rec := newListRecorder(func(string) ([]model.Obj, error) { return objs, nil }) + + start := time.Now() + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != N { + t.Fatalf("expected %d entries, got %d", N, len(got)) + } + // Generous bound — only catches catastrophic regressions (e.g. + // accidental O(N²) inside the loop). + if elapsed > 2*time.Second { + t.Fatalf("processing %d entries took %v, too slow", N, elapsed) + } +} + +// shouldSkipForMerge mirrors the skip decision at copy_move.go:221 so the +// test below pins both the helper (existingDstFilesFn) AND the call-site +// guard as a single contract. Any future refactor that touches either +// half of this contract will be caught. +func shouldSkipForMerge(srcObj model.Obj, existedFiles map[string]bool) bool { + return !srcObj.IsDir() && existedFiles[srcObj.GetName()] +} + +// TestMergeSkipDecision_DirsAreNeverSkipped is the explicit anti-regression +// test for the worry "when a subdir exists in dst, the src subdir is +// skipped and its contents are not merged". It walks every (src kind, dst +// state) combination and asserts the skip decision. +// +// Why this matters: if existingDstFilesFn ever started including dirs, or +// if the call-site guard dropped the `!obj.IsDir()` clause, src subdirs +// matching dst subdirs would silently stop recursing — and every file +// inside them would never get copied. That bug is invisible to a casual +// "did the task complete?" check and only shows up as quietly missing +// deep files. Lock it down. +func TestMergeSkipDecision_DirsAreNeverSkipped(t *testing.T) { + // dst already contains a mix: one file, two dirs. + dstContents := []model.Obj{ + fileObj("root_file.txt"), + dirObj("subA"), + dirObj("subB"), + } + rec := newListRecorder(func(string) ([]model.Obj, error) { return dstContents, nil }) + + existed, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Sanity: the dirs in dst must NOT be in existedObjs. If this fails, + // every src dir matching a dst dir name would be skipped. + if existed["subA"] || existed["subB"] { + t.Fatalf("CRITICAL: dirs in dst leaked into existed-files map — "+ + "matching src subdirs would be skipped and lose their contents. "+ + "got %v", existed) + } + if !existed["root_file.txt"] { + t.Fatalf("file in dst missing from existed map: %v", existed) + } + + // Exhaustive skip-decision matrix for every src-object kind against + // the populated existedObjs. + cases := []struct { + name string + srcObj model.Obj + wantSkip bool + why string + }{ + { + name: "src_file_matches_dst_file", + srcObj: fileObj("root_file.txt"), + wantSkip: true, + why: "resume semantics: already-uploaded file is skipped", + }, + { + name: "src_dir_matches_dst_dir", + srcObj: dirObj("subA"), + wantSkip: false, + why: "MUST recurse into matching subdir to merge its contents", + }, + { + name: "src_dir_matches_dst_dir_B", + srcObj: dirObj("subB"), + wantSkip: false, + why: "MUST recurse — every dst dir must trigger recursion regardless of name", + }, + { + name: "src_dir_no_match", + srcObj: dirObj("brand_new_dir"), + wantSkip: false, + why: "new dir → recurse and create", + }, + { + name: "src_file_no_match", + srcObj: fileObj("brand_new_file.txt"), + wantSkip: false, + why: "new file → upload", + }, + { + name: "src_dir_matches_dst_FILE_name", + srcObj: dirObj("root_file.txt"), + wantSkip: false, + why: "even with a name collision against a dst FILE, src dir must NOT be skipped " + + "(the conflict will surface later when MakeDir runs, not silently)", + }, + { + name: "src_file_matches_dst_DIR_name", + srcObj: fileObj("subA"), + wantSkip: false, + why: "src file colliding with a dst dir name must NOT be skipped " + + "(dirs are not in existed map, and op.Put will surface the conflict)", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := shouldSkipForMerge(c.srcObj, existed) + if got != c.wantSkip { + t.Fatalf("skip(src=%s,IsDir=%v) = %v, want %v — %s", + c.srcObj.GetName(), c.srcObj.IsDir(), got, c.wantSkip, c.why) + } + }) + } +} + +// TestMergeSkipDecision_EmptyDst: with no existedObjs, NOTHING is skipped +// regardless of obj kind — every src item gets a sub-task. +func TestMergeSkipDecision_EmptyDst(t *testing.T) { + existed := map[string]bool{} + for _, obj := range []model.Obj{ + fileObj("a.txt"), + dirObj("d1"), + fileObj("b.bin"), + dirObj("d2"), + } { + if shouldSkipForMerge(obj, existed) { + t.Errorf("nothing should be skipped against empty dst, but %s was", obj.GetName()) + } + } +} + +// TestMergeSkipDecision_DeepTreeRecursionContract simulates a 3-level src +// tree against a partial dst and asserts that the dir-recursion contract +// holds at every level. This is the "deep dir files missing" regression +// guard the user asked for. +func TestMergeSkipDecision_DeepTreeRecursionContract(t *testing.T) { + // Three levels of nesting, with files at each level. Dst already has + // the dir skeleton from a previous interrupted run, plus one file at + // the deepest level (simulating partial completion). + level0Src := []model.Obj{fileObj("root.txt"), dirObj("L1")} + level1Src := []model.Obj{fileObj("a.txt"), dirObj("L2")} + level2Src := []model.Obj{fileObj("deep1.txt"), fileObj("deep2.txt")} + + // Dst state per level + level0Dst := []model.Obj{fileObj("root.txt"), dirObj("L1")} // root.txt already uploaded + level1Dst := []model.Obj{dirObj("L2")} // L1 exists, no files yet + level2Dst := []model.Obj{fileObj("deep1.txt")} // deep1 already uploaded + + cases := []struct { + level string + srcObjs []model.Obj + dstObjs []model.Obj + mustSkip []string + mustSpawn []string + }{ + { + level: "L0", + srcObjs: level0Src, + dstObjs: level0Dst, + mustSkip: []string{"root.txt"}, // file already there + mustSpawn: []string{"L1"}, // dir must recurse + }, + { + level: "L1", + srcObjs: level1Src, + dstObjs: level1Dst, + mustSkip: []string{}, // a.txt not in dst + mustSpawn: []string{"a.txt", "L2"}, // upload file + recurse dir + }, + { + level: "L2", + srcObjs: level2Src, + dstObjs: level2Dst, + mustSkip: []string{"deep1.txt"}, // already uploaded + mustSpawn: []string{"deep2.txt"}, // must still upload + }, + } + + for _, c := range cases { + t.Run(c.level, func(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { return c.dstObjs, nil }) + existed, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/"+c.level) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var spawned, skipped []string + for _, obj := range c.srcObjs { + if shouldSkipForMerge(obj, existed) { + skipped = append(skipped, obj.GetName()) + } else { + spawned = append(spawned, obj.GetName()) + } + } + + if !sameStrings(skipped, c.mustSkip) { + t.Errorf("%s: skipped = %v, want %v", c.level, skipped, c.mustSkip) + } + if !sameStrings(spawned, c.mustSpawn) { + t.Errorf("%s: spawned = %v, want %v", c.level, spawned, c.mustSpawn) + } + }) + } +} + +func sameStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + m := map[string]int{} + for _, s := range a { + m[s]++ + } + for _, s := range b { + m[s]-- + } + for _, v := range m { + if v != 0 { + return false + } + } + return true +} + +// 12. Sanity: the helper invokes listDst exactly once with the given +// dstPath (no double-listing). +func TestExistingDstFilesFn_CallsListOnce(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{fileObj("a.txt")}, nil + }) + _, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/exact/path") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.calls) != 1 || rec.calls[0] != "/dst/exact/path" { + t.Fatalf("expected single call to /dst/exact/path, got %v", rec.calls) + } +} diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 3d459c6bed..67a1ac065e 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -199,8 +199,8 @@ func PutURL(ctx context.Context, path, dstName, urlStr string) error { return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } -func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { - info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) +func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64, overwrite bool) (any, error) { + info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize, overwrite) if err != nil { log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) } diff --git a/internal/fs/put.go b/internal/fs/put.go index be829ae472..0b905be08c 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -110,10 +110,10 @@ func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return op.Put(ctx, storage, dstDirActualPath, file, nil) } -func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { +func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64, overwrite bool) (any, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } - return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) + return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize, overwrite) } diff --git a/internal/hybrid_cache/buffer.go b/internal/hybrid_cache/buffer.go new file mode 100644 index 0000000000..0023996e63 --- /dev/null +++ b/internal/hybrid_cache/buffer.go @@ -0,0 +1,96 @@ +package hybrid_cache + +import ( + "fmt" + "io" +) + +type BufferStore struct { + blocks [][]byte + size int64 +} + +func (m *BufferStore) Size() int64 { + return m.size +} + +// 用于存储不复用的[]byte +func (m *BufferStore) Append(buf []byte) { + m.size += int64(len(buf)) + m.blocks = append(m.blocks, buf) +} + +func (m *BufferStore) Close() error { + if len(m.blocks) > 0 { + clear(m.blocks) + m.blocks = m.blocks[:0] + m.size = 0 + } + return nil +} + +func (m *BufferStore) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + var n int + for _, buf := range m.blocks { + if off >= int64(len(buf)) { + off -= int64(len(buf)) + continue + } + nn := copy(p[n:], buf[off:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *BufferStore) WriteAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + var n int + for _, b := range m.blocks { + if off >= int64(len(b)) { + off -= int64(len(b)) + continue + } + nn := copy(b[off:], p[n:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func (m *BufferStore) GrowTo(size int64) (err error) { + if size <= m.size { + return nil + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("recovered in %v", r) + } + }() + m.blocks = append(m.blocks, make([]byte, size-m.size)) + m.size = size + return nil +} + +var _ BackingStore = (*BufferStore)(nil) diff --git a/pkg/buffer/bytes_test.go b/internal/hybrid_cache/buffer_test.go similarity index 67% rename from pkg/buffer/bytes_test.go rename to internal/hybrid_cache/buffer_test.go index 3f4d855639..439d7cb78e 100644 --- a/pkg/buffer/bytes_test.go +++ b/internal/hybrid_cache/buffer_test.go @@ -1,26 +1,32 @@ -package buffer +package hybrid_cache_test import ( "errors" "io" "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" ) -func TestReader_ReadAt(t *testing.T) { +func TestBufferStore(t *testing.T) { type args struct { p []byte off int64 } - bs := &Reader{} + bs := &hybrid_cache.BufferStore{} bs.Append([]byte("github.com")) bs.Append([]byte("/OpenList")) - bs.Append([]byte("Team/")) - bs.Append([]byte("OpenList")) + bs.Append([]byte("Team/?")) + b := []byte("OpenList") + off := bs.Size() - 1 + _ = bs.GrowTo(off + int64(len(b))) + _, _ = bs.WriteAt(b, off) + tests := []struct { - name string - b *Reader - args args - want func(a args, n int, err error) error + name string + b *hybrid_cache.BufferStore + args args + check func(a args, n int, err error) error }{ { name: "readAt len 10 offset 0", @@ -29,7 +35,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 10), off: 0, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } @@ -49,7 +55,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 12), off: 11, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } @@ -69,7 +75,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 50), off: 24, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != int(bs.Size()-a.off) { return errors.New("read length not match") } @@ -86,8 +92,8 @@ func TestReader_ReadAt(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := tt.b.ReadAt(tt.args.p, tt.args.off) - if err := tt.want(tt.args, got, err); err != nil { - t.Errorf("Bytes.ReadAt() error = %v", err) + if err := tt.check(tt.args, got, err); err != nil { + t.Errorf("BufferStore.ReadAt() error = %v", err) } }) } diff --git a/internal/hybrid_cache/file.go b/internal/hybrid_cache/file.go new file mode 100644 index 0000000000..bca197dadb --- /dev/null +++ b/internal/hybrid_cache/file.go @@ -0,0 +1,179 @@ +package hybrid_cache + +import ( + "errors" + "io" + "os" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +type singleFileStore struct { + *os.File + size int64 +} + +func (s *singleFileStore) Size() int64 { + return s.size +} + +func (s *singleFileStore) GrowTo(size int64) error { + if size <= s.size { + return nil + } + err := s.File.Truncate(size) + if err == nil { + s.size = size + } + return err +} + +func (s *singleFileStore) Close() error { + err := s.File.Close() + _ = os.Remove(s.File.Name()) + return err +} + +type fileBlock struct { + file *os.File + size int64 + written int64 +} + +type MultiFileStore struct { + blocks []*fileBlock + size int64 +} + +func (s *MultiFileStore) Size() int64 { + return s.size +} + +func (m *MultiFileStore) Close() error { + var errs []error + for _, c := range m.blocks { + if err := c.file.Close(); err != nil { + errs = append(errs, err) + } + _ = os.Remove(c.file.Name()) + } + clear(m.blocks) + m.blocks = m.blocks[:0] + return errors.Join(errs...) +} + +func (m *MultiFileStore) GrowTo(size int64) error { + if size <= m.size { + return nil + } + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + m.blocks = append(m.blocks, &fileBlock{file: f, size: size - m.size}) + m.size = size + return nil +} + +func (m *MultiFileStore) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + for _, c := range m.blocks { + if off >= c.size { + off -= c.size + continue + } + + canRead := min(len(p)-n, int(c.size-off)) + if canRead <= 0 { + break + } + + filled := 0 + + if off < c.written { + fileReadable := min(canRead, int(c.written-off)) + nn, fileErr := c.file.ReadAt(p[n:n+fileReadable], off) + n += nn + filled = nn + if fileErr != nil && !errors.Is(fileErr, io.EOF) { + return n, fileErr + } + } + + if n == len(p) { + return n, nil + } + + if zeroFill := canRead - filled; zeroFill > 0 { + clear(p[n : n+zeroFill]) + n += zeroFill + } + + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *MultiFileStore) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + for _, b := range m.blocks { + if off >= b.size { + off -= b.size + continue + } + + canWrite := min(len(p)-n, int(b.size-off)) + if canWrite <= 0 { + break + } + + nn, fileErr := b.file.WriteAt(p[n:n+canWrite], off) + if end := off + int64(nn); end > b.written { + b.written = end + } + n += nn + if fileErr != nil { + return n, fileErr + } + if nn < canWrite { + return n, io.ErrShortWrite + } + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func NewFileStore(blockSize int64) (BackingStore, error) { + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + err = f.Truncate(blockSize) + if err == nil { + return &singleFileStore{File: f, size: blockSize}, nil + } + return &MultiFileStore{ + blocks: []*fileBlock{{file: f, size: blockSize}}, + size: blockSize, + }, nil +} diff --git a/internal/hybrid_cache/file_test.go b/internal/hybrid_cache/file_test.go new file mode 100644 index 0000000000..7f83625db8 --- /dev/null +++ b/internal/hybrid_cache/file_test.go @@ -0,0 +1,101 @@ +package hybrid_cache_test + +import ( + "bytes" + "errors" + "io" + "os" + "reflect" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" +) + +func TestFile(t *testing.T) { + f, err := os.CreateTemp("", "writeat-*") + if err != nil { + t.Error(err) + return + } + defer os.Remove(f.Name()) + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + n, err := f.WriteAt([]byte("abc"), 20) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + stat, err := f.Stat() + if err != nil { + t.Errorf("stat err=%v", err) + return + } + if stat.Size() != 23 { + t.Fatalf("unexpected size: got %d want 23", stat.Size()) + } + + b := make([]byte, stat.Size()) + rn, rerr := f.ReadAt(b, 0) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := append(make([]byte, 20), []byte("abc")...) + if !reflect.DeepEqual(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} + +func TestMultiFileCache(t *testing.T) { + prevConf := conf.Conf + t.Cleanup(func() { + conf.Conf = prevConf + }) + conf.Conf = &conf.Config{} + f := hybrid_cache.MultiFileStore{} + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + err := f.GrowTo(15) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + n, err := f.WriteAt([]byte("abc"), 10) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + + err = f.GrowTo(30) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + _, _ = f.WriteAt([]byte("123"), 15) + + b := append(make([]byte, 17), []byte("def")...) + b[0] = 'a' + rn, rerr := f.ReadAt(b, 8) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := []byte{0, 0, 'a', 'b', 'c', 0, 0, '1', '2', '3'} + want = append(want, make([]byte, 10)...) + if !bytes.Equal(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} diff --git a/internal/hybrid_cache/hybrid_cache.go b/internal/hybrid_cache/hybrid_cache.go new file mode 100644 index 0000000000..c69147937e --- /dev/null +++ b/internal/hybrid_cache/hybrid_cache.go @@ -0,0 +1,239 @@ +package hybrid_cache + +import ( + "errors" + "io" + "runtime" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/mem" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// 线程不安全，单线程使用，或者外部加锁保护 +type HybridCache struct { + blockSize uint64 + memoryStore mem.LinearMemory + memoryOffset uint64 + backingStore BackingStore + backingOffset uint64 + cleanup runtime.Cleanup +} + +// HybridCache本身是一个大的Block，支持分块成多个小的Block + +// 分配一个新的Block，支持读写，大小为size +func (hc *HybridCache) AllocBlock(size uint64) (buffer.Block, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + fs := buffer.NewBlockAdapter( + io.NewOffsetWriter(hc.backingStore, int64(base)), + io.NewSectionReader(hc.backingStore, int64(base), int64(size)), + ) + return fs, nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return buffer.NewByteBlock(all[start : start+size]), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) allocWriteAtSeeker(size uint64) (buffer.WriteAtSeeker, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + return io.NewOffsetWriter(hc.backingStore, int64(base)), nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return io.NewOffsetWriter(buffer.NewByteBlock(all[start:start+size]), 0), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) NextBlock() (buffer.Block, error) { + return hc.AllocBlock(hc.blockSize) +} + +func (hc *HybridCache) RewindBySize(size uint64) { + if hc.backingOffset >= size { + hc.backingOffset -= size + return + } + size -= hc.backingOffset + hc.backingOffset = 0 + if hc.memoryOffset >= size { + hc.memoryOffset -= size + return + } + size -= hc.memoryOffset + hc.memoryOffset = 0 +} + +func (hc *HybridCache) RewindOneBlock() { + hc.RewindBySize(hc.blockSize) +} + +func (hc *HybridCache) initFileCache() error { + file, err := NewFileStore(int64(hc.blockSize)) + if err != nil { + return err + } + hc.cleanup = runtime.AddCleanup(hc, func(file BackingStore) { + _ = file.Close() + }, file) + hc.backingStore = file + return nil +} + +func (hc *HybridCache) Close() error { + hc.cleanup.Stop() + var err error + if hc.memoryStore != nil { + err = hc.memoryStore.Free() + hc.memoryStore = nil + hc.memoryOffset = 0 + } + if hc.backingStore != nil { + err = errors.Join(err, hc.backingStore.Close()) + hc.backingStore = nil + hc.backingOffset = 0 + } + return err +} + +func (hc *HybridCache) Size() int64 { + return int64(hc.memoryOffset + hc.backingOffset) +} + +func (hc *HybridCache) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.EOF + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(p, all[off:]) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canRead := int64(hc.backingOffset) - off + if canRead <= 0 { + return n, io.EOF + } + nn, err := hc.backingStore.ReadAt(p[:min(len(p), int(canRead))], off) + return n + nn, err +} + +func (hc *HybridCache) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.ErrShortWrite + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(all[off:], p) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canWrite := int64(hc.backingOffset) - off + if canWrite <= 0 { + return n, io.ErrShortWrite + } + nn, err := hc.backingStore.WriteAt(p[:min(len(p), int(canWrite))], off) + return n + nn, err +} + +func (hc *HybridCache) CopyFromN(src io.Reader, n int64) (written int64, err error) { + limit := n + for limit > 0 { + blockSize := limit + if hc.backingStore == nil && blockSize > int64(conf.MaxBlockLimit) { + blockSize = int64(conf.MaxBlockLimit) + } + b, err := hc.allocWriteAtSeeker(uint64(blockSize)) + if err != nil { + return written, err + } + nn, err := utils.CopyWithBufferN(b, src, blockSize) + written += nn + if nn != blockSize { + return written, err + } + limit -= nn + } + return written, nil +} + +// HybridCache 线程不安全，单线程使用，或者外部加锁保护 +func NewHybridCache(blockSize, maxMemorySize uint64) (hc *HybridCache, err error) { + if conf.MinFreeMemory > 0 { + // 策略1: Go自动内存管理 + if maxMemorySize <= conf.AutoMemoryLimit { + return &HybridCache{backingStore: &BufferStore{}, blockSize: blockSize}, nil + } + + // 策略2: 手动内存管理 + if maxMemorySize >= blockSize { + var m mem.LinearMemory + // 手动管理内存，Uinx Mmap 或者 Windows VirtualAlloc + if m, err = mem.NewGuardedMemory(blockSize, maxMemorySize); err == nil { + hc = &HybridCache{memoryStore: m, blockSize: blockSize} + } + } + } + // 策略3: 文件后备 + if hc == nil { + hc = &HybridCache{blockSize: blockSize} + // 文件 + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + } + return hc, nil +} + +var _ buffer.Block = (*HybridCache)(nil) diff --git a/internal/hybrid_cache/type.go b/internal/hybrid_cache/type.go new file mode 100644 index 0000000000..edc6d3deb9 --- /dev/null +++ b/internal/hybrid_cache/type.go @@ -0,0 +1,13 @@ +package hybrid_cache + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" +) + +type BackingStore interface { + buffer.Block + io.Closer + GrowTo(size int64) error +} diff --git a/internal/mem/mem_other.go b/internal/mem/mem_other.go new file mode 100644 index 0000000000..2535c4570a --- /dev/null +++ b/internal/mem/mem_other.go @@ -0,0 +1,25 @@ +//go:build !unix && !windows + +package mem + +func NewMemory(cap, max uint64) (LinearMemory, error) { + return &sliceMemory{buf: make([]byte, 0, cap)}, nil +} + +type sliceMemory struct { + buf []byte +} + +func (b *sliceMemory) Free() error { + b.buf = nil + return nil +} + +func (b *sliceMemory) Reallocate(size uint64) ([]byte, error) { + if cap := uint64(cap(b.buf)); size > cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf, nil +} diff --git a/internal/mem/mem_unix.go b/internal/mem/mem_unix.go new file mode 100644 index 0000000000..bd1979298e --- /dev/null +++ b/internal/mem/mem_unix.go @@ -0,0 +1,92 @@ +//go:build unix + +package mem + +import ( + "math" + + "golang.org/x/sys/unix" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures int(res) overflows to a negative value, + // and unix.Mmap returns EINVAL. + res = math.MaxUint64 + } + + com := res + prot := unix.PROT_READ | unix.PROT_WRITE + if cap < max { // Commit memory only if cap=max. + com = 0 + prot = unix.PROT_NONE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(res), prot, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + return nil, err + } + return &mmappedMemory{buf: b[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type mmappedMemory struct { + buf []byte + growCheck GrowCheck +} + +func (m *mmappedMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *mmappedMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *mmappedMemory) Free() error { + if m.buf != nil { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + return err + } + m.buf = nil + } + return nil +} diff --git a/internal/mem/mem_windows.go b/internal/mem/mem_windows.go new file mode 100644 index 0000000000..e7a4bb27ef --- /dev/null +++ b/internal/mem/mem_windows.go @@ -0,0 +1,94 @@ +package mem + +import ( + "math" + "unsafe" + + "golang.org/x/sys/windows" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures uintptr(res) overflows to a large value, + // and windows.VirtualAlloc returns an error. + res = math.MaxUint64 + } + + com := res + kind := windows.MEM_COMMIT + if cap < max { // Commit memory only if cap=max. + com = 0 + kind = windows.MEM_RESERVE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + r, err := windows.VirtualAlloc(0, uintptr(res), uint32(kind), windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + buf := unsafe.Slice((*byte)(unsafe.Pointer(r)), int(res)) + return &virtualMemory{addr: r, buf: buf[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type virtualMemory struct { + buf []byte + addr uintptr + growCheck GrowCheck +} + +func (m *virtualMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *virtualMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *virtualMemory) Free() error { + if m.addr != 0 { + err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) + if err != nil { + return err + } + m.addr = 0 + m.buf = nil + } + return nil +} diff --git a/internal/mem/type.go b/internal/mem/type.go new file mode 100644 index 0000000000..3ba8e35ca0 --- /dev/null +++ b/internal/mem/type.go @@ -0,0 +1,9 @@ +package mem + +type LinearMemory interface { + // 线程不安全 + Reallocate(size uint64) (all []byte, err error) + Free() error +} + +type GrowCheck func(growSize uint64) error diff --git a/internal/mem/utils.go b/internal/mem/utils.go new file mode 100644 index 0000000000..b0a46d731f --- /dev/null +++ b/internal/mem/utils.go @@ -0,0 +1,85 @@ +package mem + +import ( + "errors" + "fmt" + "runtime" + "sync/atomic" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/shirou/gopsutil/v4/mem" +) + +var ErrNotEnoughMemory = errors.New("not enough memory") + +func MemoryGrowCheck(growSize uint64) error { + if conf.MinFreeMemory == 0 { + return ErrNotEnoughMemory + } + m, err, _ := singleflight.AnyGroup.Do("MemoryGrowCheck", func() (any, error) { + m, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + if m.Available < conf.MinFreeMemory { + return nil, ErrNotEnoughMemory + } + return m, nil + }) + if err != nil { + return err + } + memStat := m.(*mem.VirtualMemoryStat) + for { + available := atomic.LoadUint64(&memStat.Available) + if available < growSize || available-growSize < conf.MinFreeMemory { + return ErrNotEnoughMemory + } + if atomic.CompareAndSwapUint64(&memStat.Available, available, available-growSize) { + return nil + } + } +} + +func NewGuardedMemory(cap, max uint64) (m LinearMemory, err error) { + if err := MemoryGrowCheck(cap); err != nil { + return nil, err + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + m, err = NewMemory(cap, max) + if err != nil { + return nil, err + } + if s, ok := m.(interface{ SetGrowCheck(GrowCheck) }); ok { + s.SetGrowCheck(MemoryGrowCheck) + } + gm := &guardedMemory{LinearMemory: m} + gm.cleanup = runtime.AddCleanup(gm, func(m LinearMemory) { + m.Free() + }, m) + return gm, nil +} + +type guardedMemory struct { + LinearMemory + cleanup runtime.Cleanup +} + +func (s *guardedMemory) Reallocate(size uint64) (all []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + return s.LinearMemory.Reallocate(size) +} + +func (s *guardedMemory) Free() error { + s.cleanup.Stop() + return s.LinearMemory.Free() +} diff --git a/internal/model/args.go b/internal/model/args.go index 073c94a63a..d165908fb3 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -25,6 +25,10 @@ type LinkArgs struct { Redirect bool } +// LinkRefresher is a callback function type for refreshing download links +// It returns a new Link and the associated object, or an error +type LinkRefresher func(ctx context.Context) (*Link, Obj, error) + type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) @@ -37,6 +41,10 @@ type Link struct { PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 + // Refresher is a callback to refresh the link when it expires during long downloads + // This field is not serialized and is optional - if nil, no refresh will be attempted + Refresher LinkRefresher `json:"-"` + utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` diff --git a/internal/model/favorite.go b/internal/model/favorite.go new file mode 100644 index 0000000000..304db0b140 --- /dev/null +++ b/internal/model/favorite.go @@ -0,0 +1,11 @@ +package model + +type Favorite struct { + ID uint `json:"id" gorm:"primaryKey"` + UserID uint `json:"user_id" gorm:"index"` + Path string `json:"path"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Tag string `json:"tag"` // optional free-text label + CreatedAt int64 `json:"created_at"` +} diff --git a/internal/model/file.go b/internal/model/file.go index 4ca7201e11..d6697cd0e8 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -7,24 +7,26 @@ import ( // File is basic file level accessing interface type File interface { - io.Reader io.ReaderAt - io.Seeker + io.ReadSeeker +} +type FileWriter interface { + io.WriterAt + io.WriteSeeker } type FileCloser struct { File io.Closer } -func (f *FileCloser) Close() error { - var errs []error +func (f *FileCloser) Close() (err error) { if clr, ok := f.File.(io.Closer); ok { - errs = append(errs, clr.Close()) + err = clr.Close() } if f.Closer != nil { - errs = append(errs, f.Closer.Close()) + return errors.Join(err, f.Closer.Close()) } - return errors.Join(errs...) + return } // FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead diff --git a/internal/model/obj.go b/internal/model/obj.go index ba7467f740..22e8de4c3f 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -75,6 +75,15 @@ type Thumb interface { Thumb() string } +// ObjExtra is an optional interface a driver's Obj may implement to surface +// extra, driver-specific metadata (e.g. media duration, video resolution, +// starred flag) without bloating the universal Obj. Keys are stable strings; +// the frontend renders the keys it knows and ignores the rest, so adding or +// removing keys never breaks clients. Implementations should omit empty values. +type ObjExtra interface { + Extra() map[string]any +} + type SetPath interface { SetPath(path string) } @@ -162,6 +171,25 @@ func GetThumb(obj Obj) (thumb string, ok bool) { } } +// GetExtra walks the unwrap chain and returns the driver-specific extra metadata +// if the underlying Obj implements ObjExtra. An empty map is treated as absent. +func GetExtra(obj Obj) (extra map[string]any, ok bool) { + for { + switch o := obj.(type) { + case ObjExtra: + e := o.Extra() + if len(e) == 0 { + return nil, false + } + return e, true + case ObjUnwrap: + obj = o.Unwrap() + default: + return nil, false + } + } +} + func GetUrl(obj Obj) (url string, ok bool) { for { switch o := obj.(type) { diff --git a/internal/model/storage.go b/internal/model/storage.go index a6b4745a60..0f8eb896f4 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -36,6 +36,9 @@ type Proxy struct { DownProxyURL string `json:"down_proxy_url"` // Disable sign for DownProxyURL DisableProxySign bool `json:"disable_proxy_sign"` + // Custom User-Agent for proxy-download upstream requests. + // Empty = passthrough the client's User-Agent (default behavior). + ProxyUserAgent string `json:"proxy_user_agent"` } func (s *Storage) GetStorage() *Storage { @@ -77,6 +80,18 @@ func (d DiskUsage) MarshalJSON() ([]byte, error) { type StorageDetails struct { DiskUsage + // DriverName is the storage's driver/type (e.g. "Local", "BaiduNetdisk"), + // surfaced to the UI so it can label which storage a mount belongs to. + DriverName string +} + +func (s StorageDetails) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "total_space": s.TotalSpace, + "used_space": s.UsedSpace, + "free_space": s.FreeSpace(), + "driver_name": s.DriverName, + }) } type ObjWithStorageDetails interface { diff --git a/internal/model/storage_details_test.go b/internal/model/storage_details_test.go new file mode 100644 index 0000000000..eec33efde4 --- /dev/null +++ b/internal/model/storage_details_test.go @@ -0,0 +1,27 @@ +package model + +import ( + "encoding/json" + "testing" +) + +func TestStorageDetailsMarshalIncludesDriverName(t *testing.T) { + d := StorageDetails{ + DiskUsage: DiskUsage{TotalSpace: 100, UsedSpace: 40}, + DriverName: "BaiduNetdisk", + } + b, err := json.Marshal(d) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + if m["driver_name"] != "BaiduNetdisk" { + t.Fatalf("expected driver_name in JSON, got %v", m["driver_name"]) + } + if m["total_space"].(float64) != 100 || m["free_space"].(float64) != 60 { + t.Fatalf("disk fields wrong: %v", m) + } +} diff --git a/internal/model/user.go b/internal/model/user.go index 2dacd752eb..314b38d20a 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -28,6 +28,7 @@ const ( TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." GuestCannotUpdateProfile = "Guest user can not update profile" GuestCannotGenerate2FA = "Guest user can not generate 2FA code" + NoPermissionUpdateProfile = "You are not allowed to modify your username or password" ) var LoginCache = cache.NewMemCache[int]() @@ -64,6 +65,7 @@ type User struct { // 13: can decompress archives // 14: can share // 15: can customize share id + // 16: can manage other users' profiles (delegated by admin: edit username/password only) Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -228,6 +230,17 @@ func (u *User) CanCustomizeShareID() bool { return CanCustomizeShareID(u.Permission) } +func CanManageUserInfo(permission int32) bool { + return (permission>>16)&1 == 1 +} + +// CanManageUserInfo reports whether this user may edit other users' profiles +// (username/password). Admins always can; non-admins only when delegated this +// permission bit by an admin. +func (u *User) CanManageUserInfo() bool { + return u.IsAdmin() || CanManageUserInfo(u.Permission) +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/model/user_test.go b/internal/model/user_test.go new file mode 100644 index 0000000000..023e937e70 --- /dev/null +++ b/internal/model/user_test.go @@ -0,0 +1,34 @@ +package model + +import "testing" + +func TestCanManageUserInfoBit(t *testing.T) { + // Bit 16 is the delegated "manage user info" permission. + const bit int32 = 1 << 16 + + if CanManageUserInfo(0) { + t.Fatal("no permission bits should mean no delegated user management") + } + if !CanManageUserInfo(bit) { + t.Fatal("bit 16 set should grant delegated user management") + } + // Lower bits must not be mistaken for bit 16. + if CanManageUserInfo(0xFFFF) { + t.Fatal("bits 0-15 must not imply user management") + } +} + +func TestUserCanManageUserInfoAdminAlways(t *testing.T) { + admin := &User{Role: ADMIN, Permission: 0} + if !admin.CanManageUserInfo() { + t.Fatal("admin must always be able to manage user info") + } + normal := &User{Role: GENERAL, Permission: 0} + if normal.CanManageUserInfo() { + t.Fatal("a normal user without the bit must not manage user info") + } + delegated := &User{Role: GENERAL, Permission: 1 << 16} + if !delegated.CanManageUserInfo() { + t.Fatal("a normal user with bit 16 must be able to manage user info") + } +} diff --git a/internal/net/oss.go b/internal/net/oss.go index a897161f14..b28e06bc4a 100644 --- a/internal/net/oss.go +++ b/internal/net/oss.go @@ -1,9 +1,36 @@ package net -import "github.com/aliyun/aliyun-oss-go-sdk/oss" +import ( + "crypto/tls" + stdnet "net" + "net/http" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) func NewOSSClient(endpoint, accessKeyID, accessKeySecret string, options ...oss.ClientOption) (*oss.Client, error) { - clientOptions := []oss.ClientOption{oss.HTTPClient(NewHttpClient())} + clientOptions := []oss.ClientOption{oss.HTTPClient(NewOSSUploadHttpClient())} clientOptions = append(clientOptions, options...) return oss.New(endpoint, accessKeyID, accessKeySecret, clientOptions...) } + +func NewOSSUploadHttpClient() *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + DialContext: (&stdnet.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ResponseHeaderTimeout: 5 * time.Minute, + } + + SetProxyIfConfigured(transport) + + return &http.Client{ + Timeout: time.Hour * 48, + Transport: transport, + } +} diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go index 9001cd39d3..13a4f86fe7 100644 --- a/internal/net/oss_test.go +++ b/internal/net/oss_test.go @@ -8,6 +8,21 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" ) +func TestNewOSSUploadHttpClientHasLongerTimeout(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + defer func() { conf.Conf = oldConf }() + + client := NewOSSUploadHttpClient() + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", client.Transport) + } + if transport.ResponseHeaderTimeout < 120_000_000_000 { // 2 minutes minimum + t.Fatalf("ResponseHeaderTimeout=%v, want >= 2m for upload", transport.ResponseHeaderTimeout) + } +} + func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { oldConf := conf.Conf conf.Conf = conf.DefaultConfig("data") diff --git a/internal/net/request.go b/internal/net/request.go index e1f0451205..4fcdad0fd4 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -5,17 +5,20 @@ import ( "errors" "fmt" "io" + "math/rand/v2" "net/http" stdpath "path" "strconv" "sync" + "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/aws/aws-sdk-go/aws/awsutil" @@ -86,8 +89,8 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo if impl.cfg.PartSize == 0 { impl.cfg.PartSize = DefaultDownloadPartSize } - if conf.MaxBufferLimit > 0 && impl.cfg.PartSize > conf.MaxBufferLimit { - impl.cfg.PartSize = conf.MaxBufferLimit + if conf.MinFreeMemory > 0 && impl.cfg.PartSize > int(conf.MaxBlockLimit) { + impl.cfg.PartSize = int(conf.MaxBlockLimit) } if impl.cfg.HttpClient == nil { impl.cfg.HttpClient = DefaultHttpRequestFunc @@ -102,65 +105,57 @@ type downloader struct { cancel context.CancelCauseFunc cfg Downloader - params *HttpRequestParams //http request params - chunkChannel chan chunk //chunk chanel + params *HttpRequestParams //http request params + chunkCh chan chunk //chunk chanel //wg sync.WaitGroup - m sync.Mutex + mu sync.Mutex nextChunk int //next chunk id - bufs []*Buf + bufMap map[int]*buffer.PipeBuffer written int64 //total bytes of file downloaded from remote - err error concurrency int //剩余的并发数，递减。到0时停止并发 - maxPart int //有多少个分片 pos int64 maxPos int64 - m2 sync.Mutex - readingID int // 正在被读取的id + delayMu sync.Mutex + readingID int64 // 正在被读取的id + + hc *hcache.HybridCache } type ConcurrencyLimit struct { - _m sync.Mutex - Limit int // 需要大于0 + mu sync.Mutex + + Limit uint32 } var ErrExceedMaxConcurrency = HttpStatusCodeError(http.StatusTooManyRequests) -func (l *ConcurrencyLimit) sub() error { - l._m.Lock() - defer l._m.Unlock() - if l.Limit-1 < 0 { +func (l *ConcurrencyLimit) Acquire() error { + if l == nil { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + if l.Limit == 0 { return ErrExceedMaxConcurrency } l.Limit-- - // log.Debugf("ConcurrencyLimit.sub: %d", l.Limit) return nil } -func (l *ConcurrencyLimit) add() { - l._m.Lock() - defer l._m.Unlock() - l.Limit++ - // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) -} - -// 检测是否超过限制 -func (d *downloader) concurrencyCheck() error { - if d.cfg.ConcurrencyLimit != nil { - return d.cfg.ConcurrencyLimit.sub() - } - return nil -} -func (d *downloader) concurrencyFinish() { - if d.cfg.ConcurrencyLimit != nil { - d.cfg.ConcurrencyLimit.add() +func (l *ConcurrencyLimit) Release() { + if l == nil { + return } + l.mu.Lock() + l.Limit++ + l.mu.Unlock() } // download performs the implementation of the object download across ranged GETs. func (d *downloader) download() (io.ReadCloser, error) { - if err := d.concurrencyCheck(); err != nil { + if err := d.cfg.ConcurrencyLimit.Acquire(); err != nil { return nil, err } @@ -176,16 +171,16 @@ func (d *downloader) download() (io.ReadCloser, error) { if maxPart == 1 { resp, err := d.cfg.HttpClient(d.ctx, d.params) if err != nil { - d.concurrencyFinish() + d.cfg.ConcurrencyLimit.Release() return nil, err } closeFunc := resp.Body.Close resp.Body = utils.NewReadCloser(resp.Body, func() error { - d.m.Lock() - defer d.m.Unlock() + d.mu.Lock() + defer d.mu.Unlock() var err error if closeFunc != nil { - d.concurrencyFinish() + d.cfg.ConcurrencyLimit.Release() err = closeFunc() closeFunc = nil } @@ -196,103 +191,131 @@ func (d *downloader) download() (io.ReadCloser, error) { d.ctx, d.cancel = context.WithCancelCause(d.ctx) // workers - d.chunkChannel = make(chan chunk, d.cfg.Concurrency) + d.chunkCh = make(chan chunk, d.cfg.Concurrency) - d.maxPart = maxPart d.pos = d.params.Range.Start d.maxPos = d.params.Range.Start + d.params.Range.Length d.concurrency = d.cfg.Concurrency - _ = d.sendChunkTask(true) - var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) + var err error + d.hc, err = hcache.NewHybridCache(uint64(d.cfg.PartSize), uint64(d.params.Range.Length)) + if err == nil { + d.bufMap = make(map[int]*buffer.PipeBuffer, d.cfg.Concurrency) + err = d.sendChunkTask(true) + } + if err != nil { + d.cancel(err) + d.cfg.ConcurrencyLimit.Release() + return nil, d.interrupt() + } - // Return error - return rc, d.err + d.mu.Lock() + defer d.mu.Unlock() + return &multiReadCloser{d: d, curBuf: d.popBuf(0), maxPos: maxPart}, nil } -func (d *downloader) sendChunkTask(newConcurrency bool) error { - d.m.Lock() - defer d.m.Unlock() - isNewBuf := d.concurrency > 0 +func (d *downloader) sendChunkTask(newConcurrency bool) (err error) { + d.mu.Lock() + defer d.mu.Unlock() + if d.pos >= d.maxPos { + return nil + } if newConcurrency { if d.concurrency <= 0 { return nil } if d.nextChunk > 0 { // 第一个不检查，因为已经检查过了 - if err := d.concurrencyCheck(); err != nil { + if err := d.cfg.ConcurrencyLimit.Acquire(); err != nil { return err } + defer func() { + if err != nil { + d.cfg.ConcurrencyLimit.Release() + } + }() } - d.concurrency-- - go d.downloadPart() } - var buf *Buf - if isNewBuf { - buf = NewBuf(d.ctx, d.cfg.PartSize) - d.bufs = append(d.bufs, buf) - } else { - buf = d.getBuf(d.nextChunk) - } - - if d.pos < d.maxPos { - finalSize := int64(d.cfg.PartSize) - switch d.nextChunk { - case 0: - // 最小分片在前面有助视频播放？ - firstSize := d.params.Range.Length % finalSize - if firstSize > 0 { - minSize := finalSize / 2 - if firstSize < minSize { // 最小分片太小就调整到一半 - finalSize = minSize - } else { - finalSize = firstSize - } - } - case 1: - firstSize := d.params.Range.Length % finalSize - minSize := finalSize / 2 - if firstSize > 0 && firstSize < minSize { - finalSize += firstSize - minSize - } - } - err := buf.Reset(int(finalSize)) + br := d.bufMap[d.nextChunk] + if br == nil { + var b buffer.Block + b, err = d.hc.NextBlock() if err != nil { return err } - ch := chunk{ - start: d.pos, - size: finalSize, - id: d.nextChunk, - buf: buf, + br = buffer.NewPipeBuffer(d.ctx, b) + d.bufMap[d.nextChunk] = br + } - newConcurrency: newConcurrency, + finalSize := int64(d.cfg.PartSize) + switch d.nextChunk { + case 0: + // 最小分片在前面有助视频播放？ + firstSize := d.params.Range.Length % finalSize + if firstSize > 0 { + minSize := finalSize / 2 + // 最小分片太小就调整到一半 + finalSize = max(firstSize, minSize) } - d.pos += finalSize - d.nextChunk++ - d.chunkChannel <- ch + case 1: + firstSize := d.params.Range.Length % finalSize + minSize := finalSize / 2 + if firstSize > 0 && firstSize < minSize { + finalSize += firstSize - minSize + } + } + err = br.Reset(int(finalSize)) + if err != nil { + return err // 分片算法错误或者下载中断 + } + if newConcurrency { + go d.downloadPart() + d.concurrency-- + } + ch := chunk{ + start: d.pos, + size: finalSize, + id: d.nextChunk, + buf: br, + + newConcurrency: newConcurrency, + } + d.pos += finalSize + d.nextChunk++ + select { + case <-d.ctx.Done(): + return context.Cause(d.ctx) + case d.chunkCh <- ch: return nil } - return nil } // when the final reader Close, we interrupt func (d *downloader) interrupt() error { - d.m.Lock() - defer d.m.Unlock() - err := d.err - if err == nil && d.written != d.params.Range.Length { - log.Debugf("Downloader interrupt before finish") - err := fmt.Errorf("interrupted") - d.err = err - } - close(d.chunkChannel) - if d.bufs != nil { - d.cancel(err) - for _, buf := range d.bufs { - buf.Close() - } - d.bufs = nil + err := context.Cause(d.ctx) + if err == nil { + if atomic.LoadInt64(&d.written) != d.params.Range.Length { + err = fmt.Errorf("interrupted") + } + } else if errors.Is(err, context.Canceled) { + err = nil + } + d.cancel(err) + d.mu.Lock() + defer d.mu.Unlock() + if d.bufMap != nil { + for _, buf := range d.bufMap { + _ = buf.Close() + } + d.bufMap = nil + } + if d.hc != nil { + _ = d.hc.Close() + d.hc = nil + } + if d.maxPos != 0 { + d.maxPos = 0 + close(d.chunkCh) if d.concurrency > 0 { d.concurrency = -d.concurrency } @@ -300,49 +323,49 @@ func (d *downloader) interrupt() error { } return err } -func (d *downloader) getBuf(id int) (b *Buf) { - return d.bufs[id%len(d.bufs)] +func (d *downloader) popBuf(id int) *buffer.PipeBuffer { + br := d.bufMap[id] + if br != nil { + delete(d.bufMap, id) + d.bufMap[-1] = br // -1 保存最后一次取出的buf用来关闭 + } + return br } -func (d *downloader) finishBuf(id int) (isLast bool, nextBuf *Buf) { - id++ - if id >= d.maxPart { - return true, nil + +func (d *downloader) finishBuf(nextId int, prev *buffer.PipeBuffer) (next *buffer.PipeBuffer) { + atomic.StoreInt64(&d.readingID, int64(nextId)) + + d.mu.Lock() + shouldSendTask := d.bufMap[d.nextChunk] == nil + if shouldSendTask { + d.bufMap[d.nextChunk] = prev } + d.mu.Unlock() - _ = d.sendChunkTask(false) + if shouldSendTask { + _ = d.sendChunkTask(false) + } else { + _ = prev.Close() + } - d.readingID = id - return false, d.getBuf(id) + d.mu.Lock() + defer d.mu.Unlock() + return d.popBuf(nextId) } // downloadPart is an individual goroutine worker reading from the ch channel // and performing Http request on the data with a given byte range. func (d *downloader) downloadPart() { - defer d.concurrencyFinish() + defer d.cfg.ConcurrencyLimit.Release() for { select { case <-d.ctx.Done(): return - case c, ok := <-d.chunkChannel: + case c, ok := <-d.chunkCh: if !ok { return } - if d.getErr() != nil { - // Drain the channel if there is an error, to prevent deadlocking - // of download producer. - return - } - if err := d.downloadChunk(&c); err != nil { - if err == errCancelConcurrency { - return - } - if err == context.Canceled { - if e := context.Cause(d.ctx); e != nil { - err = e - } - } - d.setErr(err) - d.cancel(err) + if !d.downloadChunk(&c) { return } } @@ -350,26 +373,20 @@ func (d *downloader) downloadPart() { } // downloadChunk downloads the chunk -func (d *downloader) downloadChunk(ch *chunk) error { +func (d *downloader) downloadChunk(ch *chunk) bool { log.Debugf("start chunk_%d, %+v", ch.id, ch) params := d.getParamsFromChunk(ch) - var n int64 var err error for retry := 0; retry <= d.cfg.PartBodyMaxRetries; retry++ { - if d.getErr() != nil { - return nil - } + var n int64 n, err = d.tryDownloadChunk(params, ch) if err == nil { d.incrWritten(n) log.Debugf("chunk_%d downloaded", ch.id) - break + return true } - if d.getErr() != nil { - return nil - } - if utils.IsCanceled(d.ctx) { - return d.ctx.Err() + if d.ctx.Err() != nil { + return false } // Check if the returned error is an errNeedRetry. // If this occurs we unwrap the err to set the underlying error @@ -389,17 +406,31 @@ func (d *downloader) downloadChunk(ch *chunk) error { ch.id, params.URL, retry, err) } else if err == errInfiniteRetry { retry-- - continue + } else if err == errCancelConcurrency { + return false // 取消一个的并发 } else { break } } + if err != nil { + d.cancel(err) // 取消所有的并发 + } + return false +} - return err +func (d *downloader) delay(ti time.Duration) bool { + t := time.NewTimer(ti) + select { + case <-d.ctx.Done(): + t.Stop() + return false + case <-t.C: + return true + } } -var errCancelConcurrency = errors.New("cancel concurrency") -var errInfiniteRetry = errors.New("infinite retry") +var errCancelConcurrency = errors.New("") +var errInfiniteRetry = errors.New("") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) @@ -420,15 +451,17 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int case http.StatusServiceUnavailable: case http.StatusGatewayTimeout: } - <-time.After(time.Millisecond * 200) - return 0, &errNeedRetry{err: err} + if !d.delay(time.Millisecond * time.Duration(rand.Uint32N(300)+200)) { + return 0, errCancelConcurrency + } + return 0, &errNeedRetry{err} } // 来到这 说明第1个分片下载 连接成功了 // 后续分片下载出错都当超载处理 log.Debugf("err chunk_%d, try downloading:%v", ch.id, err) - d.m.Lock() + d.mu.Lock() isCancelConcurrency := ch.newConcurrency if d.concurrency > 0 { // 取消剩余的并发任务 // 用于计算实际的并发数 @@ -437,18 +470,25 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int } if isCancelConcurrency { d.concurrency-- - d.chunkChannel <- *ch - d.m.Unlock() - return 0, errCancelConcurrency + d.mu.Unlock() + select { + case <-d.ctx.Done(): + return 0, errCancelConcurrency + case d.chunkCh <- *ch: + return 0, errCancelConcurrency + } } - d.m.Unlock() - if ch.id != d.readingID { //正在被读取的优先重试 - d.m2.Lock() - defer d.m2.Unlock() - <-time.After(time.Millisecond * 200) + d.mu.Unlock() + if int64(ch.id) != atomic.LoadInt64(&d.readingID) { //正在被读取的优先重试 + d.delayMu.Lock() + defer d.delayMu.Unlock() + if !d.delay(time.Millisecond * time.Duration(rand.Uint32N(300)+200)) { + return 0, errCancelConcurrency + } } return 0, errInfiniteRetry } + defer resp.Body.Close() //only check file size on the first task if ch.id == 0 { @@ -461,11 +501,11 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { - return n, &errNeedRetry{err: err} + return n, &errNeedRetry{err} } if n != ch.size { err = fmt.Errorf("chunk download size incorrect, expected=%d, got=%d", ch.size, n) - return n, &errNeedRetry{err: err} + return n, &errNeedRetry{err} } return n, nil @@ -497,7 +537,8 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { totalStr := stdpath.Base(contentRange) if totalStr != "*" { - if total, err := strconv.ParseInt(totalStr, 10, 64); err != nil { + var total int64 + if total, err = strconv.ParseInt(totalStr, 10, 64); err != nil { err = fmt.Errorf("failed extracting file size: %s", totalStr) } else { totalBytes = total @@ -510,35 +551,12 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { if totalBytes != d.params.Size && err == nil { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } - if err != nil { - d.setErr(err) - d.cancel(err) - } return err } func (d *downloader) incrWritten(n int64) { - d.m.Lock() - defer d.m.Unlock() - - d.written += n -} - -// getErr is a thread-safe getter for the error object -func (d *downloader) getErr() error { - d.m.Lock() - defer d.m.Unlock() - - return d.err -} - -// setErr is a thread-safe setter for the error object -func (d *downloader) setErr(e error) { - d.m.Lock() - defer d.m.Unlock() - - d.err = e + atomic.AddInt64(&d.written, n) } // Chunk represents a single chunk of data to write by the worker routine. @@ -548,7 +566,7 @@ func (d *downloader) setErr(e error) { type chunk struct { start int64 size int64 - buf *Buf + buf *buffer.PipeBuffer id int newConcurrency bool @@ -587,196 +605,39 @@ type HttpRequestParams struct { Size int64 } type errNeedRetry struct { - err error -} - -func (e *errNeedRetry) Error() string { - return e.err.Error() + error } func (e *errNeedRetry) Unwrap() error { - return e.err -} - -type MultiReadCloser struct { - cfg *cfg - closer closerFunc - finish finishBufFUnc -} - -type cfg struct { - rPos int //current reader position, start from 0 - curBuf *Buf + return e.error } -type closerFunc func() error -type finishBufFUnc func(id int) (isLast bool, buf *Buf) - -// NewMultiReadCloser to save memory, we re-use limited Buf, and feed data to Read() -func NewMultiReadCloser(buf *Buf, c closerFunc, fb finishBufFUnc) *MultiReadCloser { - return &MultiReadCloser{closer: c, finish: fb, cfg: &cfg{curBuf: buf}} +type multiReadCloser struct { + pos int //current reader position, start from 0 + maxPos int + curBuf *buffer.PipeBuffer + d *downloader } -func (mr MultiReadCloser) Read(p []byte) (n int, err error) { - if mr.cfg.curBuf == nil { +func (mr *multiReadCloser) Read(p []byte) (n int, err error) { + if mr.curBuf == nil { return 0, io.EOF } - n, err = mr.cfg.curBuf.Read(p) - //log.Debugf("read_%d read current buffer, n=%d ,err=%+v", mr.cfg.rPos, n, err) + n, err = mr.curBuf.Read(p) + // log.Debugf("read_%d read current buffer, n=%d ,err=%+v", mr.rPos, n, err) if err == io.EOF { - log.Debugf("read_%d finished current buffer", mr.cfg.rPos) + log.Debugf("read_%d finished current buffer", mr.pos) - isLast, next := mr.finish(mr.cfg.rPos) - if isLast { + mr.pos++ + if mr.pos >= mr.maxPos { return n, io.EOF } - mr.cfg.curBuf = next - mr.cfg.rPos++ + mr.curBuf = mr.d.finishBuf(mr.pos, mr.curBuf) return n, nil } - if err == context.Canceled { - if e := context.Cause(mr.cfg.curBuf.ctx); e != nil { - err = e - } - } return n, err } -func (mr MultiReadCloser) Close() error { - return mr.closer() -} - -type Buf struct { - size int //expected size - ctx context.Context - offR int - offW int - rw sync.Mutex - buf []byte - mmap bool - - readSignal chan struct{} - readPending bool -} - -// NewBuf is a buffer that can have 1 read & 1 write at the same time. -// when read is faster write, immediately feed data to read after written -func NewBuf(ctx context.Context, maxSize int) *Buf { - br := &Buf{ - ctx: ctx, - size: maxSize, - readSignal: make(chan struct{}, 1), - } - if conf.MmapThreshold > 0 && maxSize >= conf.MmapThreshold { - m, err := mmap.Alloc(maxSize) - if err == nil { - br.buf = m - br.mmap = true - return br - } - } - br.buf = make([]byte, maxSize) - return br -} - -func (br *Buf) Reset(size int) error { - br.rw.Lock() - defer br.rw.Unlock() - if br.buf == nil { - return io.ErrClosedPipe - } - if size > cap(br.buf) { - return fmt.Errorf("reset size %d exceeds max size %d", size, cap(br.buf)) - } - br.size = size - br.offR = 0 - br.offW = 0 - return nil -} -func (br *Buf) Read(p []byte) (int, error) { - if err := br.ctx.Err(); err != nil { - return 0, err - } - if len(p) == 0 { - return 0, nil - } - if br.offR >= br.size { - return 0, io.EOF - } - for { - br.rw.Lock() - if br.buf == nil { - br.rw.Unlock() - return 0, io.ErrClosedPipe - } - - if br.offW < br.offR { - br.rw.Unlock() - return 0, io.ErrUnexpectedEOF - } - if br.offW == br.offR { - br.readPending = true - br.rw.Unlock() - select { - case <-br.ctx.Done(): - return 0, br.ctx.Err() - case _, ok := <-br.readSignal: - if !ok { - return 0, io.ErrClosedPipe - } - continue - } - } - - n := copy(p, br.buf[br.offR:br.offW]) - br.offR += n - br.rw.Unlock() - if n < len(p) && br.offR >= br.size { - return n, io.EOF - } - return n, nil - } -} - -func (br *Buf) Write(p []byte) (int, error) { - if err := br.ctx.Err(); err != nil { - return 0, err - } - if len(p) == 0 { - return 0, nil - } - br.rw.Lock() - defer br.rw.Unlock() - if br.buf == nil { - return 0, io.ErrClosedPipe - } - if br.offW >= br.size { - return 0, io.ErrShortWrite - } - n := copy(br.buf[br.offW:], p[:min(br.size-br.offW, len(p))]) - br.offW += n - if br.readPending { - br.readPending = false - select { - case br.readSignal <- struct{}{}: - default: - } - } - if n < len(p) { - return n, io.ErrShortWrite - } - return n, nil -} - -func (br *Buf) Close() error { - br.rw.Lock() - defer br.rw.Unlock() - var err error - if br.mmap { - err = mmap.Free(br.buf) - br.mmap = false - } - br.buf = nil - close(br.readSignal) - return err +func (mr *multiReadCloser) Close() error { + return mr.d.interrupt() } diff --git a/internal/net/request_test.go b/internal/net/request_test.go index da16a31658..0fdc56eb33 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -11,13 +11,12 @@ import ( "net/http" "sync" "testing" + "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/sirupsen/logrus" ) -var buf22MB = make([]byte, 1024*1024*22) - func containsString(slice []string, val string) bool { for _, item := range slice { if item == val { @@ -27,18 +26,6 @@ func containsString(slice []string, val string) bool { return false } -func dummyHttpRequest(data []byte, p http_range.Range) io.ReadCloser { - - end := p.Start + p.Length - 1 - - if end >= int64(len(data)) { - end = int64(len(data)) - } - - bodyBytes := data[p.Start:end] - return io.NopCloser(bytes.NewReader(bodyBytes)) -} - func TestDownloadOrder(t *testing.T) { buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} downloader, invocations, ranges := newDownloadRangeClient(buff) @@ -67,8 +54,8 @@ func TestDownloadOrder(t *testing.T) { if err != nil { t.Fatalf("expect no error, got %v", err) } - if exp, a := int(length), len(resultBuf); exp != a { - t.Errorf("expect buffer length=%d, got %d", exp, a) + if exp, a := buff[start:start+length2], resultBuf; !bytes.Equal(exp, a) { + t.Errorf("expect buffer %v, got %v", exp, a) } chunkSize := int(length+int64(partSize)-1) / partSize if e, a := chunkSize, *invocations; e != a { @@ -84,7 +71,100 @@ func TestDownloadOrder(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } + +func TestDownloadInterrupt(t *testing.T) { + buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + buff = append(buff, buff...) + downloader, _, _ := newDownloadRangeClient(buff) + con, partSize := 6, 3 + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: 5, + } + }) + + var start, length int64 = 0, int64(len(buff)) + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + ctx, cancel := context.WithCancel(context.Background()) + readCloser, err := d.Download(ctx, req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + _, err = io.CopyN(io.Discard, readCloser, 8) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + cancel() + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } +} + +func TestHighConcurrency(t *testing.T) { + buff := make([]byte, 8<<10) + for i := range len(buff) { + buff[i] = byte(i % 256) + } + downloader, invocations, _ := newDownloadRangeClient(buff) + con, partSize := 64, 100 + concurrencyLimit := uint32(32) + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: concurrencyLimit, + } + }) + + var start, length int64 = 2, 7 << 10 + length2 := length + if length2 == -1 { + length2 = int64(len(buff)) - start + } + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + readCloser, err := d.Download(context.Background(), req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + resultBuf, err := io.ReadAll(readCloser) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + if !bytes.Equal(buff[start:start+length2], resultBuf) { + t.Error("expect buffer content matches, but got mismatch") + } + chunkSize := int(length+int64(partSize)-1) / partSize + if e, a := chunkSize, *invocations; e != a { + t.Errorf("expect %v API calls, got %v", e, a) + } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } + for range 100 { + time.Sleep(10 * time.Millisecond) + if d.ConcurrencyLimit.Limit == concurrencyLimit { + return + } + } + t.Errorf("expect concurrency limit to be %v, got %v", concurrencyLimit, d.ConcurrencyLimit.Limit) +} + func init() { Formatter := new(logrus.TextFormatter) Formatter.TimestampFormat = "2006-01-02T15:04:05.999999999" @@ -136,6 +216,9 @@ func TestDownloadSingle(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } type downloadCaptureClient struct { diff --git a/internal/net/serve.go b/internal/net/serve.go index 6a20460b1b..70b2ed587c 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "mime/multipart" + gonet "net" // 标准库 net 包：用于 Dialer 与 safeTransport 的云元数据 IP 拦截 "net/http" "strconv" "strings" @@ -286,12 +287,45 @@ func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + // 快速连接超时：10秒建立连接，失败快速重试 + DialContext: (&gonet.Dialer{ + Timeout: 10 * time.Second, // TCP握手超时 + KeepAlive: 30 * time.Second, // TCP keep-alive + }).DialContext, + // 响应头超时：15秒等待服务器响应头（平衡API调用与下载检测） + ResponseHeaderTimeout: 15 * time.Second, + // 允许长时间读取数据（无 IdleConnTimeout 限制） } SetProxyIfConfigured(transport) return &http.Client{ - Timeout: time.Hour * 48, - Transport: transport, + Timeout: time.Hour * 48, // 总超时保持48小时（允许大文件慢速下载） + Transport: &safeTransport{base: transport}, } } + +type safeTransport struct { + base http.RoundTripper +} + +func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + host := req.URL.Hostname() + addrs, err := gonet.DefaultResolver.LookupIPAddr(req.Context(), host) + if err != nil || len(addrs) == 0 { + return nil, errors.Wrapf(err, "failed to resolve host: %s", host) + } + for _, addr := range addrs { + if isCloudMetadataIP(addr.IP) { + return nil, ErrCloudMetadataEndpoint + } + } + return t.base.RoundTrip(req) +} + +var ErrCloudMetadataEndpoint = errors.New("access to cloud metadata endpoint is not allowed") + +func isCloudMetadataIP(ip gonet.IP) bool { + ip = ip.To4() + return ip != nil && ip[0] == 169 && ip[1] == 254 && ip[2] == 169 && ip[3] == 254 +} diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index 9e9f702d55..c026ee7eee 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -62,18 +62,16 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - hashs, err := driver115.OfflineDownload(ctx, []string{args.Url}, parentDir) + hashs, err := driver115.OfflineDownload(args.Ctx, []string{args.Url}, parentDir) if err != nil || len(hashs) < 1 { return "", fmt.Errorf("failed to add offline download task: %w", err) } @@ -127,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index d12e02ec59..3e002ff811 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -2,8 +2,16 @@ package _115_open import ( "context" + "encoding/base32" + "encoding/hex" + "errors" "fmt" + "net/url" + "strconv" + "strings" + "time" + sdk "github.com/OpenListTeam/115-sdk-go" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -12,11 +20,34 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + log "github.com/sirupsen/logrus" ) type Open115 struct { } +type offlineTaskClient interface { + OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) + DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error +} + +type offlineTaskDetailClient interface { + OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) +} + +type offlineTaskLimiter interface { + WaitLimit(ctx context.Context) error +} + +func waitOfflineTaskLimit(ctx context.Context, client offlineTaskClient) error { + limiter, ok := client.(offlineTaskLimiter) + if !ok { + return nil + } + return limiter.WaitLimit(ctx) +} + func (o *Open115) Name() string { return "115 Open" } @@ -58,25 +89,481 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } - ctx := context.Background() + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { + return "", err + } - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) + if err != nil { return "", err } + log.Infof("[115_open] AddURL start: temp_dir=%q actual_path=%q parent_id=%q parent_name=%q url=%q", args.TempDir, actualPath, parentDir.GetID(), parentDir.GetName(), args.Url) + logOfflineURLDetails("[115_open] AddURL input", args.Url) - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + hashs, err := addOfflineDownloadTask(args.Ctx, driver115Open, args.Url, parentDir) if err != nil { return "", err } - hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - if err != nil || len(hashs) < 1 { - return "", fmt.Errorf("failed to add offline download task: %w", err) + if len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: no task hash returned") } return hashs[0], nil } +func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, error) { + parentID, parentName := "", "" + if parentDir != nil { + parentID = parentDir.GetID() + parentName = parentDir.GetName() + } + log.Infof("[115_open] addOfflineDownloadTask: parent_id=%q parent_name=%q url=%q", parentID, parentName, url) + logOfflineURLDetails("[115_open] addOfflineDownloadTask target", url) + if err := preCleanDuplicateOfflineTasks(ctx, client, url); err != nil { + return nil, err + } + hashs, addItems, rawResp, err := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] addOfflineDownloadTask first attempt result: hashes=%v err=%v add_items=%d", hashs, err, len(addItems)) + if err == nil { + return hashs, nil + } + if !isDuplicateOfflineTaskError(err) { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] duplicate offline task detected, trying cleanup before retry") + if rawResp != "" { + log.Infof("[115_open] duplicate add response: %s", rawResp) + } + for _, item := range addItems { + log.Infof("[115_open] duplicate add item: state=%v code=%d info_hash=%q url=%q", item.State, item.Code, item.InfoHash, item.URL) + logOfflineURLDetails("[115_open] duplicate add item url", item.URL) + if item.InfoHash == "" { + log.Infof("[115_open] skipping add-response duplicate item: empty info_hash") + continue + } + if item.URL != "" && !offlineTaskURLMatches(item.URL, url) { + log.Infof("[115_open] skipping add-response duplicate item: url mismatch") + continue + } + log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) + } + log.Infof("[115_open] delete duplicate task from add response success: info_hash=%s", item.InfoHash) + waitForOfflineTaskRemoval(ctx, client, item.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after add-response delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after add-response delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) + log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + log.Infof("[115_open] delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after matched delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after matched delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + log.Warnf("[115_open] duplicate offline task detected but no matching task found in offline list") + return nil, fmt.Errorf("failed to add offline download task: %w", err) +} + +func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) + return nil + } + log.Infof("[115_open] pre-add offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + deleted := 0 + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] pre-add duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] pre-add duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + deleted++ + log.Infof("[115_open] pre-add delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + } + if deleted == 0 { + log.Infof("[115_open] pre-add duplicate scan found no matches") + } + return nil +} + +func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, nil, "", err + } + if detailClient, ok := client.(offlineTaskDetailClient); ok { + return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) + } + hashs, err := client.OfflineDownload(ctx, []string{url}, parentDir) + return hashs, nil, "", err +} + +func isDuplicateOfflineTaskError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + return strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") +} + +func offlineTaskURLMatches(taskURL string, rawURL string) bool { + taskVariants := normalizedOfflineTaskURLVariants(taskURL) + rawVariants := normalizedOfflineTaskURLVariants(rawURL) + for candidate := range taskVariants { + if _, ok := rawVariants[candidate]; ok { + return true + } + } + return false +} + +func offlineTaskMatches(task sdk.OfflineTask, rawURL string) bool { + matched, _ := offlineTaskMatchReason(task, rawURL) + return matched +} + +func offlineTaskMatchReason(task sdk.OfflineTask, rawURL string) (bool, string) { + if offlineTaskURLMatches(task.URL, rawURL) { + return true, "url variants matched" + } + if httpURLMatches(task.URL, rawURL) { + return true, "http url host+path matched" + } + rawMagnet := parseMagnetBTIH(rawURL) + if rawMagnet != "" { + taskHash := normalizeInfoHash(task.InfoHash) + if taskHash != "" && taskHash == rawMagnet { + return true, "task info_hash matched raw magnet" + } + taskURLHash := parseMagnetBTIH(task.URL) + if taskURLHash != "" && taskURLHash == rawMagnet { + return true, "task url magnet hash matched" + } + return false, fmt.Sprintf("task magnet hash mismatch: task_info_hash=%q task_url_hash=%q raw_hash=%q", taskHash, taskURLHash, rawMagnet) + } + taskED2K, rawED2K := parseED2KLink(task.URL), parseED2KLink(rawURL) + if taskED2K != nil && rawED2K != nil { + if taskED2K.Hash == rawED2K.Hash { + if taskED2K.Size == rawED2K.Size { + return true, "task url ed2k hash matched" + } + return true, "task url ed2k hash matched despite size mismatch" + } + return false, fmt.Sprintf("task url ed2k mismatch: task=%s raw=%s", taskED2K.String(), rawED2K.String()) + } + if rawED2K == nil { + return false, "raw url is not ed2k and url variants did not match" + } + if normalizeOfflineTaskURL(task.InfoHash) == rawED2K.Hash { + if task.Size == rawED2K.Size { + return true, "task info_hash matched raw ed2k" + } + return true, "task info_hash matched raw ed2k despite size mismatch" + } + taskName := normalizeOfflineTaskURL(task.Name) + if taskName == normalizeOfflineTaskURL(rawED2K.Name) && task.Size == rawED2K.Size { + return true, "task name and size matched raw ed2k" + } + return false, fmt.Sprintf("task name/hash/size mismatch: task_name=%q raw_name=%q task_info_hash=%q raw_hash=%q task_size=%d raw_size=%d", taskName, normalizeOfflineTaskURL(rawED2K.Name), normalizeOfflineTaskURL(task.InfoHash), rawED2K.Hash, task.Size, rawED2K.Size) +} + +func normalizedOfflineTaskURLVariants(raw string) map[string]struct{} { + variants := map[string]struct{}{} + queue := []string{raw} + for len(queue) > 0 { + current := normalizeOfflineTaskURL(queue[0]) + queue = queue[1:] + if current == "" { + continue + } + if _, ok := variants[current]; ok { + continue + } + variants[current] = struct{}{} + if decoded, err := url.QueryUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + if decoded, err := url.PathUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + } + return variants +} + +func httpURLMatches(taskURL, rawURL string) bool { + taskNormalized := normalizeHTTPURL(taskURL) + rawNormalized := normalizeHTTPURL(rawURL) + if taskNormalized == "" || rawNormalized == "" { + return false + } + return taskNormalized == rawNormalized +} + +func normalizeHTTPURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed == nil { + return "" + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + host := strings.ToLower(parsed.Host) + if host == "" { + return "" + } + path := strings.ToLower(parsed.Path) + path = strings.TrimSuffix(path, "/") + if path == "" { + path = "/" + } + return fmt.Sprintf("%s://%s%s", scheme, host, path) +} + +func normalizeOfflineTaskURL(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimSuffix(normalized, "/") + return strings.ToLower(normalized) +} + +type ed2kLink struct { + Name string + Size int64 + Hash string +} + +func parseED2KLink(raw string) *ed2kLink { + normalized := strings.TrimSpace(raw) + if !strings.HasPrefix(strings.ToLower(normalized), "ed2k://|file|") { + return nil + } + parts := strings.Split(normalized, "|") + if len(parts) < 6 { + return nil + } + name, err := url.PathUnescape(parts[2]) + if err != nil { + name = parts[2] + } + size, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil + } + return &ed2kLink{ + Name: normalizeOfflineTaskURL(name), + Size: size, + Hash: normalizeOfflineTaskURL(parts[4]), + } +} + +func parseMagnetBTIH(raw string) string { + if raw == "" { + return "" + } + lower := strings.ToLower(raw) + idx := strings.Index(lower, "btih:") + if idx == -1 { + return "" + } + candidate := raw[idx+len("btih:"):] + if candidate == "" { + return "" + } + for i, ch := range candidate { + if ch == '&' || ch == '#' || ch == '/' { + candidate = candidate[:i] + break + } + } + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + if decoded, err := url.QueryUnescape(candidate); err == nil { + candidate = decoded + } + return normalizeInfoHash(candidate) +} + +func normalizeInfoHash(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimPrefix(strings.ToLower(normalized), "urn:btih:") + if normalized == "" { + return "" + } + if len(normalized) == 40 && isHexString(normalized) { + return normalized + } + if len(normalized) == 32 && isBase32String(normalized) { + decoded, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(normalized)) + if err == nil && len(decoded) == 20 { + return hex.EncodeToString(decoded) + } + } + return normalized +} + +func isHexString(value string) bool { + for _, ch := range value { + if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') { + continue + } + return false + } + return true +} + +func isBase32String(value string) bool { + for _, ch := range value { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '2' && ch <= '7') { + continue + } + return false + } + return true +} + +func (e *ed2kLink) Equal(other *ed2kLink) bool { + if e == nil || other == nil { + return false + } + return e.Name == other.Name && e.Size == other.Size && e.Hash == other.Hash +} + +func (e *ed2kLink) String() string { + if e == nil { + return "" + } + return fmt.Sprintf("name=%q size=%d hash=%q", e.Name, e.Size, e.Hash) +} + +func logOfflineURLDetails(prefix string, raw string) { + if raw == "" { + log.Infof("%s details: raw is empty", prefix) + return + } + variants := normalizedOfflineTaskURLVariants(raw) + log.Infof("%s details: raw=%q normalized_variants=%v", prefix, raw, mapKeys(variants)) + if parsedMagnet := parseMagnetBTIH(raw); parsedMagnet != "" { + log.Infof("%s details: parsed_magnet_hash=%s", prefix, parsedMagnet) + } + if parsed := parseED2KLink(raw); parsed != nil { + log.Infof("%s details: parsed_ed2k=%s", prefix, parsed.String()) + } +} + +func mapKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} + +func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { + const maxChecks = 3 + for attempt := 1; attempt <= maxChecks; attempt++ { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + log.Warnf("[115_open] post-delete wait limit failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + taskList, err := client.OfflineList(ctx) + if err != nil { + log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + stillExists := false + taskStatus := -999 + taskName := "" + for _, task := range taskList.Tasks { + if normalizeOfflineTaskURL(task.InfoHash) != normalizeOfflineTaskURL(infoHash) { + continue + } + stillExists = true + taskStatus = task.Status + taskName = task.Name + break + } + log.Infof("[115_open] post-delete check: info_hash=%s attempt=%d exists=%v status=%d name=%q task_count=%d", infoHash, attempt, stillExists, taskStatus, taskName, len(taskList.Tasks)) + if !stillExists { + return + } + if attempt < maxChecks { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } +} + func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { @@ -124,13 +611,15 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = errors.New(t.GetStatus()) } return s, nil } } - s.Err = fmt.Errorf("the task has been deleted") - return nil, nil + // 任务不在列表中，可能已完成或被删除 + s.Progress = 100 + s.Completed = true + return s, nil } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go new file mode 100644 index 0000000000..2e8b947786 --- /dev/null +++ b/internal/offline_download/115_open/client_test.go @@ -0,0 +1,616 @@ +package _115_open + +import ( + "context" + "fmt" + "strings" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type mockOfflineTaskClient struct { + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error + waitLimitFunc func(ctx context.Context) error + waitLimitCalls int +} + +func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return m.offlineDownloadFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if m.offlineDownloadWithDetailsFunc == nil { + hashes, err := m.OfflineDownload(ctx, uris, dstDir) + return hashes, nil, "", err + } + return m.offlineDownloadWithDetailsFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return m.offlineListFunc(ctx) +} + +func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +} + +func (m *mockOfflineTaskClient) WaitLimit(ctx context.Context) error { + m.waitLimitCalls++ + if m.waitLimitFunc != nil { + return m.waitLimitFunc(ctx) + } + return nil +} + +func TestIsDuplicateOfflineTaskError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "code 10008", err: fmt.Errorf("code: 10008"), want: true}, + {name: "chinese duplicate", err: fmt.Errorf("任务重复"), want: true}, + {name: "already exists", err: fmt.Errorf("任务已存在"), want: true}, + {name: "english duplicate", err: fmt.Errorf("duplicate task"), want: true}, + {name: "other", err: fmt.Errorf("network timeout"), want: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isDuplicateOfflineTaskError(tc.err); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskURLMatches(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + taskURL string + rawURL string + want bool + }{ + { + name: "exact match", + taskURL: "ed2k://|file|test.avi|123|ABC|/", + rawURL: "ed2k://|file|test.avi|123|ABC|/", + want: true, + }, + { + name: "percent encoded file name", + taskURL: "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + rawURL: "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + want: true, + }, + { + name: "case and trailing slash normalized", + taskURL: "ED2K://|FILE|TEST.AVI|123|ABC|", + rawURL: "ed2k://|file|test.avi|123|abc|/", + want: true, + }, + { + name: "different link", + taskURL: "ed2k://|file|a.avi|123|ABC|/", + rawURL: "ed2k://|file|b.avi|123|ABC|/", + want: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := offlineTaskURLMatches(tc.taskURL, tc.rawURL); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskMatches(t *testing.T) { + t.Parallel() + + t.Run("match ed2k by parsed fields when task url differs", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + InfoHash: "server-task-hash", + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected task to match by ed2k parsed fields") + } + }) + + t.Run("do not match different ed2k size", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1, + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if offlineTaskMatches(task, rawURL) { + t.Fatal("expected task not to match") + } + }) + + t.Run("magnet still matches by url", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: rawURL, + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by url") + } + }) + + t.Run("match magnet by btih despite noisy tracker", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test&tr=%3C!DOCTYPE%20html%3E", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by btih") + } + }) + + t.Run("match http by host and path", func(t *testing.T) { + t.Parallel() + + rawURL := "https://example.com/files/test.mp4" + task := sdk.OfflineTask{ + URL: "https://EXAMPLE.com/files/test.mp4?token=abc", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected http task to match by host and path") + } + }) +} + +func TestAddOfflineDownloadTask(t *testing.T) { + t.Parallel() + + const ( + testURL = "https://example.com/test.torrent" + firstHash = "hash-1" + staleHash = "hash-stale" + deleteError = "delete failed" + ) + + t.Run("success on first try", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("wait limit applied for add flow", func(t *testing.T) { + t.Parallel() + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.waitLimitCalls < 2 { + t.Fatalf("want wait limit calls >= 2, got %d", client.waitLimitCalls) + } + }) + + t.Run("delete duplicate and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate magnet and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + magnetURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: magnetURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, magnetURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with decoded ed2k url", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + decodedURL := "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: decodedURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with empty task url but matching name and size", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: staleHash, + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate directly from add response info hash", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadWithDetailsFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + callCount++ + if callCount == 1 { + return nil, []sdk.AddOfflineTaskURIsResp{ + {InfoHash: staleHash, URL: testURL}, + }, `{"state":false,"code":10008,"message":"任务已存在","data":[{"info_hash":"hash-stale","url":"` + testURL + `"}]}`, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil, "", nil + }, + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("unexpected fallback OfflineDownload call") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 1 { + t.Fatalf("want 1 delete attempt, got %d", deleteCount) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("duplicate delete failure", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("duplicate task") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return fmt.Errorf(deleteError) + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), deleteError) { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("non duplicate error is returned", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("network timeout") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "network timeout") { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) +} + +func TestOpen115BasicMethods(t *testing.T) { + t.Parallel() + + o := &Open115{} + + if o.Name() != "115 Open" { + t.Fatalf("unexpected name: %s", o.Name()) + } + if o.Items() != nil { + t.Fatal("Items should return nil") + } + msg, err := o.Init() + if err != nil { + t.Fatalf("unexpected init error: %v", err) + } + if msg != "ok" { + t.Fatalf("unexpected init message: %s", msg) + } +} diff --git a/internal/offline_download/123/client.go b/internal/offline_download/123/client.go index 2c4f470489..6138608909 100644 --- a/internal/offline_download/123/client.go +++ b/internal/offline_download/123/client.go @@ -58,15 +58,14 @@ func (*Pan123) AddURL(args *tool.AddUrlArgs) (string, error) { if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") } - ctx := context.Background() - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - taskID, err := driver123.OfflineDownload(ctx, args.Url, parentDir) + taskID, err := driver123.OfflineDownload(args.Ctx, args.Url, parentDir) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/123_open/client.go b/internal/offline_download/123_open/client.go index ce1453c321..997eeabefa 100644 --- a/internal/offline_download/123_open/client.go +++ b/internal/offline_download/123_open/client.go @@ -56,16 +56,15 @@ func (*Open123) AddURL(args *tool.AddUrlArgs) (string, error) { if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") } - ctx := context.Background() - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } cb := setting.GetStr(conf.Pan123OpenOfflineDownloadCallbackUrl) - taskID, err := driver123Open.OfflineDownload(ctx, args.Url, parentDir, cb) + taskID, err := driver123Open.OfflineDownload(args.Ctx, args.Url, parentDir, cb) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index b04435aca1..5c037ff4cb 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -61,6 +62,10 @@ func (a *Aria2) IsReady() bool { } func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { + // aria2 不支持 ed2k 协议，提前检测并返回明确错误 + if strings.HasPrefix(strings.ToLower(args.Url), "ed2k://") { + return "", fmt.Errorf("aria2 does not support ed2k protocol. Please use Thunder/ThunderX/ThunderBrowser tool for ed2k links") + } options := map[string]interface{}{ "dir": args.TempDir, } diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go index 5dcb6d5902..6b3c45a5fd 100644 --- a/internal/offline_download/http/client.go +++ b/internal/offline_download/http/client.go @@ -5,20 +5,19 @@ import ( "math/rand/v2" "net/http" "os" - "path" "path/filepath" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type SimpleHttp struct { - client http.Client } func (s SimpleHttp) Name() string { @@ -63,7 +62,7 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { if streamPut { req.Header.Set("Range", "bytes=0-") } - resp, err := s.client.Do(req) + resp, err := net.HttpClient().Do(req) if err != nil { return err } @@ -73,11 +72,11 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { } filename, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")) if err != nil { - filename = path.Base(resp.Request.URL.Path) + filename, err = sanitizeFilename(resp.Request.URL.Path) } - filename = strings.Trim(filename, "/") - if len(filename) == 0 { - filename = fmt.Sprintf("%s-%d-%x", strings.ReplaceAll(req.URL.Host, ".", "_"), time.Now().UnixMilli(), rand.Uint32()) + if err != nil { + filename = strings.ReplaceAll(req.URL.Host, ":", "_") + filename = fmt.Sprintf("%s-%d-%x", filename, time.Now().UnixMilli(), rand.Uint32()) } fileSize := resp.ContentLength if streamPut { @@ -91,8 +90,14 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { } task.SetTotalBytes(fileSize) // save to temp dir - _ = os.MkdirAll(task.TempDir, os.ModePerm) + if err := os.MkdirAll(task.TempDir, os.ModePerm); err != nil { + return err + } filePath := filepath.Join(task.TempDir, filename) + cleanTempDir := filepath.Clean(task.TempDir) + string(filepath.Separator) + if !strings.HasPrefix(filepath.Clean(filePath)+string(filepath.Separator), cleanTempDir) { + return fmt.Errorf("filename illegal") + } file, err := os.Create(filePath) if err != nil { return err diff --git a/internal/offline_download/http/util.go b/internal/offline_download/http/util.go index eefefec24a..9622c58bed 100644 --- a/internal/offline_download/http/util.go +++ b/internal/offline_download/http/util.go @@ -3,8 +3,22 @@ package http import ( "fmt" "mime" + "path/filepath" + "strings" ) +func sanitizeFilename(raw string) (string, error) { + raw = strings.TrimSpace(raw) + sep := string(filepath.Separator) + raw = strings.ReplaceAll(raw, "/", sep) + raw = strings.ReplaceAll(raw, "\\", sep) + filename := filepath.Base(filepath.Clean(raw)) + if filename == "." || filename == sep || !filepath.IsLocal(filename) { + return "", fmt.Errorf("invalid filename: %q", raw) + } + return filename, nil +} + func parseFilenameFromContentDisposition(contentDisposition string) (string, error) { if contentDisposition == "" { return "", fmt.Errorf("Content-Disposition is empty") @@ -14,8 +28,15 @@ func parseFilenameFromContentDisposition(contentDisposition string) (string, err return "", err } filename := params["filename"] + if filename == "" { + filename = params["filename*"] + } if filename == "" { return "", fmt.Errorf("filename not found in Content-Disposition: [%s]", contentDisposition) } + filename, err = sanitizeFilename(filename) + if err != nil { + return "", fmt.Errorf("invalid filename in Content-Disposition: [%s]", contentDisposition) + } return filename, nil } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index f48ed99519..e223937e7c 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -63,18 +63,16 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "") + t, err := pikpakDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } @@ -129,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { - s.Err = fmt.Errorf(t.Message) + s.Err = fmt.Errorf("%s", t.Message) } return s, nil } diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go index 7cbff89df0..1a134f34a7 100644 --- a/internal/offline_download/thunder/thunder.go +++ b/internal/offline_download/thunder/thunder.go @@ -64,18 +64,16 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") + task, err := thunderDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/thunder_browser/thunder_browser.go b/internal/offline_download/thunder_browser/thunder_browser.go index 9324d7a760..d1c7671221 100644 --- a/internal/offline_download/thunder_browser/thunder_browser.go +++ b/internal/offline_download/thunder_browser/thunder_browser.go @@ -63,13 +63,11 @@ func (t *ThunderBrowser) AddURL(args *tool.AddUrlArgs) (string, error) { return "", err } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } @@ -77,9 +75,9 @@ func (t *ThunderBrowser) AddURL(args *tool.AddUrlArgs) (string, error) { var task *thunder_browser.OfflineTask switch v := storage.(type) { case *thunder_browser.ThunderBrowser: - task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") + task, err = v.OfflineDownload(args.Ctx, args.Url, parentDir, "") case *thunder_browser.ThunderBrowserExpert: - task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") + task, err = v.OfflineDownload(args.Ctx, args.Url, parentDir, "") default: return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderBrowser is supported") } diff --git a/internal/offline_download/thunderx/thunderx.go b/internal/offline_download/thunderx/thunderx.go index 3ba4a36800..9ba4c7eb87 100644 --- a/internal/offline_download/thunderx/thunderx.go +++ b/internal/offline_download/thunderx/thunderx.go @@ -58,18 +58,16 @@ func (t *ThunderX) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderX is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - task, err := thunderXDriver.OfflineDownload(ctx, args.Url, parentDir, "") + task, err := thunderXDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 0f574571ec..33128ccc2c 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,9 +2,11 @@ package tool import ( "context" + "fmt" "net/url" stdpath "path" "path/filepath" + "strings" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" @@ -67,10 +69,28 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } // try putting url if args.Tool == "SimpleHttp" { + if isSimpleHttpSchemeUnsupported(args.URL) { + return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") + } err = tryPutUrl(ctx, args.DstDirPath, args.URL) if err == nil || !errors.Is(err, errs.NotImplement) { return nil, err } + // Fallback to creating a download task when storage lacks native PutURL support. + } + + // ed2k 链接自动路由：如果当前工具不支持 ed2k，自动尝试使用迅雷系工具 + if isEd2kURL(args.URL) { + if !isEd2kCapableTool(args.Tool) { + // 尝试找到一个可用的支持 ed2k 的工具 + fallbackTool, fallbackName := findEd2kCapableTool() + if fallbackTool != nil { + // 使用找到的迅雷工具替代 + args.Tool = fallbackName + } else { + return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + } + } } // get tool @@ -171,3 +191,48 @@ func tryPutUrl(ctx context.Context, path, urlStr string) error { } return fs.PutURL(ctx, path, dstName, urlStr) } + +func isSimpleHttpSchemeUnsupported(urlStr string) bool { + u, err := url.Parse(strings.TrimSpace(urlStr)) + if err != nil || u.Scheme == "" { + return false + } + scheme := strings.ToLower(u.Scheme) + return scheme != "http" && scheme != "https" +} + +// isEd2kURL 检测 URL 是否为 ed2k 协议 +func isEd2kURL(urlStr string) bool { + return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") +} + +// ed2kCapableTools 支持 ed2k 协议的工具列表（迅雷系） +var ed2kCapableTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} + +// isEd2kCapableTool 检查工具是否支持 ed2k 协议 +func isEd2kCapableTool(toolName string) bool { + for _, t := range ed2kCapableTools { + if t == toolName { + return true + } + } + return false +} + +// findEd2kCapableTool 查找一个可用的支持 ed2k 的工具 +func findEd2kCapableTool() (Tool, string) { + for _, name := range ed2kCapableTools { + t, err := Tools.Get(name) + if err != nil { + continue + } + if t.IsReady() { + return t, name + } + // 尝试初始化 + if _, err := t.Init(); err == nil && t.IsReady() { + return t, name + } + } + return nil, "" +} diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 6de05e5b2f..823bac5266 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -1,6 +1,8 @@ package tool import ( + "context" + "github.com/OpenListTeam/OpenList/v4/internal/model" ) @@ -9,6 +11,7 @@ type AddUrlArgs struct { UID string TempDir string Signal chan int + Ctx context.Context } type Status struct { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 5ee6ef4ffd..bc5fe044a1 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -32,6 +32,8 @@ type DownloadTask struct { callStatusRetried int } +var completedOfflineTaskCleanupDelay = time.Second + func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) @@ -54,6 +56,7 @@ func (t *DownloadTask) Run() error { t.Signal = nil }() gid, err := t.tool.AddURL(&AddUrlArgs{ + Ctx: t.Ctx(), Url: t.Url, UID: t.ID, TempDir: t.TempDir, @@ -97,16 +100,7 @@ outer: if t.tool.Name() == "ThunderX" { return nil } - if t.tool.Name() == "115 Cloud" { - // hack for 115 - <-time.After(time.Second * 1) - err := t.tool.Remove(t) - if err != nil { - log.Errorln(err.Error()) - } - return nil - } - if t.tool.Name() == "115 Open" { + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { return nil } if t.tool.Name() == "123 Open" { @@ -147,7 +141,7 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - if t.callStatusRetried > 5 { + if t.callStatusRetried > 10 { return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) } return false, nil @@ -163,6 +157,13 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { + // For 115, remove offline task record before transfer so it gets cleaned up even if transfer fails + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { + <-time.After(completedOfflineTaskCleanupDelay) + if removeErr := t.tool.Remove(t); removeErr != nil { + log.Errorln(removeErr.Error()) + } + } err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } diff --git a/internal/offline_download/tool/download_test.go b/internal/offline_download/tool/download_test.go new file mode 100644 index 0000000000..5303d35aac --- /dev/null +++ b/internal/offline_download/tool/download_test.go @@ -0,0 +1,90 @@ +package tool + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + task2 "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +type mockTool struct { + name string + addURLFunc func(args *AddUrlArgs) (string, error) + removeFunc func(task *DownloadTask) error + statusFunc func(task *DownloadTask) (*Status, error) + runFunc func(task *DownloadTask) error +} + +func (m *mockTool) Name() string { return m.name } + +func (m *mockTool) Items() []model.SettingItem { return nil } + +func (m *mockTool) Init() (string, error) { return "ok", nil } + +func (m *mockTool) IsReady() bool { return true } + +func (m *mockTool) AddURL(args *AddUrlArgs) (string, error) { + return m.addURLFunc(args) +} + +func (m *mockTool) Remove(task *DownloadTask) error { + return m.removeFunc(task) +} + +func (m *mockTool) Status(task *DownloadTask) (*Status, error) { + return m.statusFunc(task) +} + +func (m *mockTool) Run(task *DownloadTask) error { + return m.runFunc(task) +} + +func TestDownloadTaskRun_RemovesCompleted115OpenRecord(t *testing.T) { + previousDelay := completedOfflineTaskCleanupDelay + completedOfflineTaskCleanupDelay = 0 + defer func() { + completedOfflineTaskCleanupDelay = previousDelay + }() + + removeCount := 0 + tool := &mockTool{ + name: "115 Open", + addURLFunc: func(args *AddUrlArgs) (string, error) { + return "gid-1", nil + }, + removeFunc: func(task *DownloadTask) error { + removeCount++ + if task.GID != "gid-1" { + t.Fatalf("unexpected gid: %s", task.GID) + } + return nil + }, + statusFunc: func(task *DownloadTask) (*Status, error) { + return &Status{ + Completed: true, + Status: "completed", + }, nil + }, + runFunc: func(task *DownloadTask) error { + return errs.NotSupport + }, + } + + task := &DownloadTask{ + TaskExtension: task2.TaskExtension{}, + Url: "https://example.com/test.torrent", + DstDirPath: "/115", + TempDir: "/115", + tool: tool, + } + task.SetCtx(context.Background()) + + if err := task.Run(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if removeCount != 1 { + t.Fatalf("want 1 cleanup remove, got %d", removeCount) + } +} diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index fd6b8f4648..7109669ee7 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" @@ -201,8 +203,28 @@ func transferStdFile(t *TransferTask) error { } info, err := rc.Stat() if err != nil { + rc.Close() return errors.Wrapf(err, "failed to get file %s", t.SrcActualPath) } + + // 尝试对天翼云进行秒传（计算 MD5 + sliceMD5） + if rapidObj, rapidErr := tryRapidUpload189(t, rc, info.Size()); rapidErr == nil && rapidObj != nil { + rc.Close() + log.Infof("秒传成功: %s -> %s", t.SrcActualPath, t.DstStorageMp) + return nil + } + + // 秒传失败或不支持，回退到普通上传 + // 重新 seek 到文件开头 + if _, err := rc.Seek(0, 0); err != nil { + rc.Close() + // 重新打开文件 + rc, err = os.Open(t.SrcActualPath) + if err != nil { + return errors.Wrapf(err, "failed to reopen file %s", t.SrcActualPath) + } + } + mimetype := utils.GetMimeType(t.SrcActualPath) s := &stream.FileStream{ Ctx: t.Ctx(), @@ -217,7 +239,12 @@ func transferStdFile(t *TransferTask) error { Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) - return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + if err != nil { + return err + } + + return nil } func removeStdTemp(t *TransferTask) { @@ -341,3 +368,47 @@ func removeObjTemp(t *TransferTask) { log.Errorf("failed to delete temp obj %s, error: %s", t.SrcActualPath, err.Error()) } } + +// tryRapidUpload189 尝试对天翼云进行秒传 +// 通过计算文件的 MD5 来尝试秒传（使用旧版接口） +// 返回上传成功的对象和错误，如果不支持秒传则返回 nil, error +func tryRapidUpload189(t *TransferTask, file *os.File, fileSize int64) (model.Obj, error) { + // 检查目标存储是否是天翼云 PC 驱动 + cloud189PC, ok := t.DstStorage.(*_189pc.Cloud189PC) + if !ok { + return nil, fmt.Errorf("not 189pc storage") + } + + // 计算整文件 MD5（旧接口只需要 fileMD5） + fileMD5, _, err := _189pc.ComputeSliceMD5sFromReader(file, torrent.DefaultPieceSize) + if err != nil { + return nil, fmt.Errorf("计算 MD5 失败: %w", err) + } + + // 获取目标目录 + dstDir, err := op.Get(t.Ctx(), t.DstStorage, t.DstActualPath) + if err != nil { + return nil, fmt.Errorf("获取目标目录失败: %w", err) + } + + // 构造文件名 + fileName := filepath.Base(t.SrcActualPath) + + // 尝试秒传（使用旧接口） + uploadInfo, err := cloud189PC.OldUploadCreate(t.Ctx(), dstDir.GetID(), fileMD5, fileName, fmt.Sprint(fileSize), false) + if err != nil { + return nil, fmt.Errorf("创建上传任务失败: %w", err) + } + + if uploadInfo.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件") + } + + // 秒传成功，提交 + obj, err := cloud189PC.OldUploadCommit(t.Ctx(), uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, false, true) + if err != nil { + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return obj, nil +} diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index f86390fb8c..49e48aa68f 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -9,9 +9,11 @@ import ( "net/url" "strconv" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -84,7 +86,12 @@ func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { } // http url for .torrent file if endpoint.Scheme == "http" || endpoint.Scheme == "https" { - resp, err := http.Get(args.Url) + resp, err := net.RequestHttp( + args.Ctx, + http.MethodGet, + http.Header{"User-Agent": []string{base.UserAgent}}, + args.Url, + ) if err != nil { return "", errors.Wrap(err, "failed to get .torrent file") } @@ -106,7 +113,7 @@ func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { rpcPayload.Filename = &args.Url } - torrent, err := t.client.TorrentAdd(context.TODO(), rpcPayload) + torrent, err := t.client.TorrentAdd(args.Ctx, rpcPayload) if err != nil { return "", err } @@ -123,7 +130,7 @@ func (t *Transmission) Remove(task *tool.DownloadTask) error { if err != nil { return err } - err = t.client.TorrentRemove(context.TODO(), transmissionrpc.TorrentRemovePayload{ + err = t.client.TorrentRemove(task.Ctx(), transmissionrpc.TorrentRemovePayload{ IDs: []int64{gid}, DeleteLocalData: false, }) @@ -135,7 +142,7 @@ func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { if err != nil { return nil, err } - infos, err := t.client.TorrentGetAllFor(context.TODO(), []int64{gid}) + infos, err := t.client.TorrentGetAllFor(task.Ctx(), []int64{gid}) if err != nil { return nil, err } diff --git a/internal/op/driver.go b/internal/op/driver.go index 5b79b0aed6..90f7cc84eb 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -145,6 +145,11 @@ func getMainItems(config driver.Config) []driver.Item { Default: "false", Help: "Disable sign for Download proxy URL", }) + items = append(items, driver.Item{ + Name: "proxy_user_agent", + Type: conf.TypeString, + Help: "Custom User-Agent for proxy download. Empty = passthrough the client's User-Agent.", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", diff --git a/internal/op/fs.go b/internal/op/fs.go index f7b1c45b50..0148521203 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -149,6 +149,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, + HashInfo: utils.NewHashInfo(nil, ""), }, nil case driver.IRootPath: return &model.Object{ @@ -157,6 +158,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") @@ -246,6 +248,8 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } + // SyncClosers 已关闭（文件句柄已关闭），删除缓存条目，重新获取 + Cache.linkCache.DeleteKey(key) } fn := func() (*objWithLink, error) { @@ -261,11 +265,37 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } + + // Set up link refresher for automatic refresh on expiry during long downloads + // This enables all download scenarios to handle link expiration gracefully + if link.Refresher == nil { + storageCopy := storage + pathCopy := path + argsCopy := args + link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + log.Infof("Refreshing download link for: %s", pathCopy) + // Get fresh link directly from storage, bypassing cache + file, err := GetUnwrap(refreshCtx, storageCopy, pathCopy) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file for refresh") + } + newLink, err := storageCopy.Link(refreshCtx, file, argsCopy) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to refresh link") + } + return newLink, file, nil + } + } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) - } else { + } else if link.RequireReference { + // 本地文件等需要引用计数的链接，缓存与文件句柄生命周期绑定 Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) + } else { + // 不需要引用计数（如云盘链接无过期时间），使用默认 TTL，多客户端复用 + Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } @@ -325,9 +355,16 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + if errs.IsObjectNotFound(err) { + // Retry once after a short delay (handles cloud storage API sync delay) + log.Debugf("[op] parent dir [%s] not found immediately after creation, retrying...", parentPath) + time.Sleep(100 * time.Millisecond) + parentDir, err = GetUnwrap(ctx, storage, parentPath) + } + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } } if !parentDir.IsDir() { return nil, errs.NotFolder @@ -360,6 +397,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) @@ -460,6 +498,7 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) if model.ObjHasMask(srcRawObj, model.NoRename) { return errors.WithStack(errs.PermissionDenied) } + oldName := srcRawObj.GetName() srcObj := model.UnwrapObjName(srcRawObj) var newObj model.Obj @@ -477,19 +516,19 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) dirKey := Key(storage, stdpath.Dir(srcPath)) if !srcRawObj.IsDir() { - Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, oldName)) Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) } if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(dirKey); exist { if srcRawObj.IsDir() { - Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.deleteDirectoryTree(stdpath.Join(dirKey, oldName)) } if newObj == nil { newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } newObj = wrapObjName(storage, newObj) - cache.UpdateObject(srcRawObj.GetName(), newObj) + cache.UpdateObject(oldName, newObj) } } @@ -684,6 +723,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -752,6 +792,7 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -778,7 +819,7 @@ func GetDirectUploadTools(storage driver.Driver) []string { return du.GetDirectUploadTools() } -func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { +func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64, overwrite bool) (any, error) { du, ok := storage.(driver.DirectUploader) if !ok { return nil, errors.WithStack(errs.NotImplement) @@ -788,9 +829,15 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver } dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, dstName) - _, err := Get(ctx, storage, dstPath) - if err == nil { - return nil, errors.WithStack(errs.ObjectAlreadyExists) + var err error + if !overwrite { + _, err = Get(ctx, storage, dstPath) + if err == nil { + return nil, errors.WithStack(errs.ObjectAlreadyExists) + } + if !errs.IsObjectNotFound(err) { + return nil, errors.WithMessage(err, "failed to check if object exists") + } } err = MakeDir(ctx, storage, dstDirPath) if err != nil { diff --git a/internal/op/hook.go b/internal/op/hook.go index 5cf01730d2..89a3f9b55b 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -4,8 +4,11 @@ import ( "context" "regexp" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -98,8 +101,28 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { // Storage type StorageHook func(typ string, storage driver.Driver) +type StorageHealthHook func(storage driver.Driver) -var storageHooks = make([]StorageHook, 0) +// StorageCredentialEvent binds an authentication result to the exact Addition +// snapshot used by that request. A storage may be reinitialized with a peer +// credential while an earlier request is still in flight; listeners must never +// let that older result change the newer pair's cluster state. +type StorageCredentialEvent struct { + Storage driver.Driver + Addition string + Modified time.Time +} + +type StorageCredentialHook func(typ string, event StorageCredentialEvent) +type StorageCredentialHealthHook func(event StorageCredentialEvent) + +var ( + storageHooks = make([]StorageHook, 0) + storageHealthHooks = make([]StorageHealthHook, 0) + storageCredentialHooks = make([]StorageCredentialHook, 0) + storageCredentialHealthHooks = make([]StorageCredentialHealthHook, 0) + storageTokenStatusMu sync.Mutex +) func callStorageHooks(typ string, storage driver.Driver) { for _, hook := range storageHooks { @@ -110,3 +133,199 @@ func callStorageHooks(typ string, storage driver.Driver) { func RegisterStorageHook(hook StorageHook) { storageHooks = append(storageHooks, hook) } + +// RegisterStorageHealthHook receives a cheap proof that an authenticated +// request succeeded. It is distinct from StorageHook: callers use it for +// in-memory liveness only, never for database writes or lifecycle work. +func RegisterStorageHealthHook(hook StorageHealthHook) { + storageHealthHooks = append(storageHealthHooks, hook) +} + +// RegisterStorageCredentialHook receives token-valid/token-invalid events +// accompanied by the immutable Addition snapshot used on the wire. +func RegisterStorageCredentialHook(hook StorageCredentialHook) { + storageCredentialHooks = append(storageCredentialHooks, hook) +} + +// RegisterStorageCredentialHealthHook receives authenticated-success proofs +// accompanied by the exact Addition snapshot that succeeded. +func RegisterStorageCredentialHealthHook(hook StorageCredentialHealthHook) { + storageCredentialHealthHooks = append(storageCredentialHealthHooks, hook) +} + +func NotifyStorageTokenHealthy(storage driver.Driver) { + for _, hook := range storageHealthHooks { + hook(storage) + } +} + +// NotifyStorageTokenHealthyWithAddition is the generation-safe version used by +// drivers whose client can outlive a storage reconfiguration. +func NotifyStorageTokenHealthyWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenHealthyWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenHealthyWithSnapshot rejects a late success from an older +// storage generation before it can renew local cluster-health proof. +func NotifyStorageTokenHealthyWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + if !storageSnapshotMatches(storage, addition, modified) { + return + } + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHealthHooks { + hook(event) + } +} + +// NotifyStorageTokenInvalid signals that a storage's credentials were found to be +// invalid/expired while in use (as opposed to successfully refreshed). It fires +// the storage hook with the "token-invalid" type so listeners — notably cluster +// sync — can react by pulling fresh credentials from a healthy peer instead of +// propagating the broken token. Drivers may call this when an API call fails with +// an unrecoverable auth error. +func NotifyStorageTokenInvalid(storage driver.Driver) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenInvalidWithAddition(storage, st.Addition) +} + +// NotifyStorageTokenInvalidWithAddition records a 401 result only while the +// driver still serves the exact pair that made the failed request. A late 401 +// from an old driver generation is therefore harmless after peer recovery has +// installed a newer pair. +func NotifyStorageTokenInvalidWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenInvalidWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenInvalidWithSnapshot records a 401 only while both the +// in-memory storage and its durable row still match the request generation. +func NotifyStorageTokenInvalidWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + st := storage.GetStorage() + if st == nil || st.Disabled || !storageSnapshotMatches(storage, addition, modified) { + return + } + storageTokenStatusMu.Lock() + recorded := markStorageTokenInvalidStatus(storage, addition, modified) + storageTokenStatusMu.Unlock() + if !recorded { + return + } + // Persisting status is edge-triggered, but recovery is not. A newly adopted + // remote candidate can fail while this mount is already marked invalid; that + // failure must still reach cluster recovery, which performs its own per-mount + // queue and candidate de-duplication. + go callStorageHooks("token-invalid", storage) + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHooks { + hook := hook + go hook("token-invalid", event) + } +} + +func markStorageTokenInvalidStatus(storage driver.Driver, addition string, modified time.Time) bool { + st := storage.GetStorage() + if st == nil || st.Disabled || !storageSnapshotMatches(storage, addition, modified) { + return false + } + const invalidStatus = "token invalid" + if st.ID == 0 { + st.SetStatus(invalidStatus) + return true + } + updated, err := db.UpdateStorageStatusIfModified(st.ID, modified, invalidStatus) + if err != nil { + log.Errorf("failed mark storage token invalid: %s", err) + return false + } + if !updated { + return false + } + // The conditional database update proves the durable row still belongs to + // this request generation. Re-check memory before changing its status too. + if !storageSnapshotMatches(storage, addition, modified) { + return false + } + st.SetStatus(invalidStatus) + return true +} + +// NotifyStorageTokenValid signals that a storage's credentials were just proven +// good by a successful authenticated request. It restores a stale failure status +// before firing the storage hook, so listeners — notably cluster sync — can +// (re)share the proven token with peers. Drivers should call this only on a real +// transition or stale-status recovery to avoid per-request churn. +func NotifyStorageTokenValid(storage driver.Driver) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenValidWithAddition(storage, st.Addition) +} + +// NotifyStorageTokenValidWithAddition restores status and emits a lifecycle +// event only when the authenticated request proved the pair currently mounted. +func NotifyStorageTokenValidWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenValidWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenValidWithSnapshot restores status only for the exact +// request generation that completed successfully. +func NotifyStorageTokenValidWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + if !storageSnapshotMatches(storage, addition, modified) { + return + } + storageTokenStatusMu.Lock() + changed := restoreStorageTokenValidStatus(storage, addition, modified) + storageTokenStatusMu.Unlock() + if changed { + go callStorageHooks("token-valid", storage) + } + // Credential proof is distinct from lifecycle status. A successful refresh + // rotates the pair while the mount may remain WORK; cluster sync must still + // receive and publish that newly proven generation. + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHooks { + hook := hook + go hook("token-valid", event) + } +} + +func storageSnapshotMatches(storage driver.Driver, addition string, modified time.Time) bool { + st := storage.GetStorage() + return st != nil && st.Addition == addition && st.Modified.Equal(modified) +} + +func restoreStorageTokenValidStatus(storage driver.Driver, addition string, modified time.Time) bool { + st := storage.GetStorage() + if st == nil || st.Disabled || st.Status == WORK || !storageSnapshotMatches(storage, addition, modified) { + return false + } + if st.ID == 0 { + st.SetStatus(WORK) + return true + } + updated, err := db.UpdateStorageStatusIfModified(st.ID, modified, WORK) + if err != nil { + log.Errorf("failed mark storage token valid: %s", err) + return false + } + if !updated || !storageSnapshotMatches(storage, addition, modified) { + return false + } + st.SetStatus(WORK) + return true +} diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go new file mode 100644 index 0000000000..36a5716d28 --- /dev/null +++ b/internal/op/hook_test.go @@ -0,0 +1,238 @@ +package op_test + +import ( + "context" + "net/http" + "testing" + "time" + + open115 "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type tokenValidStorageDriver struct { + model.Storage + addition struct{} +} + +func (d *tokenValidStorageDriver) Config() driver.Config { + return driver.Config{Name: "TokenValidTest", CheckStatus: true} +} + +func (d *tokenValidStorageDriver) GetAddition() driver.Additional { + return &d.addition +} + +func (d *tokenValidStorageDriver) Init(ctx context.Context) error { return nil } + +func (d *tokenValidStorageDriver) Drop(ctx context.Context) error { return nil } + +func (d *tokenValidStorageDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return nil, nil +} + +func (d *tokenValidStorageDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return &model.Link{Header: http.Header{}}, nil +} + +func TestNotifyStorageTokenValidRestoresStatus(t *testing.T) { + storage := model.Storage{ + Driver: "TokenValidTest", + MountPath: "/token-valid-status", + Status: "old auth error", + Addition: "{}", + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + + d := &tokenValidStorageDriver{Storage: storage} + op.NotifyStorageTokenValid(d) + + if d.GetStorage().Status != op.WORK { + t.Fatalf("in-memory status = %q, want %q", d.GetStorage().Status, op.WORK) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status != op.WORK { + t.Fatalf("persisted status = %q, want %q", got.Status, op.WORK) + } +} + +func TestNotifyStorageTokenValidPublishesProvenPairWhileAlreadyWork(t *testing.T) { + d := &open115.Open115{Storage: model.Storage{ + Driver: "115 Open", + MountPath: "/token-valid-rotated-pair", + Status: op.WORK, + Addition: `{"access_token":"fresh","refresh_token":"rotated"}`, + }} + proof := make(chan op.StorageCredentialEvent, 1) + op.RegisterStorageCredentialHook(func(typ string, event op.StorageCredentialEvent) { + if typ != "token-valid" || event.Storage != d { + return + } + select { + case proof <- event: + default: + } + }) + + op.NotifyStorageTokenValidWithSnapshot(d, d.Storage.Addition, d.Storage.Modified) + + select { + case event := <-proof: + if event.Addition != d.Storage.Addition || !event.Modified.Equal(d.Storage.Modified) { + t.Fatalf("credential proof used the wrong generation: %#v", event) + } + case <-time.After(time.Second): + t.Fatal("a proven rotated pair was not published while status was already WORK") + } +} + +func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { + storage := model.Storage{ + Driver: "TokenInvalidTest", + MountPath: "/token-invalid-status", + Status: op.WORK, + Addition: "{}", + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + + d := &tokenValidStorageDriver{Storage: storage} + op.NotifyStorageTokenInvalid(d) + + if d.GetStorage().Status == op.WORK { + t.Fatal("token-invalid must make the in-memory storage non-work before recovery hooks run") + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status == op.WORK { + t.Fatal("token-invalid must persist a non-work status before peer recovery") + } +} + +// An old client's 401 must be ignored after cluster recovery has installed a +// different pair. This uses the real 115 driver storage type plus real SQLite +// persistence; no test driver or callback mock is involved. +func TestNotifyStorageTokenInvalidWithStaleAdditionKeepsReplacementWorking(t *testing.T) { + newAddition := `{"access_token":"new","refresh_token":"new"}` + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/token-invalid-stale-addition", + Status: op.WORK, + Addition: newAddition, + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + d := &open115.Open115{Storage: storage} + op.NotifyStorageTokenInvalidWithAddition(d, `{"access_token":"old","refresh_token":"old"}`) + if d.GetStorage().Status != op.WORK { + t.Fatalf("stale invalidation changed in-memory status to %q", d.GetStorage().Status) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status != op.WORK { + t.Fatalf("stale invalidation changed persisted status to %q", got.Status) + } +} + +// Addition equality alone is not enough: UpdateStorage can replace a row while +// an old client's callback is racing. The durable Modified generation is the +// compare-and-swap guard that prevents that callback from touching the new row. +func TestNotifyStorageTokenInvalidWithStaleGenerationKeepsReplacementWorking(t *testing.T) { + oldAddition := `{"access_token":"old","refresh_token":"old"}` + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/token-invalid-stale-generation", + Status: op.WORK, + Addition: oldAddition, + Modified: time.Now().Add(-time.Minute), + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + oldGeneration := storage.Modified + newStorage := storage + newStorage.Addition = `{"access_token":"new","refresh_token":"new"}` + newStorage.Modified = time.Now() + if err := db.UpdateStorage(&newStorage); err != nil { + t.Fatalf("UpdateStorage replacement failed: %v", err) + } + + // d deliberately models the old client object whose request began before + // the durable replacement. It is a real 115 driver value, not a fake. + d := &open115.Open115{Storage: storage} + op.NotifyStorageTokenInvalidWithSnapshot(d, oldAddition, oldGeneration) + if d.GetStorage().Status != op.WORK { + t.Fatalf("stale generation changed old in-memory status to %q", d.GetStorage().Status) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Addition != newStorage.Addition || got.Status != op.WORK { + t.Fatalf("stale generation changed replacement row: %#v", got) + } +} + +func TestNotifyStorageTokenInvalidDispatchesRecoveryWhenAlreadyInvalid(t *testing.T) { + d := &tokenValidStorageDriver{Storage: model.Storage{ + Driver: "TokenInvalidTest", + MountPath: "/token-invalid-repeat", + Status: "token invalid", + Addition: "{}", + }} + called := make(chan struct{}, 1) + op.RegisterStorageHook(func(typ string, got driver.Driver) { + if typ != "token-invalid" || got != d { + return + } + select { + case called <- struct{}{}: + default: + } + }) + + op.NotifyStorageTokenInvalid(d) + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("an already-invalid storage must still dispatch recovery") + } +} + +func TestNotifyStorageTokenHealthyDoesNotChangeLifecycleStatus(t *testing.T) { + d := &tokenValidStorageDriver{Storage: model.Storage{Status: op.WORK}} + called := make(chan struct{}, 1) + op.RegisterStorageHealthHook(func(got driver.Driver) { + if got != d { + return + } + select { + case called <- struct{}{}: + default: + } + }) + + op.NotifyStorageTokenHealthy(d) + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("health hook was not invoked") + } + if d.GetStorage().Status != op.WORK { + t.Fatal("a healthy proof must not mutate the storage lifecycle status") + } +} diff --git a/internal/op/loading.go b/internal/op/loading.go new file mode 100644 index 0000000000..e05b5611af --- /dev/null +++ b/internal/op/loading.go @@ -0,0 +1,125 @@ +package op + +import "sync" + +// StorageLoadState is the lifecycle state of a single storage during startup load. +type StorageLoadState string + +const ( + LoadPending StorageLoadState = "pending" + LoadLoading StorageLoadState = "loading" + LoadLoaded StorageLoadState = "loaded" + LoadFailed StorageLoadState = "failed" +) + +func (s StorageLoadState) terminal() bool { + return s == LoadLoaded || s == LoadFailed +} + +// LoadTarget is the minimal description of a storage to be loaded. Kept free of +// model.Storage so the tracker stays trivially testable. +type LoadTarget struct { + MountPath string + Driver string +} + +// StorageLoadInfo is the public, snapshot-safe view of one storage's load state. +type StorageLoadInfo struct { + MountPath string `json:"mount_path"` + Driver string `json:"driver"` + State StorageLoadState `json:"state"` + Error string `json:"error,omitempty"` +} + +// ProgressSnapshot is an immutable view of the overall loading progress, safe to +// serialise to JSON and hand to the frontend status bar. +type ProgressSnapshot struct { + Total int `json:"total"` + Pending int `json:"pending"` + Loading int `json:"loading"` + Loaded int `json:"loaded"` + Failed int `json:"failed"` + Finished bool `json:"finished"` + Items []StorageLoadInfo `json:"items"` +} + +// LoadingProgress tracks per-storage load state during startup. All methods are +// safe for concurrent use, so storages may be loaded in parallel while the HTTP +// API polls Snapshot(). +type LoadingProgress struct { + mu sync.RWMutex + items map[string]*StorageLoadInfo + order []string + started bool +} + +func NewLoadingProgress() *LoadingProgress { + return &LoadingProgress{items: map[string]*StorageLoadInfo{}} +} + +// Begin (re)initialises the tracker with all targets in the pending state. +func (p *LoadingProgress) Begin(targets []LoadTarget) { + p.mu.Lock() + defer p.mu.Unlock() + p.items = make(map[string]*StorageLoadInfo, len(targets)) + p.order = make([]string, 0, len(targets)) + p.started = true + for _, t := range targets { + if _, ok := p.items[t.MountPath]; ok { + continue + } + p.items[t.MountPath] = &StorageLoadInfo{ + MountPath: t.MountPath, + Driver: t.Driver, + State: LoadPending, + } + p.order = append(p.order, t.MountPath) + } +} + +// SetState updates the state (and optional error) of one storage. Unknown mount +// paths are ignored so a late rename can't corrupt the counts. +func (p *LoadingProgress) SetState(mountPath string, state StorageLoadState, errMsg string) { + p.mu.Lock() + defer p.mu.Unlock() + info, ok := p.items[mountPath] + if !ok { + return + } + info.State = state + info.Error = errMsg +} + +// Snapshot returns a deep copy of the current progress. +func (p *LoadingProgress) Snapshot() ProgressSnapshot { + p.mu.RLock() + defer p.mu.RUnlock() + snap := ProgressSnapshot{ + Total: len(p.order), + Items: make([]StorageLoadInfo, 0, len(p.order)), + } + terminal := 0 + for _, mp := range p.order { + info := p.items[mp] + switch info.State { + case LoadPending: + snap.Pending++ + case LoadLoading: + snap.Loading++ + case LoadLoaded: + snap.Loaded++ + case LoadFailed: + snap.Failed++ + } + if info.State.terminal() { + terminal++ + } + snap.Items = append(snap.Items, *info) + } + snap.Finished = p.started && terminal == len(p.order) + return snap +} + +// StorageLoadProgress is the global tracker populated by bootstrap.LoadStorages +// and read by the storage loading-status API. +var StorageLoadProgress = NewLoadingProgress() diff --git a/internal/op/loading_test.go b/internal/op/loading_test.go new file mode 100644 index 0000000000..940afd1497 --- /dev/null +++ b/internal/op/loading_test.go @@ -0,0 +1,113 @@ +package op + +import "testing" + +func items() []LoadTarget { + return []LoadTarget{ + {MountPath: "/a", Driver: "Local"}, + {MountPath: "/b", Driver: "OneDrive"}, + {MountPath: "/c", Driver: "Local"}, + } +} + +func TestLoadingProgressBeginInitialisesPending(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + s := p.Snapshot() + if s.Total != 3 { + t.Fatalf("expected total 3, got %d", s.Total) + } + if s.Pending != 3 { + t.Fatalf("expected 3 pending, got %d", s.Pending) + } + if s.Finished { + t.Fatal("should not be finished right after Begin") + } + if len(s.Items) != 3 || s.Items[0].MountPath != "/a" { + t.Fatalf("expected ordered items, got %+v", s.Items) + } + if s.Items[0].State != LoadPending { + t.Fatalf("expected pending state, got %s", s.Items[0].State) + } +} + +func TestLoadingProgressTransitions(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + + p.SetState("/a", LoadLoading, "") + if got := p.Snapshot().Loading; got != 1 { + t.Fatalf("expected 1 loading, got %d", got) + } + + p.SetState("/a", LoadLoaded, "") + p.SetState("/b", LoadLoaded, "") + s := p.Snapshot() + if s.Loaded != 2 { + t.Fatalf("expected 2 loaded, got %d", s.Loaded) + } + if s.Finished { + t.Fatal("should not be finished with one storage left") + } +} + +func TestLoadingProgressFailedRecordsError(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/a", LoadFailed, "boom") + var found *StorageLoadInfo + for i := range p.Snapshot().Items { + if p.Snapshot().Items[i].MountPath == "/a" { + found = &p.Snapshot().Items[i] + } + } + if found == nil || found.State != LoadFailed || found.Error != "boom" { + t.Fatalf("expected failed with error, got %+v", found) + } + if p.Snapshot().Failed != 1 { + t.Fatalf("expected 1 failed, got %d", p.Snapshot().Failed) + } +} + +func TestLoadingProgressFinishedWhenAllTerminal(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/a", LoadLoaded, "") + p.SetState("/b", LoadFailed, "x") + p.SetState("/c", LoadLoaded, "") + s := p.Snapshot() + if !s.Finished { + t.Fatal("expected finished when all storages reached a terminal state") + } + if s.Loaded != 2 || s.Failed != 1 || s.Pending != 0 || s.Loading != 0 { + t.Fatalf("unexpected counts: %+v", s) + } +} + +func TestLoadingProgressEmptyIsFinished(t *testing.T) { + p := NewLoadingProgress() + p.Begin(nil) + s := p.Snapshot() + if s.Total != 0 || !s.Finished { + t.Fatalf("empty progress should be finished, got %+v", s) + } +} + +func TestLoadingProgressSetStateUnknownMountIsNoop(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/does-not-exist", LoadLoaded, "") + if p.Snapshot().Loaded != 0 { + t.Fatal("setting state on an unknown mount must not change counts") + } +} + +func TestLoadingProgressSnapshotIsCopy(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + s := p.Snapshot() + s.Items[0].State = LoadLoaded // mutate the copy + if p.Snapshot().Items[0].State != LoadPending { + t.Fatal("Snapshot must return a copy; internal state was mutated") + } +} diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e31b..c78f02d155 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -305,6 +305,11 @@ func saveDriverStorage(driver driver.Driver) error { if err != nil { return errors.WithMessage(err, "failed update storage in database") } + // A driver persisting its own state is the canonical "token refreshed / + // rotated" moment. Fire the storage hook so listeners (e.g. cluster sync) + // can propagate the fresh credentials to peer nodes. Cluster sync dedups by + // content hash, so unchanged saves cause no churn. + go callStorageHooks("update", driver) return nil } @@ -368,7 +373,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, }(d) select { case r := <-resultChan: - ret.StorageDetails = r + if r != nil { + ret.StorageDetails = r + } case <-time.After(time.Second): } return ret @@ -419,6 +426,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver Name: name, Modified: v.GetStorage().Modified, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), } if !found { idx := len(files) @@ -483,6 +491,9 @@ func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bo if err != nil { return nil, err } + // Label the details with the storage's driver type so the UI can show + // which kind of storage the current mount is. + ret.DriverName = storage.Config().Name Cache.SetStorageDetails(storage, ret) return ret, nil }) diff --git a/internal/plugin/hooks.go b/internal/plugin/hooks.go new file mode 100644 index 0000000000..7a4af93def --- /dev/null +++ b/internal/plugin/hooks.go @@ -0,0 +1,60 @@ +package plugin + +// Well-known hook points the host fires. Plugins Subscribe to these by name. +// Handlers may read and mutate ctx.Payload; mutations are visible to later +// handlers and to the host. +const ( + // Storage lifecycle. Payload: {"mount_path": string, "driver": string}. + HookStorageCreated = "storage.created" + HookStorageUpdated = "storage.updated" + HookStorageDeleted = "storage.deleted" + + // HookFsObjsUpdated fires after a directory's contents change (upload, move, + // rename, copy, delete, mkdir). Payload: {"path": string, "count": int}. + HookFsObjsUpdated = "fs.objs.updated" + + // HookFsListAfter fires after a directory is listed. + // Payload: {"path": string, "count": int, "provider": string}. + HookFsListAfter = "fs.list.after" + + // HookFsLinkAfter fires after a download link is generated. + // Payload: {"path": string}. + HookFsLinkAfter = "fs.link.after" + + // HookUserLoginAfter fires after a successful login. + // Payload: {"username": string}. + HookUserLoginAfter = "user.login.after" +) + +// Capability functions injected by bootstrap. They live here (rather than the +// plugin package importing internal/op directly) to avoid an import cycle: op +// fires plugin hooks, so plugin must not import op. Nil-safe — an unwired +// capability is simply unavailable to plugins. +var ( + SettingGetter func(key string) string + SettingSetter func(key, value string) error +) + +// FireHook fires a hook on the process-wide plugin registry, if one exists and +// actually has subscribers. It is a cheap no-op otherwise, so callers on hot +// paths (list, link) can invoke it unconditionally without building a payload +// when nothing is listening — guard the payload construction with HasSubscribers. +func FireHook(hook string, payload map[string]any) { + if Default == nil { + return + } + reg := Default.Registry() + if !reg.HasHandlers(hook) { + return + } + _, _ = reg.Fire(hook, payload) +} + +// HasSubscribers reports whether any plugin is listening on a hook. Callers on +// hot paths use it to skip building a payload when there are no subscribers. +func HasSubscribers(hook string) bool { + if Default == nil { + return false + } + return Default.Registry().HasHandlers(hook) +} diff --git a/internal/plugin/interp.go b/internal/plugin/interp.go new file mode 100644 index 0000000000..0caf3cd254 --- /dev/null +++ b/internal/plugin/interp.go @@ -0,0 +1,206 @@ +package plugin + +import ( + "fmt" + "reflect" + "sync" + + log "github.com/sirupsen/logrus" + "github.com/traefik/yaegi/interp" + "github.com/traefik/yaegi/stdlib" +) + +// Runtime is a yaegi-backed host for Go-source plugins. Plugins are plain `.go` +// files interpreted in-process — no compile step, no WASM toolchain. Reloading a +// plugin under a name that already exists atomically replaces it (hot-reload). +// +// Coupling boundary: interpreted plugins may import ONLY this package (for the +// API/HookContext types) plus the Go standard library. The host's internal/* +// packages are never registered with the interpreter, so a plugin literally +// cannot reach into backend internals — keeping plugins decoupled from churn. +// +// Trust model: yaegi blocks `unsafe`, `os/exec`, fork and `os.Exit` by default, +// but it is NOT a hostile-code sandbox. Plugins are admin-installed, trusted code. +type Runtime struct { + mu sync.Mutex + reg *Registry + plugins map[string]*loadedPlugin +} + +// loadedPlugin tracks a live interpreter and its optional teardown callback. +type loadedPlugin struct { + interp *interp.Interpreter + onUnload func() +} + +// API is the surface a backend plugin uses to talk to the host. It is the single +// coupling point between plugins and the server — deliberately small and stable. +type API interface { + // Name returns the plugin's name (its file name without extension). + Name() string + // Subscribe registers handler for a named hook. Lower order runs first. + // Subscribe to the well-known Hook* constants exported by this package. + Subscribe(hook string, order int, handler Handler) + // Log writes a line to the server log, tagged with the plugin's name. + Log(args ...any) + // Logf is the printf-style variant of Log. + Logf(format string, args ...any) + // GetSetting reads a server setting by key ("" if unset/unavailable). + GetSetting(key string) string + // SetSetting persists a server setting. Use a plugin-namespaced key. + SetSetting(key, value string) error +} + +// pluginAPI is the per-plugin API implementation, pre-bound to the plugin's name +// and the shared registry so Subscribe attributes hooks to the right plugin +// (which is what makes UnsubscribeAll-on-reload correct). +type pluginAPI struct { + name string + reg *Registry +} + +func (a *pluginAPI) Name() string { return a.name } + +func (a *pluginAPI) Subscribe(hook string, order int, handler Handler) { + a.reg.Subscribe(a.name, hook, order, handler) +} + +func (a *pluginAPI) Log(args ...any) { + log.Infof("[go-plugin %s] %s", a.name, fmt.Sprint(args...)) +} + +func (a *pluginAPI) Logf(format string, args ...any) { + log.Infof("[go-plugin %s] %s", a.name, fmt.Sprintf(format, args...)) +} + +func (a *pluginAPI) GetSetting(key string) string { + if SettingGetter == nil { + return "" + } + return SettingGetter(key) +} + +func (a *pluginAPI) SetSetting(key, value string) error { + if SettingSetter == nil { + return fmt.Errorf("setting persistence is not available") + } + return SettingSetter(key, value) +} + +// hostSymbols exposes this package's plugin-facing types to interpreted plugins +// under its real import path, so a plugin can: +// +// import plugin "github.com/OpenListTeam/OpenList/v4/internal/plugin" +// +// Only API, Handler and HookContext are exported — nothing else of the host. +var hostSymbols = interp.Exports{ + "github.com/OpenListTeam/OpenList/v4/internal/plugin/plugin": map[string]reflect.Value{ + "API": reflect.ValueOf((*API)(nil)), + "Handler": reflect.ValueOf((*Handler)(nil)), + "HookContext": reflect.ValueOf((*HookContext)(nil)), + // Well-known hook names, so plugins can use the constants instead of + // hard-coding strings. + "HookStorageCreated": reflect.ValueOf(HookStorageCreated), + "HookStorageUpdated": reflect.ValueOf(HookStorageUpdated), + "HookStorageDeleted": reflect.ValueOf(HookStorageDeleted), + "HookFsObjsUpdated": reflect.ValueOf(HookFsObjsUpdated), + "HookFsListAfter": reflect.ValueOf(HookFsListAfter), + "HookFsLinkAfter": reflect.ValueOf(HookFsLinkAfter), + "HookUserLoginAfter": reflect.ValueOf(HookUserLoginAfter), + }, +} + +// NewRuntime builds a runtime that subscribes plugins into reg. +func NewRuntime(reg *Registry) *Runtime { + return &Runtime{reg: reg, plugins: map[string]*loadedPlugin{}} +} + +// Load interprets a Go-source plugin under name and runs its entry point. The +// plugin must declare `package main` and export `func OnLoad(api plugin.API)`. +// It may optionally export `func OnUnload()`, called before the plugin is +// dropped or hot-reloaded. If a plugin with the same name is already loaded its +// teardown runs and its hooks are dropped first, so Load doubles as hot-reload. +func (r *Runtime) Load(name string, source []byte) error { + i := interp.New(interp.Options{}) + if err := i.Use(stdlib.Symbols); err != nil { + return fmt.Errorf("use stdlib: %w", err) + } + if err := i.Use(hostSymbols); err != nil { + return fmt.Errorf("use host symbols: %w", err) + } + if _, err := i.Eval(string(source)); err != nil { + return fmt.Errorf("eval %q: %w", name, err) + } + v, err := i.Eval("main.OnLoad") + if err != nil { + return fmt.Errorf("plugin %q must export func OnLoad(plugin.API): %w", name, err) + } + onLoad, ok := v.Interface().(func(API)) + if !ok { + return fmt.Errorf("plugin %q OnLoad has wrong signature, want func(plugin.API)", name) + } + // Optional teardown hook. + var onUnload func() + if uv, uerr := i.Eval("main.OnUnload"); uerr == nil { + onUnload, _ = uv.Interface().(func()) + } + + // Tear down any previous version before the new one takes over. + r.teardown(name) + + r.mu.Lock() + r.plugins[name] = &loadedPlugin{interp: i, onUnload: onUnload} + r.mu.Unlock() + + onLoad(&pluginAPI{name: name, reg: r.reg}) + return nil +} + +// teardown runs a plugin's OnUnload (if any), drops its hook subscriptions, and +// removes it from the live set. Safe to call for an unknown name. +func (r *Runtime) teardown(name string) { + r.mu.Lock() + lp, ok := r.plugins[name] + delete(r.plugins, name) + r.mu.Unlock() + if !ok { + return + } + if lp.onUnload != nil { + func() { + // A misbehaving teardown must not take down the server. + defer func() { + if rec := recover(); rec != nil { + log.Warnf("[plugin] %q OnUnload panicked: %v", name, rec) + } + }() + lp.onUnload() + }() + } + r.reg.UnsubscribeAll(name) +} + +// Unload removes a plugin: runs its teardown and drops all of its hooks. +func (r *Runtime) Unload(name string) { r.teardown(name) } + +// Loaded reports whether a plugin is currently loaded. +func (r *Runtime) Loaded(name string) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.plugins[name] + return ok +} + +// Close tears down all loaded plugins. +func (r *Runtime) Close() error { + r.mu.Lock() + names := make([]string, 0, len(r.plugins)) + for name := range r.plugins { + names = append(names, name) + } + r.mu.Unlock() + for _, name := range names { + r.teardown(name) + } + return nil +} diff --git a/internal/plugin/interp_test.go b/internal/plugin/interp_test.go new file mode 100644 index 0000000000..fd391e854c --- /dev/null +++ b/internal/plugin/interp_test.go @@ -0,0 +1,140 @@ +package plugin + +import "testing" + +// A minimal, valid plugin: subscribes to a hook and mutates the payload. +const demoPlugin = ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Log("demo loaded") + api.Subscribe("demo", 0, func(c *plug.HookContext) error { + c.Payload["touched"] = "v1" + return nil + }) +} +` + +// Same hook, different payload value — used to verify hot-reload replaces the +// old handler instead of stacking a second one. +const demoPluginV2 = ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Subscribe("demo", 0, func(c *plug.HookContext) error { + c.Payload["touched"] = "v2" + return nil + }) +} +` + +func TestRuntimeLoadAndFire(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("demo", []byte(demoPlugin)); err != nil { + t.Fatalf("Load: %v", err) + } + if !rt.Loaded("demo") { + t.Fatal("expected demo loaded") + } + ctx, errs := reg.Fire("demo", map[string]any{}) + if len(errs) != 0 { + t.Fatalf("unexpected handler errors: %v", errs) + } + if ctx.Payload["touched"] != "v1" { + t.Fatalf("expected payload touched=v1, got %v", ctx.Payload["touched"]) + } +} + +func TestRuntimeHotReloadReplacesHandlers(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("demo", []byte(demoPlugin)); err != nil { + t.Fatalf("first load: %v", err) + } + if err := rt.Load("demo", []byte(demoPluginV2)); err != nil { + t.Fatalf("reload: %v", err) + } + // Exactly one handler must remain, producing v2 (not v1, and not both). + if got := len(reg.Handlers("demo")); got != 1 { + t.Fatalf("expected 1 handler after reload, got %d", got) + } + ctx, _ := reg.Fire("demo", map[string]any{}) + if ctx.Payload["touched"] != "v2" { + t.Fatalf("expected v2 after reload, got %v", ctx.Payload["touched"]) + } +} + +func TestRuntimeUnloadDropsHooks(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + _ = rt.Load("demo", []byte(demoPlugin)) + rt.Unload("demo") + if rt.Loaded("demo") { + t.Fatal("expected demo unloaded") + } + if got := len(reg.Handlers("demo")); got != 0 { + t.Fatalf("expected 0 handlers after unload, got %d", got) + } +} + +func TestRuntimeBadSourceErrors(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("bad", []byte("package main\nthis is not go")); err == nil { + t.Fatal("expected error for invalid plugin source") + } + if rt.Loaded("bad") { + t.Fatal("a plugin that failed to eval must not be considered loaded") + } +} + +// Verifies the exported Hook* constants are usable from interpreted plugins. +func TestRuntimeHookConstantsExposed(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + src := ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Subscribe(plug.HookFsListAfter, 0, func(c *plug.HookContext) error { + c.Payload["seen"] = true + return nil + }) +} +` + if err := rt.Load("c", []byte(src)); err != nil { + t.Fatalf("Load using Hook constant: %v", err) + } + ctx, _ := reg.Fire(HookFsListAfter, map[string]any{}) + if ctx.Payload["seen"] != true { + t.Fatal("expected handler subscribed via Hook constant to fire") + } +} + +func TestRuntimeMissingOnLoadErrors(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + src := "package main\nvar X = 1\n" + if err := rt.Load("noentry", []byte(src)); err == nil { + t.Fatal("expected error when plugin has no OnLoad") + } +} diff --git a/internal/plugin/manage.go b/internal/plugin/manage.go new file mode 100644 index 0000000000..600df59fb8 --- /dev/null +++ b/internal/plugin/manage.go @@ -0,0 +1,159 @@ +package plugin + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// A disabled plugin keeps its source on disk under this suffix so Sync skips it +// (only *.go is loaded). Enabling renames it back to .go. +const disabledSuffix = ".go.disabled" + +// pluginNameRe constrains plugin names to a safe, path-traversal-proof charset. +var pluginNameRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// ValidPluginName reports whether name is an acceptable backend plugin name. +func ValidPluginName(name string) bool { return pluginNameRe.MatchString(name) } + +// PluginInfo describes a backend Go plugin for the management UI. +type PluginInfo struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Loaded bool `json:"loaded"` + Error string `json:"error,omitempty"` +} + +func (m *Manager) enabledPath(name string) string { + return filepath.Join(m.goDir(), name+".go") +} + +func (m *Manager) disabledPath(name string) string { + return filepath.Join(m.goDir(), name+disabledSuffix) +} + +// ListGoPlugins returns every backend plugin on disk (enabled or disabled) with +// its current load status and last load error, sorted by name. +func (m *Manager) ListGoPlugins() []PluginInfo { + entries, _ := os.ReadDir(m.goDir()) + seen := map[string]*PluginInfo{} + for _, e := range entries { + if e.IsDir() { + continue + } + n := e.Name() + switch { + case strings.HasSuffix(n, disabledSuffix): + name := strings.TrimSuffix(n, disabledSuffix) + seen[name] = &PluginInfo{Name: name, Enabled: false} + case strings.HasSuffix(n, ".go"): + name := strings.TrimSuffix(n, ".go") + seen[name] = &PluginInfo{Name: name, Enabled: true} + } + } + m.mu.Lock() + errs := make(map[string]string, len(m.loadErrors)) + for k, v := range m.loadErrors { + errs[k] = v + } + m.mu.Unlock() + + out := make([]PluginInfo, 0, len(seen)) + for name, info := range seen { + info.Loaded = m.runtime.Loaded(name) + info.Error = errs[name] + out = append(out, *info) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// GetGoPluginSource returns a plugin's source (enabled or disabled variant). +func (m *Manager) GetGoPluginSource(name string) (string, error) { + if !ValidPluginName(name) { + return "", fmt.Errorf("invalid plugin name %q", name) + } + if b, err := os.ReadFile(m.enabledPath(name)); err == nil { + return string(b), nil + } + b, err := os.ReadFile(m.disabledPath(name)) + if err != nil { + return "", fmt.Errorf("plugin %q not found", name) + } + return string(b), nil +} + +// SaveGoPlugin writes (creates or overwrites) a plugin's source and loads it +// immediately. Writing to a currently-disabled plugin keeps it disabled. A +// returned error means the write failed; a successful write whose code fails to +// load still returns nil — inspect the load error via ListGoPlugins. +func (m *Manager) SaveGoPlugin(name, source string) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q (use letters, digits, '-' or '_')", name) + } + if err := os.MkdirAll(m.goDir(), 0o755); err != nil { + return err + } + // Preserve enabled/disabled state when overwriting an existing plugin. + target := m.enabledPath(name) + if _, err := os.Stat(m.disabledPath(name)); err == nil { + if _, err := os.Stat(target); err != nil { + target = m.disabledPath(name) + } + } + if err := os.WriteFile(target, []byte(source), 0o644); err != nil { + return err + } + // Reconcile immediately so the change takes effect without waiting for the + // next watcher tick. + _, _ = m.Sync() + return nil +} + +// DeleteGoPlugin removes a plugin (both variants) and unloads it. +func (m *Manager) DeleteGoPlugin(name string) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q", name) + } + removed := false + for _, p := range []string{m.enabledPath(name), m.disabledPath(name)} { + if err := os.Remove(p); err == nil { + removed = true + } else if !os.IsNotExist(err) { + return err + } + } + if !removed { + return fmt.Errorf("plugin %q not found", name) + } + _, _ = m.Sync() + return nil +} + +// SetGoPluginEnabled enables or disables a plugin by renaming its file between +// .go and .go.disabled, then reconciles. +func (m *Manager) SetGoPluginEnabled(name string, enabled bool) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q", name) + } + from, to := m.disabledPath(name), m.enabledPath(name) + if !enabled { + from, to = m.enabledPath(name), m.disabledPath(name) + } + if _, err := os.Stat(from); err != nil { + // Already in the desired state (or missing) — treat as a no-op success + // if the target exists, else report not found. + if _, err2 := os.Stat(to); err2 == nil { + return nil + } + return fmt.Errorf("plugin %q not found", name) + } + if err := os.Rename(from, to); err != nil { + return err + } + _, _ = m.Sync() + return nil +} diff --git a/internal/plugin/manage_test.go b/internal/plugin/manage_test.go new file mode 100644 index 0000000000..97f90ac71c --- /dev/null +++ b/internal/plugin/manage_test.go @@ -0,0 +1,115 @@ +package plugin + +import "testing" + +func findInfo(infos []PluginInfo, name string) (PluginInfo, bool) { + for _, i := range infos { + if i.Name == name { + return i, true + } + } + return PluginInfo{}, false +} + +func TestSaveListAndLoad(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatalf("SaveGoPlugin: %v", err) + } + info, ok := findInfo(m.ListGoPlugins(), "demo") + if !ok { + t.Fatal("expected demo in plugin list") + } + if !info.Enabled || !info.Loaded || info.Error != "" { + t.Fatalf("expected demo enabled+loaded+no-error, got %+v", info) + } + // Source round-trips. + src, err := m.GetGoPluginSource("demo") + if err != nil || src != demoPlugin { + t.Fatalf("GetGoPluginSource mismatch: err=%v", err) + } +} + +func TestSaveInvalidName(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("../evil", demoPlugin); err == nil { + t.Fatal("expected invalid name to be rejected") + } +} + +func TestSaveBrokenSourceReportsError(t *testing.T) { + m := newTestManager(t) + // Write succeeds, but the code fails to load → recorded as an error, not loaded. + if err := m.SaveGoPlugin("broken", "package main\nnot valid go"); err != nil { + t.Fatalf("SaveGoPlugin (write) should succeed: %v", err) + } + info, ok := findInfo(m.ListGoPlugins(), "broken") + if !ok { + t.Fatal("expected broken plugin listed") + } + if info.Loaded { + t.Fatal("broken plugin must not be loaded") + } + if info.Error == "" { + t.Fatal("expected a load error to be reported") + } +} + +func TestEnableDisable(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatal(err) + } + if err := m.SetGoPluginEnabled("demo", false); err != nil { + t.Fatalf("disable: %v", err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after disable") + } + info, _ := findInfo(m.ListGoPlugins(), "demo") + if info.Enabled { + t.Fatal("expected demo to be marked disabled") + } + if err := m.SetGoPluginEnabled("demo", true); err != nil { + t.Fatalf("enable: %v", err) + } + if !m.runtime.Loaded("demo") { + t.Fatal("expected demo loaded after re-enable") + } +} + +func TestDeleteUnloads(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatal(err) + } + if err := m.DeleteGoPlugin("demo"); err != nil { + t.Fatalf("delete: %v", err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after delete") + } + if _, ok := findInfo(m.ListGoPlugins(), "demo"); ok { + t.Fatal("expected demo gone from list") + } +} + +func TestGetSettingCapability(t *testing.T) { + // The API exposes injected capabilities; verify the nil-safe defaults and a + // wired getter. + api := &pluginAPI{name: "x", reg: NewRegistry()} + SettingGetter = nil + if api.GetSetting("k") != "" { + t.Fatal("expected empty string when no getter is wired") + } + SettingGetter = func(key string) string { + if key == "k" { + return "v" + } + return "" + } + defer func() { SettingGetter = nil }() + if api.GetSetting("k") != "v" { + t.Fatal("expected wired getter to return v") + } +} diff --git a/internal/plugin/manager.go b/internal/plugin/manager.go new file mode 100644 index 0000000000..2aa0303d54 --- /dev/null +++ b/internal/plugin/manager.go @@ -0,0 +1,225 @@ +package plugin + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// ManifestEntry describes a frontend (JS) plugin the backend serves to the web UI. +type ManifestEntry struct { + ID string `json:"id"` + URL string `json:"url"` +} + +// Manifest is the payload the frontend fetches to hot-load JS plugins. +type Manifest struct { + Plugins []ManifestEntry `json:"plugins"` +} + +type fileState struct { + mod time.Time + size int64 +} + +// Manager watches a plugins directory and keeps the yaegi Runtime in sync with +// the .go source files on disk, and builds the frontend manifest from the JS files. +// +// Layout under Root: +// +// /go/*.go backend Go-source plugins (yaegi) +// /frontend/*.js frontend JS plugins (served via AssetURLBase) +type Manager struct { + Root string + AssetURLBase string // e.g. "/api/plugin/asset" + runtime *Runtime + registry *Registry + mu sync.Mutex + known map[string]fileState // go-plugin path -> state + loadErrors map[string]string // plugin name -> last load error (for the UI) + stop chan struct{} + stopOnce sync.Once +} + +// Default is the process-wide manager, set during bootstrap. HTTP handlers read +// it; it may be nil if plugin support failed to initialise. +var Default *Manager + +func NewManager(ctx context.Context, root, assetURLBase string) (*Manager, error) { + _ = ctx // kept for signature/back-compat; the yaegi runtime needs no context + reg := NewRegistry() + return &Manager{ + Root: root, + AssetURLBase: strings.TrimRight(assetURLBase, "/"), + runtime: NewRuntime(reg), + registry: reg, + known: map[string]fileState{}, + loadErrors: map[string]string{}, + stop: make(chan struct{}), + }, nil +} + +func (m *Manager) Registry() *Registry { return m.registry } +func (m *Manager) Runtime() *Runtime { return m.runtime } + +func (m *Manager) goDir() string { return filepath.Join(m.Root, "go") } +func (m *Manager) frontendDir() string { return filepath.Join(m.Root, "frontend") } + +func nameFromPath(path, ext string) string { + return strings.TrimSuffix(filepath.Base(path), ext) +} + +// Sync reconciles loaded plugins with the .go files on disk: new or changed +// files are (re)loaded, deleted files are unloaded. Returns the names that were +// loaded/reloaded this pass. A missing directory is treated as empty. +func (m *Manager) Sync() ([]string, error) { + entries, err := os.ReadDir(m.goDir()) + if err != nil { + if os.IsNotExist(err) { + m.unloadMissing(map[string]bool{}) + return nil, nil + } + return nil, err + } + + present := map[string]bool{} + var changed []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") { + continue + } + path := filepath.Join(m.goDir(), e.Name()) + info, statErr := e.Info() + if statErr != nil { + continue + } + present[path] = true + + m.mu.Lock() + prev, ok := m.known[path] + unchanged := ok && prev.mod.Equal(info.ModTime()) && prev.size == info.Size() + m.mu.Unlock() + if unchanged { + continue + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + log.Warnf("[plugin] read %s: %v", path, readErr) + continue + } + name := nameFromPath(path, ".go") + // Runtime.Load drops the old version's hooks before re-subscribing, so it + // is safe to call repeatedly for hot-reload. + if err := m.runtime.Load(name, data); err != nil { + log.Warnf("[plugin] load %s: %v", name, err) + // Surface the error to the management UI, but still remember the + // file state so a syntactically-broken plugin isn't retried every + // tick — it retries when the file is edited (mtime/size changes). + m.mu.Lock() + m.loadErrors[name] = err.Error() + m.known[path] = fileState{mod: info.ModTime(), size: info.Size()} + m.mu.Unlock() + continue + } + m.mu.Lock() + delete(m.loadErrors, name) + m.known[path] = fileState{mod: info.ModTime(), size: info.Size()} + m.mu.Unlock() + changed = append(changed, name) + log.Infof("[plugin] loaded go plugin %q", name) + } + + m.unloadMissing(present) + return changed, nil +} + +func (m *Manager) unloadMissing(present map[string]bool) { + m.mu.Lock() + var gone []string + for path := range m.known { + if !present[path] { + gone = append(gone, path) + } + } + for _, path := range gone { + delete(m.known, path) + } + m.mu.Unlock() + + for _, path := range gone { + name := nameFromPath(path, ".go") + m.runtime.Unload(name) // also drops the plugin's hook subscriptions + m.mu.Lock() + delete(m.loadErrors, name) + m.mu.Unlock() + log.Infof("[plugin] unloaded go plugin %q", name) + } +} + +// FrontendManifest scans the frontend dir for .js plugins and returns the +// manifest the web UI fetches. A missing dir yields an empty manifest. +func (m *Manager) FrontendManifest() Manifest { + manifest := Manifest{Plugins: []ManifestEntry{}} + entries, err := os.ReadDir(m.frontendDir()) + if err != nil { + return manifest + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".js") { + continue + } + id := nameFromPath(e.Name(), ".js") + manifest.Plugins = append(manifest.Plugins, ManifestEntry{ + ID: id, + URL: m.AssetURLBase + "/" + e.Name(), + }) + } + return manifest +} + +// AssetPath resolves a requested asset filename to a path inside the frontend +// dir, rejecting any traversal outside it. Returns ("", false) when unsafe. +func (m *Manager) AssetPath(name string) (string, bool) { + if name == "" || strings.Contains(name, "..") || filepath.IsAbs(name) { + return "", false + } + full := filepath.Join(m.frontendDir(), filepath.Clean(name)) + rel, err := filepath.Rel(m.frontendDir(), full) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + return full, true +} + +// Start runs Sync immediately and then every interval until Close, for hot-reload. +func (m *Manager) Start(interval time.Duration) { + if _, err := m.Sync(); err != nil { + log.Warnf("[plugin] initial sync: %v", err) + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-m.stop: + return + case <-ticker.C: + if _, err := m.Sync(); err != nil { + log.Warnf("[plugin] sync: %v", err) + } + } + } + }() +} + +// Close stops the watcher and tears down the runtime. +func (m *Manager) Close() error { + m.stopOnce.Do(func() { close(m.stop) }) + return m.runtime.Close() +} diff --git a/internal/plugin/manager_test.go b/internal/plugin/manager_test.go new file mode 100644 index 0000000000..773a460732 --- /dev/null +++ b/internal/plugin/manager_test.go @@ -0,0 +1,157 @@ +package plugin + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +func writeGo(t *testing.T, dir, name, source string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(source), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func newTestManager(t *testing.T) *Manager { + t.Helper() + m, err := NewManager(context.Background(), t.TempDir(), "/api/plugin/asset") + if err != nil { + t.Fatalf("NewManager: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + return m +} + +func TestManagerSyncLoadsGo(t *testing.T) { + m := newTestManager(t) + writeGo(t, m.goDir(), "demo.go", demoPlugin) + + changed, err := m.Sync() + if err != nil { + t.Fatalf("Sync: %v", err) + } + if len(changed) != 1 || changed[0] != "demo" { + t.Fatalf("expected [demo] loaded, got %v", changed) + } + if !m.runtime.Loaded("demo") { + t.Fatal("expected demo loaded in runtime") + } + // The loaded plugin's hook must actually fire through the manager's registry. + ctx, errs := m.registry.Fire("demo", map[string]any{}) + if len(errs) != 0 || ctx.Payload["touched"] != "v1" { + t.Fatalf("expected touched=v1, got %v errs=%v", ctx.Payload["touched"], errs) + } +} + +func TestManagerSyncMissingDirIsEmpty(t *testing.T) { + m := newTestManager(t) + changed, err := m.Sync() + if err != nil { + t.Fatalf("expected nil error for missing dir, got %v", err) + } + if len(changed) != 0 { + t.Fatalf("expected no changes, got %v", changed) + } +} + +func TestManagerSyncUnchangedNotReloaded(t *testing.T) { + m := newTestManager(t) + writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + changed, err := m.Sync() + if err != nil { + t.Fatal(err) + } + if len(changed) != 0 { + t.Fatalf("expected no reload on unchanged file, got %v", changed) + } +} + +func TestManagerSyncUnloadsDeleted(t *testing.T) { + m := newTestManager(t) + path := writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after file removed") + } + if got := len(m.registry.Handlers("demo")); got != 0 { + t.Fatalf("expected hooks dropped after unload, got %d", got) + } +} + +func TestManagerSyncReloadsChanged(t *testing.T) { + m := newTestManager(t) + path := writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + // Bump mtime forward so the change is detected deterministically. + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(path, future, future); err != nil { + t.Fatal(err) + } + changed, err := m.Sync() + if err != nil { + t.Fatal(err) + } + if len(changed) != 1 || changed[0] != "demo" { + t.Fatalf("expected demo reloaded, got %v", changed) + } +} + +func TestManagerFrontendManifest(t *testing.T) { + m := newTestManager(t) + dir := m.frontendDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "hello.js"), []byte("export default {}"), 0o644); err != nil { + t.Fatal(err) + } + mf := m.FrontendManifest() + if len(mf.Plugins) != 1 { + t.Fatalf("expected 1 manifest entry, got %d", len(mf.Plugins)) + } + if mf.Plugins[0].ID != "hello" { + t.Fatalf("expected id hello, got %s", mf.Plugins[0].ID) + } + if mf.Plugins[0].URL != "/api/plugin/asset/hello.js" { + t.Fatalf("unexpected url %s", mf.Plugins[0].URL) + } +} + +func TestManagerFrontendManifestEmptyWhenNoDir(t *testing.T) { + m := newTestManager(t) + mf := m.FrontendManifest() + if mf.Plugins == nil || len(mf.Plugins) != 0 { + t.Fatalf("expected empty (non-nil) manifest, got %v", mf.Plugins) + } +} + +func TestManagerAssetPathRejectsTraversal(t *testing.T) { + m := newTestManager(t) + if _, ok := m.AssetPath("../../etc/passwd"); ok { + t.Fatal("expected traversal to be rejected") + } + if _, ok := m.AssetPath("ok.js"); !ok { + t.Fatal("expected normal filename to be accepted") + } +} diff --git a/internal/plugin/registry.go b/internal/plugin/registry.go new file mode 100644 index 0000000000..04d58c88aa --- /dev/null +++ b/internal/plugin/registry.go @@ -0,0 +1,120 @@ +// Package plugin provides a cross-platform, hot-reloadable plugin system. +// +// The architecture has two halves: +// - A hook Registry (this file): host code fires named hooks at well-known +// points; plugins subscribe to the hooks they care about. The set of hook +// points is host-provided, but which hooks a plugin subscribes to — and the +// plugin's logic — is fully dynamic and reloadable. +// - A wazero-based WASM Runtime (wasm.go): runs plugin logic in-process, +// sandboxed, on every platform, and can be reloaded without restarting. +package plugin + +import ( + "sort" + "sync" +) + +// HookContext carries the data for a fired hook. Handlers may read and mutate +// Payload; mutations are visible to later handlers and to the host. +type HookContext struct { + Hook string + Payload map[string]any +} + +// Handler reacts to a fired hook. Returning an error is logged but does not stop +// other handlers from running. +type Handler func(ctx *HookContext) error + +type subscription struct { + pluginID string + hook string + order int + handler Handler +} + +// Registry is a concurrency-safe hook bus. The zero value is not usable; call +// NewRegistry. +type Registry struct { + mu sync.RWMutex + subs []subscription +} + +func NewRegistry() *Registry { + return &Registry{} +} + +// Subscribe registers handler for hook on behalf of pluginID. Lower order runs +// first. A plugin may subscribe to the same hook multiple times. +func (r *Registry) Subscribe(pluginID, hook string, order int, handler Handler) { + r.mu.Lock() + defer r.mu.Unlock() + r.subs = append(r.subs, subscription{pluginID, hook, order, handler}) +} + +// UnsubscribeAll removes every subscription owned by pluginID. This is what makes +// hot-reload safe: drop a plugin's hooks before loading its new version. +func (r *Registry) UnsubscribeAll(pluginID string) { + r.mu.Lock() + defer r.mu.Unlock() + kept := r.subs[:0] + for _, s := range r.subs { + if s.pluginID != pluginID { + kept = append(kept, s) + } + } + // Avoid retaining handlers of removed plugins in the backing array. + for i := len(kept); i < len(r.subs); i++ { + r.subs[i] = subscription{} + } + r.subs = kept +} + +// HasHandlers reports whether at least one handler is subscribed to a hook. It +// is cheap and used on hot paths to avoid building a payload when nothing listens. +func (r *Registry) HasHandlers(hook string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + for _, s := range r.subs { + if s.hook == hook { + return true + } + } + return false +} + +// Handlers returns the ordered handlers subscribed to a hook (mainly for tests). +func (r *Registry) Handlers(hook string) []Handler { + r.mu.RLock() + defer r.mu.RUnlock() + var matched []subscription + for _, s := range r.subs { + if s.hook == hook { + matched = append(matched, s) + } + } + sort.SliceStable(matched, func(i, j int) bool { + return matched[i].order < matched[j].order + }) + out := make([]Handler, len(matched)) + for i, s := range matched { + out[i] = s.handler + } + return out +} + +// Fire invokes every handler subscribed to hook, in order, passing a shared +// context. It returns the (possibly mutated) context and a slice of any errors +// handlers returned. One failing handler never blocks the others. +func (r *Registry) Fire(hook string, payload map[string]any) (*HookContext, []error) { + if payload == nil { + payload = map[string]any{} + } + ctx := &HookContext{Hook: hook, Payload: payload} + var errs []error + for _, h := range r.Handlers(hook) { + if err := h(ctx); err != nil { + errs = append(errs, err) + } + } + return ctx, errs +} diff --git a/internal/plugin/registry_test.go b/internal/plugin/registry_test.go new file mode 100644 index 0000000000..a3452ab1e4 --- /dev/null +++ b/internal/plugin/registry_test.go @@ -0,0 +1,101 @@ +package plugin + +import ( + "errors" + "testing" +) + +func TestRegistrySubscribeAndFire(t *testing.T) { + r := NewRegistry() + var calls []string + r.Subscribe("p1", "fs.list.after", 0, func(ctx *HookContext) error { + calls = append(calls, "p1") + return nil + }) + r.Subscribe("p2", "fs.list.after", 0, func(ctx *HookContext) error { + calls = append(calls, "p2") + return nil + }) + r.Subscribe("p3", "other.hook", 0, func(ctx *HookContext) error { + calls = append(calls, "p3") + return nil + }) + + _, errs := r.Fire("fs.list.after", nil) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(calls) != 2 { + t.Fatalf("expected 2 handlers fired, got %v", calls) + } +} + +func TestRegistryOrder(t *testing.T) { + r := NewRegistry() + var calls []string + r.Subscribe("late", "h", 10, func(*HookContext) error { + calls = append(calls, "late") + return nil + }) + r.Subscribe("early", "h", -5, func(*HookContext) error { + calls = append(calls, "early") + return nil + }) + r.Fire("h", nil) + if len(calls) != 2 || calls[0] != "early" || calls[1] != "late" { + t.Fatalf("expected [early late], got %v", calls) + } +} + +func TestRegistryPayloadMutationVisibleToLaterHandlers(t *testing.T) { + r := NewRegistry() + r.Subscribe("a", "h", 0, func(ctx *HookContext) error { + ctx.Payload["n"] = 1 + return nil + }) + r.Subscribe("b", "h", 1, func(ctx *HookContext) error { + if ctx.Payload["n"] != 1 { + t.Errorf("expected payload from earlier handler, got %v", ctx.Payload["n"]) + } + ctx.Payload["n"] = 2 + return nil + }) + ctx, _ := r.Fire("h", nil) + if ctx.Payload["n"] != 2 { + t.Fatalf("expected final payload 2, got %v", ctx.Payload["n"]) + } +} + +func TestRegistryErrorsDoNotStopOthers(t *testing.T) { + r := NewRegistry() + ran := false + r.Subscribe("bad", "h", 0, func(*HookContext) error { + return errors.New("boom") + }) + r.Subscribe("good", "h", 1, func(*HookContext) error { + ran = true + return nil + }) + _, errs := r.Fire("h", nil) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d", len(errs)) + } + if !ran { + t.Fatal("later handler should still run after an earlier error") + } +} + +func TestRegistryUnsubscribeAll(t *testing.T) { + r := NewRegistry() + r.Subscribe("p1", "h", 0, func(*HookContext) error { return nil }) + r.Subscribe("p1", "h2", 0, func(*HookContext) error { return nil }) + r.Subscribe("p2", "h", 0, func(*HookContext) error { return nil }) + + r.UnsubscribeAll("p1") + if got := len(r.Handlers("h")); got != 1 { + t.Fatalf("expected 1 handler on h after unsubscribe, got %d", got) + } + if got := len(r.Handlers("h2")); got != 0 { + t.Fatalf("expected 0 handlers on h2 after unsubscribe, got %d", got) + } +} diff --git a/internal/stream/section_reader_prefetch_test.go b/internal/stream/section_reader_prefetch_test.go new file mode 100644 index 0000000000..57e14db66b --- /dev/null +++ b/internal/stream/section_reader_prefetch_test.go @@ -0,0 +1,399 @@ +package stream + +import ( + "bytes" + "context" + "crypto/rand" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// signalReader is an io.Reader that records the high-water mark of bytes +// read so tests can deterministically observe how much of the source has +// been consumed at a given moment. It also exposes an optional per-Read +// delay so the test can create a window in which prefetch can run. +type signalReader struct { + data []byte + readPos atomic.Int64 + delay time.Duration + readCnt atomic.Int32 + chunkSig chan int64 // optional: receives high-water mark after each Read +} + +func newSignalReader(data []byte, delay time.Duration) *signalReader { + return &signalReader{data: data, delay: delay, chunkSig: make(chan int64, 1024)} +} + +func (r *signalReader) Read(p []byte) (int, error) { + pos := r.readPos.Load() + if pos >= int64(len(r.data)) { + return 0, io.EOF + } + if r.delay > 0 { + time.Sleep(r.delay) + } + n := copy(p, r.data[pos:]) + newPos := pos + int64(n) + r.readPos.Store(newPos) + r.readCnt.Add(1) + select { + case r.chunkSig <- newPos: + default: + } + return n, nil +} + +// fakeFileStream wraps a signalReader into a FileStreamer-compatible value +// reusing the production FileStream type. +func newFakeFileStream(t *testing.T, data []byte, delay time.Duration) (*FileStream, *signalReader) { + t.Helper() + sr := newSignalReader(data, delay) + fs := &FileStream{ + Ctx: context.Background(), + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Reader: io.NopCloser(sr), + } + return fs, sr +} + +// withStreamConf sets minimal stream/conf values needed for the +// hybridSectionReader path and restores them afterwards. +func withStreamConf(t *testing.T, cacheThreshold, maxBlock uint64) { + t.Helper() + prevCT := conf.AutoMemoryLimit + prevMB := conf.MaxBlockLimit + prevMF := conf.MinFreeMemory + prevConf := conf.Conf + t.Cleanup(func() { + conf.AutoMemoryLimit = prevCT + conf.MaxBlockLimit = prevMB + conf.MinFreeMemory = prevMF + conf.Conf = prevConf + }) + conf.AutoMemoryLimit = cacheThreshold + conf.MaxBlockLimit = maxBlock + conf.MinFreeMemory = 1 // keep memory path enabled + conf.Conf = &conf.Config{} +} + +// TestHybridSectionReader_PrefetchAdvancesSourceAhead verifies that after +// GetSectionReader returns block N, the underlying source has been read +// past the end of block N (i.e., prefetch of block N+1 is in flight or +// already complete). This is the core behavior of Pass 2 prefetch. +func TestHybridSectionReader_PrefetchAdvancesSourceAhead(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*4) + _, _ = rand.Read(data) + + // Per-Read delay so each chunk takes measurable time; this gives + // prefetch a window to advance during the "upload" phase below. + fs, sr := newFakeFileStream(t, data, 5*time.Millisecond) + + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + hsr, ok := ss.(*hybridSectionReader) + if !ok { + t.Fatalf("expected *hybridSectionReader, got %T", ss) + } + _ = hsr + + // Read block 0 + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll block0: %v", err) + } + if !bytes.Equal(got, data[:partSize]) { + t.Fatalf("block0 data mismatch") + } + + // Simulate "upload" time; prefetch should pull block 1 from source. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if sr.readPos.Load() > partSize { + break + } + time.Sleep(2 * time.Millisecond) + } + + if got := sr.readPos.Load(); got <= partSize { + t.Fatalf("expected prefetch to advance source past %d, got %d", partSize, got) + } + + ss.FreeSectionReader(rs) + + // Read block 1 — should be served from prefetch (data must still be correct). + rs2, err := ss.GetSectionReader(partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader(1): %v", err) + } + got2, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll block1: %v", err) + } + if !bytes.Equal(got2, data[partSize:2*partSize]) { + t.Fatalf("block1 data mismatch") + } + ss.FreeSectionReader(rs2) +} + +// TestHybridSectionReader_PrefetchSequentialCorrectness verifies that a +// full sequential walk through the file returns the exact original bytes +// when prefetch is enabled. +func TestHybridSectionReader_PrefetchSequentialCorrectness(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + const partCount = 6 + data := make([]byte, partSize*partCount) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + for i := int64(0); i < partCount; i++ { + off := i * partSize + rs, err := ss.GetSectionReader(off, partSize) + if err != nil { + t.Fatalf("GetSectionReader(%d): %v", i, err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll part %d: %v", i, err) + } + if !bytes.Equal(got, data[off:off+partSize]) { + t.Fatalf("part %d data mismatch", i) + } + ss.FreeSectionReader(rs) + } +} + +// TestHybridSectionReader_PrefetchLastChunkPartial covers the final +// partial chunk: when the file isn't a multiple of partSize, the last +// GetSectionReader call asks for a smaller length than the previous +// (prefetched) call expected. The data returned must still be correct. +func TestHybridSectionReader_PrefetchLastChunkPartial(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + // 2.5 parts: last chunk is partSize/2 + data := make([]byte, partSize*2+partSize/2) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + offsets := []struct{ off, length int64 }{ + {0, partSize}, + {partSize, partSize}, + {partSize * 2, partSize / 2}, + } + for i, p := range offsets { + rs, err := ss.GetSectionReader(p.off, p.length) + if err != nil { + t.Fatalf("GetSectionReader(%d, %d): %v", p.off, p.length, err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll part %d: %v", i, err) + } + if !bytes.Equal(got, data[p.off:p.off+p.length]) { + t.Fatalf("part %d data mismatch", i) + } + ss.FreeSectionReader(rs) + } +} + +// TestHybridSectionReader_PrefetchErrorSurfaces verifies that a read +// error encountered during prefetch is reported on the next +// GetSectionReader call (rather than silently ignored). +func TestHybridSectionReader_PrefetchErrorSurfaces(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*3) + _, _ = rand.Read(data) + + // Wrap the signal reader so reads targeting the second chunk return + // a hard error and zero bytes — simulating mid-chunk download failure. + sr := newSignalReader(data, 0) + fs := &FileStream{ + Ctx: context.Background(), + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Reader: io.NopCloser(readerFunc(func(p []byte) (int, error) { + if sr.readPos.Load() >= partSize { + return 0, io.ErrUnexpectedEOF + } + remaining := partSize - sr.readPos.Load() + if int64(len(p)) > remaining { + p = p[:remaining] + } + return sr.Read(p) + })), + } + + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + // Wait for prefetch to complete (and presumably fail). + time.Sleep(20 * time.Millisecond) + + _, err = ss.GetSectionReader(partSize, partSize) + if err == nil { + t.Fatalf("expected error from prefetch failure, got nil") + } +} + +// TestHybridSectionReader_NoPrefetchAfterEOF makes sure we don't start a +// prefetch goroutine past the end of the file. +func TestHybridSectionReader_NoPrefetchAfterEOF(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*2) + _, _ = rand.Read(data) + + fs, sr := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + for i := int64(0); i < 2; i++ { + rs, err := ss.GetSectionReader(i*partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader(%d): %v", i, err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + } + + // Allow any (incorrect) prefetch to fire. + time.Sleep(20 * time.Millisecond) + + // Source must not have been read past end of file. + if got := sr.readPos.Load(); got != int64(len(data)) { + t.Fatalf("source read pos = %d, want %d (no over-read past EOF)", got, len(data)) + } + + hsr := ss.(*hybridSectionReader) + if hsr.prefetch != nil { + t.Fatalf("expected no in-flight prefetch after final chunk") + } +} + +// TestHybridSectionReader_PrefetchClampedToFileSize ensures the prefetch +// length is clamped to the remaining bytes so the goroutine doesn't +// request more than the file holds. +func TestHybridSectionReader_PrefetchClampedToFileSize(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + // 1.5 parts so prefetch after block 0 must be clamped to partSize/2. + data := make([]byte, partSize+partSize/2) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + rs2, err := ss.GetSectionReader(partSize, partSize/2) + if err != nil { + t.Fatalf("GetSectionReader(1): %v", err) + } + got, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[partSize:]) { + t.Fatalf("partial-tail data mismatch") + } + ss.FreeSectionReader(rs2) +} + +type readerFunc func(p []byte) (int, error) + +func (f readerFunc) Read(p []byte) (int, error) { return f(p) } + +// TestHybridSectionReader_PrefetchHonorsDiscardSection verifies that +// DiscardSection invalidates any pending prefetch and that subsequent +// GetSectionReader calls at the new offset return correct data. +func TestHybridSectionReader_PrefetchHonorsDiscardSection(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*4) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + // Read block 0 (kicks off prefetch of block 1) + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader: %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + // Give prefetch a moment to complete + time.Sleep(20 * time.Millisecond) + + // Caller decides to skip block 1 — calls DiscardSection + if err := ss.DiscardSection(partSize, partSize); err != nil { + t.Fatalf("DiscardSection: %v", err) + } + + // Now read block 2 — must return correct data + rs2, err := ss.GetSectionReader(2*partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader after discard: %v", err) + } + got, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[2*partSize:3*partSize]) { + t.Fatalf("block 2 data mismatch after discard") + } + ss.FreeSectionReader(rs2) +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100e..d7a0936ac4 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -1,6 +1,7 @@ package stream import ( + "bytes" "context" "errors" "fmt" @@ -10,11 +11,11 @@ import ( "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" "go4.org/readerutil" ) @@ -28,8 +29,9 @@ type FileStream struct { Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers size int64 - peekBuff *buffer.Reader oriReader io.Reader // the original reader, used for caching + hc *hcache.HybridCache + peek buffer.SizedReadAtSeeker } func (f *FileStream) GetSize() int64 { @@ -51,15 +53,6 @@ func (f *FileStream) IsForceStreamUpload() bool { return f.ForceStreamUpload } -func (f *FileStream) Close() error { - if f.peekBuff != nil { - f.peekBuff.Reset() - f.oriReader = nil - f.peekBuff = nil - } - return f.Closers.Close() -} - func (f *FileStream) GetExist() model.Obj { return f.Exist } @@ -101,79 +94,57 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } reader := f.Reader - if f.peekBuff != nil { - f.peekBuff.Seek(0, io.SeekStart) + if f.peek != nil { + f.peek.Seek(0, io.SeekStart) if writer != nil { - _, err := utils.CopyWithBuffer(writer, f.peekBuff) + _, err := utils.CopyWithBuffer(writer, f.peek) if err != nil { return nil, err } - f.peekBuff.Seek(0, io.SeekStart) + f.peek.Seek(0, io.SeekStart) } reader = f.oriReader } if writer != nil { reader = io.TeeReader(reader, writer) } + + // 如果文件大小未知，直接缓存到磁盘 if f.GetSize() < 0 { - if f.peekBuff == nil { - f.peekBuff = &buffer.Reader{} - } // 检查是否有数据 buf := []byte{0} n, err := io.ReadFull(reader, buf) - if n > 0 { - f.peekBuff.Append(buf[:n]) - } - if err == io.ErrUnexpectedEOF { - f.size = f.peekBuff.Size() - f.Reader = f.peekBuff - return f.peekBuff, nil + br := bytes.NewReader(buf[:n]) + if err == io.ErrUnexpectedEOF || err == io.EOF { + f.size = br.Size() + f.Reader = br + return br, nil } else if err != nil { return nil, err } - if conf.MaxBufferLimit-n > conf.MmapThreshold && conf.MmapThreshold > 0 { - m, err := mmap.Alloc(conf.MaxBufferLimit - n) - if err == nil { - f.Add(utils.CloseFunc(func() error { - return mmap.Free(m) - })) - n, err = io.ReadFull(reader, m) - if n > 0 { - f.peekBuff.Append(m[:n]) - } - if err == io.ErrUnexpectedEOF { - f.size = f.peekBuff.Size() - f.Reader = f.peekBuff - return f.peekBuff, nil - } else if err != nil { - return nil, err - } - } - } - tmpF, err := utils.CreateTempFile(reader, 0) + tmpF, err := utils.CreateTempFile(io.MultiReader(br, reader), 0) if err != nil { return nil, err } f.Add(utils.CloseFunc(func() error { return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) })) - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) + stat, err := tmpF.Stat() if err != nil { return nil, err } - f.size = peekF.Size() - f.Reader = peekF - return peekF, nil + f.size = stat.Size() + f.Reader = tmpF + return tmpF, nil } if up != nil { cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) *up = model.UpdateProgressWithRange(*up, 50, 100) size := f.GetSize() - if f.peekBuff != nil { - peekSize := f.peekBuff.Size() - cacheProgress(float64(peekSize) / float64(size) * 100) + if f.peek != nil { + peekSize := f.peek.Size() + // cacheProgress(float64(peekSize) / float64(size) * 100) size -= peekSize } reader = &ReaderUpdatingProgress{ @@ -185,12 +156,12 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } } - if f.peekBuff != nil { + if f.oriReader != nil { f.oriReader = reader } else { f.Reader = reader } - return f.cache(f.GetSize()) + return f.ensureCache(f.GetSize()) } func (f *FileStream) GetFile() model.File { @@ -211,7 +182,7 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - cache, err := f.cache(httpRange.Start + httpRange.Length) + cache, err := f.ensureCache(httpRange.Start + httpRange.Length) if err != nil { return nil, err } @@ -224,64 +195,32 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 -func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { - if maxCacheSize > int64(conf.MaxBufferLimit) { - size := f.GetSize() - reader := f.Reader - if f.peekBuff != nil { - size -= f.peekBuff.Size() - reader = f.oriReader - } - tmpF, err := utils.CreateTempFile(reader, size) +func (f *FileStream) ensureCache(size int64) (model.File, error) { + if f.peek == nil { + blockSize := min(size, f.GetSize(), int64(conf.MaxBlockLimit)) + var err error + f.hc, err = hcache.NewHybridCache(uint64(blockSize), uint64(f.GetSize())) if err != nil { return nil, err } - f.Add(utils.CloseFunc(func() error { - return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) - })) - if f.peekBuff != nil { - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) - if err != nil { - return nil, err - } - f.Reader = peekF - return peekF, nil - } - f.Reader = tmpF - return tmpF, nil - } - - if f.peekBuff == nil { - f.peekBuff = &buffer.Reader{} + f.peek = buffer.NewDynamicReadAtSeeker(f.hc) f.oriReader = f.Reader - f.Reader = io.MultiReader(f.peekBuff, f.oriReader) + f.Reader = io.MultiReader(f.peek, f.oriReader) + f.Add(f.hc) } - bufSize := maxCacheSize - f.peekBuff.Size() - if bufSize <= 0 { - return f.peekBuff, nil + size = size - f.peek.Size() + if size <= 0 { + return f.peek, nil } - var buf []byte - if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) { - m, err := mmap.Alloc(int(bufSize)) - if err == nil { - f.Add(utils.CloseFunc(func() error { - return mmap.Free(m) - })) - buf = m - } - } - if buf == nil { - buf = make([]byte, bufSize) - } - n, err := io.ReadFull(f.oriReader, buf) - if bufSize != int64(n) { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err) + written, err := f.hc.CopyFromN(f.oriReader, size) + if written != size { + f.hc.RewindBySize(uint64(size - written)) + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, written, err) } - f.peekBuff.Append(buf) - if f.peekBuff.Size() >= f.GetSize() { - f.Reader = f.peekBuff + if f.peek.Size() >= f.GetSize() { + f.Reader = f.peek } - return f.peekBuff, nil + return f.peek, nil } var _ model.FileStreamer = (*SeekableStream)(nil) @@ -315,15 +254,9 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - if _, ok := rr.(*model.FileRangeReader); ok { - var rc io.ReadCloser - rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err - } - fs.Reader = rc - fs.Add(rc) - } + // IMPORTANT: Do NOT create Reader early for FileRangeReader! + // Let generateReader() create it on-demand when actually needed for reading + // This prevents the Reader from being consumed by intermediate operations like hash calculation fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil diff --git a/internal/stream/stream_test.go b/internal/stream/stream_test.go index 9a81e7d413..1d8d002e2d 100644 --- a/internal/stream/stream_test.go +++ b/internal/stream/stream_test.go @@ -1,4 +1,4 @@ -package stream +package stream_test import ( "bytes" @@ -7,27 +7,37 @@ import ( "io" "testing" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) -func TestFileStream_RangeRead(t *testing.T) { +func TestRangeRead(t *testing.T) { type args struct { httpRange http_range.Range } buf := []byte("github.com/OpenListTeam/OpenList") - f := &FileStream{ + f := &stream.FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } + prevAutoMemoryLimit := conf.AutoMemoryLimit + prevMaxBlockLimit := conf.MaxBlockLimit + t.Cleanup(func() { + conf.AutoMemoryLimit = prevAutoMemoryLimit + conf.MaxBlockLimit = prevMaxBlockLimit + }) + conf.AutoMemoryLimit = 0 + conf.MaxBlockLimit = 15 tests := []struct { name string - f *FileStream + f *stream.FileStream args args - want func(f *FileStream, got io.Reader, err error) error + want func(f *stream.FileStream, got io.Reader, err error) error }{ { name: "range 11-12", @@ -35,7 +45,7 @@ func TestFileStream_RangeRead(t *testing.T) { args: args{ httpRange: http_range.Range{Start: 11, Length: 12}, }, - want: func(f *FileStream, got io.Reader, err error) error { + want: func(f *stream.FileStream, got io.Reader, err error) error { if f.GetFile() != nil { return errors.New("cached") } @@ -52,7 +62,7 @@ func TestFileStream_RangeRead(t *testing.T) { args: args{ httpRange: http_range.Range{Start: 11, Length: 21}, }, - want: func(f *FileStream, got io.Reader, err error) error { + want: func(f *stream.FileStream, got io.Reader, err error) error { if f.GetFile() == nil { return errors.New("not cached") } @@ -84,14 +94,22 @@ func TestFileStream_RangeRead(t *testing.T) { } } -func TestFileStream_With_PreHash(t *testing.T) { +func TestPreHash(t *testing.T) { buf := []byte("github.com/OpenListTeam/OpenList") - f := &FileStream{ + f := &stream.FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } + prevAutoMemoryLimit := conf.AutoMemoryLimit + prevMaxBlockLimit := conf.MaxBlockLimit + t.Cleanup(func() { + conf.AutoMemoryLimit = prevAutoMemoryLimit + conf.MaxBlockLimit = prevMaxBlockLimit + }) + conf.AutoMemoryLimit = 0 + conf.MaxBlockLimit = 15 const hashSize int64 = 20 reader, _ := f.RangeRead(http_range.Range{Start: 0, Length: hashSize}) @@ -99,7 +117,7 @@ func TestFileStream_With_PreHash(t *testing.T) { if preHash == "" { t.Error("preHash is empty") } - tmpF, fullHash, _ := CacheFullAndHash(f, nil, utils.SHA1) + tmpF, fullHash, _ := stream.CacheFullAndHash(f, nil, utils.SHA1) fmt.Println(fullHash) fileFullHash, _ := utils.HashFile(utils.SHA1, tmpF) fmt.Println(fileFullHash) @@ -107,3 +125,61 @@ func TestFileStream_With_PreHash(t *testing.T) { t.Errorf("fullHash and fileFullHash should match: fullHash=%s fileFullHash=%s", fullHash, fileFullHash) } } + +func TestStreamSectionReader(t *testing.T) { + buf := make([]byte, 8<<10) + for i := range len(buf) { + buf[i] = byte(i % 256) + } + f := &stream.FileStream{ + Obj: &model.Object{ + Size: int64(len(buf)), + }, + Reader: io.NopCloser(bytes.NewReader(buf)), + } + prevAutoMemoryLimit := conf.AutoMemoryLimit + prevMaxBlockLimit := conf.MaxBlockLimit + prevConf := conf.Conf + t.Cleanup(func() { + conf.AutoMemoryLimit = prevAutoMemoryLimit + conf.MaxBlockLimit = prevMaxBlockLimit + conf.Conf = prevConf + }) + conf.AutoMemoryLimit = 0 + conf.MaxBlockLimit = 2 << 10 + partSize := 3 << 10 + conf.Conf = &conf.Config{} + ss, err := stream.NewStreamSectionReader(f, partSize, nil) + if err != nil { + t.Errorf("NewStreamSectionReader() error = %v", err) + } + for i := 0; i < len(buf); i += partSize { + length := partSize + if i+length > len(buf) { + length = len(buf) - i + } + rs, err := ss.GetSectionReader(int64(i), int64(length)) + if err != nil { + t.Errorf("StreamSectionReader.GetSectionReader() error = %v", err) + } + b1, err := io.ReadAll(rs) + if err != nil { + t.Errorf("StreamSectionReader.Read() error = %v", err) + } + rs.Seek(1, io.SeekStart) + b2, _ := io.ReadAll(rs) + if !bytes.Equal(b1[1:], b2) { + t.Errorf("StreamSectionReader.Read() = %s, want %s", b1[1:], b2) + } + if !bytes.Equal(buf[i:i+length], b1) { + t.Errorf("StreamSectionReader.Read() = %s, want %s", b1, buf[i:i+length]) + } + if i == 0 { + prevMinFreeMemory := conf.MinFreeMemory + conf.MinFreeMemory = 0 // 强制使用文件缓存 + t.Cleanup(func() { + conf.MinFreeMemory = prevMinFreeMemory + }) + } + } +} diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d6..e097b3cc3b 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -1,33 +1,326 @@ package stream import ( - "bytes" "context" "encoding/hex" "errors" "fmt" "io" "net/http" - "os" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" - "github.com/OpenListTeam/OpenList/v4/pkg/pool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" log "github.com/sirupsen/logrus" ) +const ( + // 链接刷新相关常量 + MAX_LINK_REFRESH_COUNT = 50 // 下载链接最大刷新次数（支持长时间传输） + + // RangeRead 重试相关常量 + MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead 最大重试次数（从3增加到5） +) + type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + + // Don't treat context cancellation as link expiration + // This happens when user pauses/seeks video or cancels download + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // All 4xx client errors may indicate expired/invalid links + // 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, 410 Gone, etc. + if code >= 400 && code < 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +// RangeRead obtains a reader reference under lock, then issues the range +// request outside the lock. The local copy of `reader` remains valid even +// if a concurrent refresh nils r.innerReader and replaces r.link, because +// doRefreshLocked only clears the pointer — it does not close or invalidate +// the old reader object. Each reader is an independent RangeReaderFunc +// (closure over a URL), so stale readers simply use the old URL; if it has +// expired, selfHealingReadCloser handles the retry transparently. +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + // Wrap the ReadCloser with self-healing capability to detect 0-byte reads + // This handles cases where cloud providers return 200 OK but empty body for expired links + return &selfHealingReadCloser{ + ReadCloser: rc, + refresher: r, + ctx: ctx, + httpRange: httpRange, + firstRead: false, + closed: false, + }, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= MAX_LINK_REFRESH_COUNT { + return fmt.Errorf("max refresh attempts (%d) reached", MAX_LINK_REFRESH_COUNT) + } + + log.Infof("Link expired, attempting to refresh...") + // Use independent context for refresh to prevent cancellation from affecting link refresh + refreshCtx := context.WithoutCancel(ctx) + newLink, _, refreshErr := r.link.Refresher(refreshCtx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + +// selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link +// if the upstream reader dies before the requested range is fully delivered. +type selfHealingReadCloser struct { + io.ReadCloser + refresher *RefreshableRangeReader + ctx context.Context + httpRange http_range.Range + firstRead bool + bytesRead int64 + closed bool + mu sync.Mutex +} + +func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return 0, errors.New("read from closed reader") + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + wasFirstRead := !s.firstRead + s.firstRead = true + + // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) + if s.shouldReconnectAfterRead(wasFirstRead, n, err) { + if reconnectErr := s.reconnectFromCurrentOffsetLocked(); reconnectErr != nil { + log.Errorf("Failed to refresh link after interrupted read: %v", reconnectErr) + return n, err + } + + if n > 0 { + return n, nil + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + return n, err + } + + return n, err +} + +func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n int, err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if s.remainingBytes() <= 0 { + return false + } + + if wasFirstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + return true + } + + if errors.Is(err, io.ErrUnexpectedEOF) { + log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + errStr := strings.ToLower(err.Error()) + if strings.Contains(errStr, "connection reset by peer") { + log.Warnf("Detected upstream connection reset after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + return false +} + +func (s *selfHealingReadCloser) reconnectFromCurrentOffsetLocked() error { + nextRange := s.httpRange + nextRange.Start += s.bytesRead + if nextRange.Length >= 0 { + nextRange.Length -= s.bytesRead + } + + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + if refreshErr != nil { + s.refresher.mu.Unlock() + return refreshErr + } + + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + if getErr != nil { + return getErr + } + + newRc, rangeErr := reader.RangeRead(s.ctx, nextRange) + if rangeErr != nil { + return rangeErr + } + + _ = s.ReadCloser.Close() + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected from offset %d", nextRange.Start) + return nil +} + +func (s *selfHealingReadCloser) remainingBytes() int64 { + length := s.httpRange.Length + if length < 0 || s.httpRange.Start+length > s.refresher.size { + length = s.refresher.size - s.httpRange.Start + } + remaining := length - s.bytesRead + if remaining < 0 { + return 0 + } + return remaining +} + +func (s *selfHealingReadCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + return s.ReadCloser.Close() +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -98,6 +391,19 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, } return nil, fmt.Errorf("http request failure, err:%w", err) } + // "Soft 200" expired-link guard: when we asked for ≥1 byte but the + // server promised 0 (e.g. 115 CDN for a stale `?t=` URL — 200 OK + + // Content-Length: 0), the body would be empty and any downstream + // client would interpret it as a corrupt/empty file. Surface this + // as an explicit "expired" error so RefreshableRangeReader can + // trigger a refresh, and callers without a Refresher fail loudly + // instead of streaming silence. ContentLength == -1 means the + // server used chunked transfer encoding and is excluded. + if httpRange.Length > 0 && response.ContentLength == 0 { + response.Body.Close() + return nil, fmt.Errorf("link expired: server returned status %d with Content-Length: 0 (expected %d bytes from %s)", + response.StatusCode, httpRange.Length, link.URL) + } if ServerDownloadLimit != nil { response.Body = &RateLimitReader{ Ctx: ctx, @@ -174,81 +480,204 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } -type StreamSectionReaderIF interface { +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// file: 文件流 +// buf: 目标缓冲区 +// off: 读取的起始偏移量 +// 返回值: 实际读取的字节数和错误 +// 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） +// 注意：链接刷新现在由 RefreshableRangeReader 内部的 selfHealingReadCloser 自动处理 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { + length := int64(len(buf)) + var lastErr error + + // 重试最多 MAX_RANGE_READ_RETRY_COUNT 次 + for retry := 0; retry < MAX_RANGE_READ_RETRY_COUNT; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return 0, err + } + lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) + log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) + // 快速重试：1秒、2秒、3秒、4秒、5秒（连接失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) + continue + } + + n, err := io.ReadFull(reader, buf) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + + if err == nil { + return n, nil + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return n, err + } + + lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + + // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + // 注意：0字节读取导致的链接过期现在由 selfHealingReadCloser 自动处理 + time.Sleep(time.Duration(retry+1) * time.Second) + } + + return 0, lastErr +} + +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + // 如果已经有完整缓存文件，直接使用 + if cache := file.GetFile(); cache != nil { + hashFunc := hashType.NewFunc() + cache.Seek(0, io.SeekStart) + _, err := io.Copy(hashFunc, cache) + if err != nil { + return "", err + } + if up != nil && progressWeight > 0 { + (*up)(progressWeight) + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil + } + + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + + if _, ok := file.(*SeekableStream); ok { + return streamHashSeekableWithPrefetch(file, hashFunc, size, chunkSize, progressWeight, up) + } + + buf := make([]byte, chunkSize) + var offset int64 = 0 + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + n, err := io.ReadFull(file, buf[:readSize]) + if err != nil { + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } + if err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + hashFunc.Write(buf[:n]) + offset += int64(n) + if up != nil && progressWeight > 0 { + (*up)(progressWeight * float64(offset) / float64(size)) + } + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + +type hashPrefetchResult struct { + buf []byte + n int + err error +} + +func streamHashSeekableWithPrefetch(file model.FileStreamer, hashFunc io.Writer, size, chunkSize int64, progressWeight float64, up *model.UpdateProgress) (string, error) { + readChunkSize := func(off int64) int64 { + if size-off < chunkSize { + return size - off + } + return chunkSize + } + + var offset int64 + + // Read first chunk synchronously + firstSize := readChunkSize(0) + curBuf := make([]byte, chunkSize) + curN, curErr := ReadFullWithRangeRead(file, curBuf[:firstSize], 0) + if curErr != nil { + return "", fmt.Errorf("calculate hash failed at offset 0: %w", curErr) + } + + for { + nextOff := offset + int64(curN) + + // Launch prefetch for next chunk while we hash current + var prefetchCh chan hashPrefetchResult + if nextOff < size { + prefetchCh = make(chan hashPrefetchResult, 1) + nextSize := readChunkSize(nextOff) + nextBuf := make([]byte, nextSize) + go func(buf []byte, off, sz int64) { + n, err := ReadFullWithRangeRead(file, buf[:sz], off) + prefetchCh <- hashPrefetchResult{buf: buf, n: n, err: err} + }(nextBuf, nextOff, nextSize) + } + + // Hash current chunk + hashFunc.Write(curBuf[:curN]) + offset += int64(curN) + + if up != nil && progressWeight > 0 { + (*up)(progressWeight * float64(offset) / float64(size)) + } + + if prefetchCh == nil { + break + } + + // Wait for prefetch + result := <-prefetchCh + if result.err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", nextOff, result.err) + } + curBuf = result.buf + curN = result.n + } + + if h, ok := hashFunc.(interface{ Sum([]byte) []byte }); ok { + return hex.EncodeToString(h.Sum(nil)), nil + } + return "", fmt.Errorf("hashFunc does not implement Sum") +} + +type StreamSectionReader interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) + // 线程安全 FreeSectionReader(sr io.ReadSeeker) // 线程不安全 DiscardSection(off int64, length int64) error } -func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (StreamSectionReaderIF, error) { +func NewStreamSectionReader(file model.FileStreamer, sectionSize int, up *model.UpdateProgress) (StreamSectionReader, error) { if file.GetFile() != nil { return &cachedSectionReader{file.GetFile()}, nil } - maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if maxBufferSize > conf.MaxBufferLimit { - f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - - if f.Truncate(file.GetSize()) != nil { - // fallback to full cache - _, _ = f.Close(), os.Remove(f.Name()) - cache, err := file.CacheFullAndWriter(up, nil) - if err != nil { - return nil, err - } - return &cachedSectionReader{cache}, nil - } - - ss := &fileSectionReader{file: file, temp: f} - ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ - New: func() *offsetWriterWithBase { - base := ss.tempOffset - ss.tempOffset += int64(maxBufferSize) - return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} - }, - } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) - })) - return ss, nil - } - - ss := &directSectionReader{file: file} - if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { - ss.bufPool = &pool.Pool[[]byte]{ - New: func() []byte { - buf, err := mmap.Alloc(maxBufferSize) - if err == nil { - file.Add(utils.CloseFunc(func() error { - return mmap.Free(buf) - })) - } else { - buf = make([]byte, maxBufferSize) - } - return buf - }, - } - } else { - ss.bufPool = &pool.Pool[[]byte]{ - New: func() []byte { - return make([]byte, maxBufferSize) - }, - } + blockSize := min(uint64(sectionSize), uint64(file.GetSize()), conf.MaxBlockLimit) + hc, err := hcache.NewHybridCache(blockSize, uint64(file.GetSize())) + if err != nil { + return nil, err } - - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return nil - })) + file.Add(hc) + ss := &hybridSectionReader{file: file, hc: hc, fileSize: file.GetSize()} + // Wait for any pending prefetch when the file is closed so we don't + // race against ss.hc being freed. + file.Add(closerFunc(ss.waitPrefetch)) return ss, nil } +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + type cachedSectionReader struct { cache io.ReaderAt } @@ -261,21 +690,52 @@ func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker } func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} -type fileSectionReader struct { +type hybridSectionReader struct { file model.FileStreamer fileOffset int64 - temp *os.File - tempOffset int64 - bufPool *pool.Pool[*offsetWriterWithBase] + fileSize int64 + hc *hcache.HybridCache + mu sync.Mutex + cache []buffer.Block + + // Pass 2 prefetch: while the caller uploads block N, we read block + // N+1 from the source in the background so download/upload overlap. + // Access is serialized through GetSectionReader/DiscardSection which + // are documented as 线程不安全; only the background prefetch goroutine + // touches `prefetch` concurrently with those methods, and waitPrefetch + // drains it before any of them touches ss.file again. + prefetch *prefetchTask } -type offsetWriterWithBase struct { - *io.OffsetWriter - base int64 +type prefetchTask struct { + off int64 // file offset the prefetch started at + length int64 // bytes requested + actual int64 // bytes actually read into block (may be < length on EOF/error) + block buffer.Block // nil if allocation/read failed before any bytes + err error // non-nil on prefetch error + done chan struct{} } // 线程不安全 -func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { +func (ss *hybridSectionReader) DiscardSection(off int64, length int64) error { + // Drain any pending prefetch first so ss.file is quiescent. If the + // prefetched range exactly matches the discard request, we're done — + // the bytes have already been read from the source. + if p := ss.prefetch; p != nil { + <-p.done + ss.prefetch = nil + if p.err == nil && p.off == off && p.actual == length { + if p.block != nil { + ss.put(p.block) + } + ss.fileOffset = off + length + return nil + } + if p.block != nil { + ss.put(p.block) + } + ss.fileOffset = p.off + p.actual + } if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -287,76 +747,171 @@ func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { return nil } -type fileBufferSectionReader struct { +type blockRefReadSeeker struct { io.ReadSeeker - fileBuf *offsetWriterWithBase + b buffer.Block } // 线程不安全 -func (ss *fileSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { +func (ss *hybridSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { + // Try prefetched block first. + if b, actual, err, ok := ss.takePrefetched(off, length); ok { + if err != nil { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("prefetch failed at offset %d: %w", off, err) + } + if actual < length { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("prefetch short read at offset %d: (expect=%d, actual=%d)", off, length, actual) + } + ss.fileOffset = off + length + ss.schedulePrefetch(off+length, length) + return makeBlockReadSeeker(b, length) + } + if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - fileBuf := ss.bufPool.Get() - _, _ = fileBuf.Seek(0, io.SeekStart) - n, err := utils.CopyWithBufferN(fileBuf, ss.file, length) - ss.fileOffset += n - if err != nil { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + b, actual, err := ss.readBlock(length) + if err != nil || actual != length { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, actual, err) } - return &fileBufferSectionReader{io.NewSectionReader(ss.temp, fileBuf.base, length), fileBuf}, nil + ss.fileOffset += actual + ss.schedulePrefetch(off+length, length) + return makeBlockReadSeeker(b, length) } -func (ss *fileSectionReader) FreeSectionReader(rs io.ReadSeeker) { - if sr, ok := rs.(*fileBufferSectionReader); ok { - ss.bufPool.Put(sr.fileBuf) - sr.fileBuf = nil - sr.ReadSeeker = nil +// readBlock reads `length` bytes from ss.file into a freshly populated +// buffer.Block. The returned block may be nil if no bytes could be read. +// Caller is responsible for returning the block to the pool on error. +func (ss *hybridSectionReader) readBlock(length int64) (buffer.Block, int64, error) { + b := ss.get() + if b == nil { + offset := int64(ss.hc.Size()) + written, err := ss.hc.CopyFromN(ss.file, length) + if written == 0 { + return nil, 0, err + } + b = buffer.NewBlockAdapter( + io.NewOffsetWriter(ss.hc, offset), + io.NewSectionReader(ss.hc, offset, written), + ) + return b, written, err + } + ws := buffer.WriteAtSeekerOf(b) + if _, err := ws.Seek(0, io.SeekStart); err != nil { + ss.put(b) + return nil, 0, fmt.Errorf("failed to reset cached block writer: %w", err) } + written, err := utils.CopyWithBufferN(ws, ss.file, length) + return b, written, err } -type directSectionReader struct { - file model.FileStreamer - fileOffset int64 - bufPool *pool.Pool[[]byte] +// schedulePrefetch starts a background read of `length` bytes at file +// offset `off`. It is a no-op if there is nothing more to read or a +// prefetch is already in flight. +func (ss *hybridSectionReader) schedulePrefetch(off, length int64) { + if length <= 0 || off >= ss.fileSize { + return + } + if ss.prefetch != nil { + return + } + // Clamp to remaining file size so the last partial chunk doesn't + // produce a synthetic short-read error. + if remaining := ss.fileSize - off; remaining < length { + length = remaining + } + task := &prefetchTask{off: off, length: length, done: make(chan struct{})} + ss.prefetch = task + go func() { + defer close(task.done) + b, actual, err := ss.readBlock(length) + task.block = b + task.actual = actual + task.err = err + }() } -// 线程不安全 -func (ss *directSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.fileOffset { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) +// takePrefetched returns the prefetched block if it matches the caller's +// requested offset. The ok return distinguishes "no prefetch present" +// (false) from "prefetch was consumed" (true). +func (ss *hybridSectionReader) takePrefetched(off, length int64) (buffer.Block, int64, error, bool) { + p := ss.prefetch + if p == nil { + return nil, 0, nil, false } - n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) - ss.fileOffset += n - if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) + <-p.done + ss.prefetch = nil + if p.off != off { + if p.block != nil { + ss.put(p.block) + } + // Source has already advanced by p.actual bytes past p.off. + // Reflect that so subsequent calls see a consistent fileOffset. + ss.fileOffset = p.off + p.actual + return nil, 0, nil, false } - return nil + // Caller may ask for fewer bytes than we prefetched (e.g. final + // partial chunk after we over-prefetched). Allow that. + if length > p.actual && p.err == nil { + // Did not get as many bytes as caller wants and source didn't + // signal an error — surface a short-read for the caller to + // handle, but keep block so it can be returned. + return p.block, p.actual, fmt.Errorf("short read"), true + } + return p.block, p.actual, p.err, true } -type bufferSectionReader struct { - io.ReadSeeker - buf []byte +func (ss *hybridSectionReader) waitPrefetch() error { + if p := ss.prefetch; p != nil { + <-p.done + ss.prefetch = nil + if p.block != nil { + ss.put(p.block) + } + } + return nil } -// 线程不安全 -func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.fileOffset { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) +func makeBlockReadSeeker(b buffer.Block, length int64) (io.ReadSeeker, error) { + if length == b.Size() { + rs := buffer.ReadAtSeekerOf(b) + if _, err := rs.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("failed to reset cached block reader: %w", err) + } + return &blockRefReadSeeker{rs, b}, nil } - tempBuf := ss.bufPool.Get() - buf := tempBuf[:length] - n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + return &blockRefReadSeeker{io.NewSectionReader(b, 0, length), b}, nil +} + +func (ss *hybridSectionReader) get() buffer.Block { + ss.mu.Lock() + defer ss.mu.Unlock() + if len(ss.cache) > 0 { + b := ss.cache[len(ss.cache)-1] + ss.cache = ss.cache[:len(ss.cache)-1] + return b } - return &bufferSectionReader{bytes.NewReader(buf), buf}, nil + return nil +} +func (ss *hybridSectionReader) put(b buffer.Block) { + ss.mu.Lock() + defer ss.mu.Unlock() + ss.cache = append(ss.cache, b) } -func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { - if sr, ok := rs.(*bufferSectionReader); ok { - ss.bufPool.Put(sr.buf[0:cap(sr.buf)]) - sr.buf = nil + +func (ss *hybridSectionReader) FreeSectionReader(rs io.ReadSeeker) { + if sr, ok := rs.(*blockRefReadSeeker); ok { + ss.put(sr.b) + sr.b = nil sr.ReadSeeker = nil } } diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go new file mode 100644 index 0000000000..4bdd1f5068 --- /dev/null +++ b/internal/stream/util_test.go @@ -0,0 +1,496 @@ +package stream + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// ensureConfForHTTP makes sure conf.Conf is non-nil before any test reaches +// net.HttpClient — that helper deferences conf.Conf.TlsInsecureSkipVerify +// during a sync.Once init and panics on a nil pointer otherwise. +func ensureConfForHTTP() { + if conf.Conf == nil { + conf.Conf = &conf.Config{} + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != int64(len(data)-5) { + t.Fatalf("resumed range length = %d, want %d", resumedRanges[0].Length, len(data)-5) + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != -1 { + t.Fatalf("resumed range length = %d, want -1", resumedRanges[0].Length) + } +} + +// TestSelfHealingReadCloser_NormalEOFDoesNotTriggerReconnect verifies that a +// legitimate io.EOF (all data delivered) does NOT trigger a link refresh. +func TestSelfHealingReadCloser_NormalEOFDoesNotTriggerReconnect(t *testing.T) { + data := []byte("hello world") + refreshes := 0 + + inner := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: inner} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: inner}, nil, nil + } + + rrr := NewRefreshableRangeReader(link, int64(len(data))) + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead error: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("got %q, want %q", got, data) + } + if refreshes != 0 { + t.Fatalf("refreshes = %d, want 0 (normal EOF should not trigger refresh)", refreshes) + } +} + +// TestSelfHealingReadCloser_UnexpectedEOFTriggersReconnect verifies that +// io.ErrUnexpectedEOF (stream interrupted) DOES trigger reconnect. +func TestSelfHealingReadCloser_UnexpectedEOFTriggersReconnect(t *testing.T) { + data := []byte("0123456789") + refreshes := 0 + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 4, io.ErrUnexpectedEOF), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + rrr := NewRefreshableRangeReader(link, int64(len(data))) + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead error: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("got %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } +} + +// TestStreamHashFile_SeekablePrefetchProducesSameHash verifies that +// the prefetch optimization in StreamHashFile produces the exact same +// hash as a sequential read. +func TestStreamHashFile_SeekablePrefetchProducesSameHash(t *testing.T) { + // 50 bytes = will be split into 10MB chunks in real code, but we + // override chunkSize for testing. The key point: hash must be identical. + data := []byte("The quick brown fox jumps over the lazy dog!!!!!") // 48 bytes + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := &SeekableStream{ + FileStream: &FileStream{ + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Ctx: context.Background(), + }, + rangeReader: rr, + } + + hash1, err := StreamHashFile(seekable, utils.SHA1, 0, nil) + if err != nil { + t.Fatalf("StreamHashFile error: %v", err) + } + + // Compute expected hash directly + h := utils.SHA1.NewFunc() + h.Write(data) + expected := hex.EncodeToString(h.Sum(nil)) + + if hash1 != expected { + t.Fatalf("hash mismatch: got %s, want %s", hash1, expected) + } +} + +type flakyReadCloser struct { + data []byte + failAfter int + failErr error + failed bool +} + +func newFlakyReadCloser(data []byte, failAfter int, failErr error) *flakyReadCloser { + return &flakyReadCloser{ + data: data, + failAfter: failAfter, + failErr: failErr, + } +} + +func (f *flakyReadCloser) Read(p []byte) (int, error) { + if f.failed { + return 0, io.EOF + } + if f.failAfter >= len(f.data) { + f.failed = true + n := copy(p, f.data) + return n, io.EOF + } + + n := copy(p, f.data[:f.failAfter]) + f.failed = true + return n, f.failErr +} + +func (f *flakyReadCloser) Close() error { + return nil +} + +// TestRangeReaderFromLink_SoftExpiredLink_TriggersRefresh covers the +// 115 CDN "soft 200 + empty body" failure mode: when a signed URL has +// expired, 115 still answers with HTTP 200 and Content-Length: 0 instead +// of a 4xx. Without explicit detection, OP forwards an empty stream to +// the client (mpv sees "Failed to recognize file format"). This test +// pins the contract that GetRangeReaderFromLink-derived readers treat +// "non-zero range requested → zero bytes promised" as an expired link +// and let RefreshableRangeReader trigger a refresh + retry. +func TestRangeReaderFromLink_SoftExpiredLink_TriggersRefresh(t *testing.T) { + ensureConfForHTTP() + data := []byte("The quick brown fox jumps over the lazy dog") + size := int64(len(data)) + + var expiredHits int32 + expired := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&expiredHits, 1) + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusOK) + })) + defer expired.Close() + + var freshHits int32 + fresh := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&freshHits, 1) + rangeHeader := r.Header.Get("Range") + start := int64(0) + length := size + if rangeHeader != "" { + ranges, err := http_range.ParseRange(rangeHeader, size) + if err == nil && len(ranges) == 1 { + start = ranges[0].Start + length = ranges[0].Length + w.Header().Set("Content-Range", ranges[0].ContentRange(size)) + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : start+length]) + return + } + } + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data[start : start+length]) + })) + defer fresh.Close() + + link := &model.Link{URL: expired.URL} + var refreshes int32 + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return &model.Link{URL: fresh.URL}, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("body mismatch: got %q, want %q", got, data) + } + if atomic.LoadInt32(&refreshes) != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + if atomic.LoadInt32(&expiredHits) < 1 { + t.Fatalf("expired server never hit (= %d), expected at least once", expiredHits) + } + if atomic.LoadInt32(&freshHits) < 1 { + t.Fatalf("fresh server never hit (= %d), expected at least once after refresh", freshHits) + } +} + +// TestRangeReaderFromLink_NormalResponse_NoFalsePositive guards against the +// soft-expired check ever firing on a healthy 206 response. +func TestRangeReaderFromLink_NormalResponse_NoFalsePositive(t *testing.T) { + ensureConfForHTTP() + data := []byte("0123456789abcdef") + size := int64(len(data)) + + var refreshes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ranges, _ := http_range.ParseRange(r.Header.Get("Range"), size) + if len(ranges) == 1 { + ra := ranges[0] + w.Header().Set("Content-Range", ra.ContentRange(size)) + w.Header().Set("Content-Length", strconv.FormatInt(ra.Length, 10)) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[ra.Start : ra.Start+ra.Length]) + return + } + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + _, _ = w.Write(data) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return link, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 4, Length: 6}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[4:10]) { + t.Fatalf("body mismatch: got %q, want %q", got, data[4:10]) + } + if atomic.LoadInt32(&refreshes) != 0 { + t.Fatalf("refreshes = %d, want 0 (healthy response must not trigger refresh)", refreshes) + } +} + +// TestRangeReaderFromLink_ChunkedResponse_NoFalsePositive guards against +// false positives on responses without a Content-Length (chunked transfer). +// Go's http.Response sets ContentLength = -1 for chunked, which must not +// be treated as "0 bytes promised". +func TestRangeReaderFromLink_ChunkedResponse_NoFalsePositive(t *testing.T) { + ensureConfForHTTP() + data := []byte("chunked-payload-bytes-here-yo!") + size := int64(len(data)) + + var refreshes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Force chunked: do not set Content-Length, write headers then body + // in two flushes. + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + _, _ = w.Write(data[:len(data)/2]) + if flusher != nil { + flusher.Flush() + } + _, _ = w.Write(data[len(data)/2:]) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return link, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("body mismatch: got %q, want %q", got, data) + } + if atomic.LoadInt32(&refreshes) != 0 { + t.Fatalf("refreshes = %d, want 0 (chunked response must not be treated as expired)", refreshes) + } +} + +// TestRangeReaderFromLink_SoftExpiredLink_NoRefresher_ReturnsError verifies +// that when a soft-expired link is encountered without a Refresher set, +// the error surfaces cleanly to the caller instead of returning an empty +// body that the client then misinterprets as a corrupt file. +func TestRangeReaderFromLink_SoftExpiredLink_NoRefresher_ReturnsError(t *testing.T) { + ensureConfForHTTP() + size := int64(100) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} // no Refresher + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + _, err = rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err == nil { + t.Fatalf("RangeRead returned nil error; expected an 'expired link' error so callers see a real failure instead of an empty stream") + } + if !IsLinkExpiredError(err) { + t.Fatalf("error %q is not classified as expired by IsLinkExpiredError", err) + } +} + +func sliceForRange(data []byte, httpRange http_range.Range) []byte { + start := int(httpRange.Start) + length := int(httpRange.Length) + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > int64(len(data)) { + length = len(data) - start + } + return data[start : start+length] +} diff --git a/pkg/buffer/buffer.go b/pkg/buffer/buffer.go new file mode 100644 index 0000000000..54d13c56d3 --- /dev/null +++ b/pkg/buffer/buffer.go @@ -0,0 +1,41 @@ +package buffer + +import ( + "io" +) + +type byteBlock struct { + buf []byte +} + +func NewByteBlock(buf []byte) Block { + return &byteBlock{buf: buf} +} + +func (b *byteBlock) Size() int64 { + return int64(len(b.buf)) +} + +func (b *byteBlock) ReadAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.EOF + } + n = copy(p, b.buf[off:]) + if n < len(p) { + err = io.EOF + } + return +} + +func (b *byteBlock) WriteAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.ErrShortWrite + } + n = copy(b.buf[off:], p) + if n < len(p) { + err = io.ErrShortWrite + } + return +} + +var _ Block = (*byteBlock)(nil) diff --git a/pkg/buffer/bytes.go b/pkg/buffer/bytes.go deleted file mode 100644 index 3e6cb54054..0000000000 --- a/pkg/buffer/bytes.go +++ /dev/null @@ -1,95 +0,0 @@ -package buffer - -import ( - "errors" - "io" -) - -// 用于存储不复用的[]byte -type Reader struct { - bufs [][]byte - size int64 - offset int64 -} - -func (r *Reader) Size() int64 { - return r.size -} - -func (r *Reader) Append(buf []byte) { - r.size += int64(len(buf)) - r.bufs = append(r.bufs, buf) -} - -func (r *Reader) Read(p []byte) (int, error) { - n, err := r.ReadAt(p, r.offset) - if n > 0 { - r.offset += int64(n) - } - return n, err -} - -func (r *Reader) ReadAt(p []byte, off int64) (int, error) { - if off < 0 || off >= r.size { - return 0, io.EOF - } - - n := 0 - readFrom := false - for _, buf := range r.bufs { - if readFrom { - nn := copy(p[n:], buf) - n += nn - if n == len(p) { - return n, nil - } - } else if newOff := off - int64(len(buf)); newOff >= 0 { - off = newOff - } else { - nn := copy(p, buf[off:]) - if nn == len(p) { - return nn, nil - } - n += nn - readFrom = true - } - } - - return n, io.EOF -} - -func (r *Reader) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - case io.SeekCurrent: - offset = r.offset + offset - case io.SeekEnd: - offset = r.size + offset - default: - return 0, errors.New("Seek: invalid whence") - } - - if offset < 0 || offset > r.size { - return 0, errors.New("Seek: invalid offset") - } - - r.offset = offset - return offset, nil -} - -func (r *Reader) Reset() { - clear(r.bufs) - r.bufs = nil - r.size = 0 - r.offset = 0 -} - -func NewReader(buf ...[]byte) *Reader { - b := &Reader{ - bufs: make([][]byte, 0, len(buf)), - } - for _, b1 := range buf { - b.Append(b1) - } - return b -} diff --git a/pkg/buffer/file.go b/pkg/buffer/file.go deleted file mode 100644 index 48edf5a4cb..0000000000 --- a/pkg/buffer/file.go +++ /dev/null @@ -1,88 +0,0 @@ -package buffer - -import ( - "errors" - "io" - "os" -) - -type PeekFile struct { - peek *Reader - file *os.File - offset int64 - size int64 -} - -func (p *PeekFile) Read(b []byte) (n int, err error) { - n, err = p.ReadAt(b, p.offset) - if n > 0 { - p.offset += int64(n) - } - return n, err -} - -func (p *PeekFile) ReadAt(b []byte, off int64) (n int, err error) { - if off < p.peek.Size() { - n, err = p.peek.ReadAt(b, off) - if err == nil || n == len(b) { - return n, nil - } - // EOF - } - var nn int - nn, err = p.file.ReadAt(b[n:], off+int64(n)-p.peek.Size()) - return n + nn, err -} - -func (p *PeekFile) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - case io.SeekCurrent: - if offset == 0 { - return p.offset, nil - } - offset = p.offset + offset - case io.SeekEnd: - offset = p.size + offset - default: - return 0, errors.New("Seek: invalid whence") - } - - if offset < 0 || offset > p.size { - return 0, errors.New("Seek: invalid offset") - } - if offset <= p.peek.Size() { - _, err := p.peek.Seek(offset, io.SeekStart) - if err != nil { - return 0, err - } - _, err = p.file.Seek(0, io.SeekStart) - if err != nil { - return 0, err - } - } else { - _, err := p.peek.Seek(p.peek.Size(), io.SeekStart) - if err != nil { - return 0, err - } - _, err = p.file.Seek(offset-p.peek.Size(), io.SeekStart) - if err != nil { - return 0, err - } - } - - p.offset = offset - return offset, nil -} - -func (p *PeekFile) Size() int64 { - return p.size -} - -func NewPeekFile(peek *Reader, file *os.File) (*PeekFile, error) { - stat, err := file.Stat() - if err == nil { - return &PeekFile{peek: peek, file: file, size: stat.Size() + peek.Size()}, nil - } - return nil, err -} diff --git a/pkg/buffer/pipe.go b/pkg/buffer/pipe.go new file mode 100644 index 0000000000..194fd58cf4 --- /dev/null +++ b/pkg/buffer/pipe.go @@ -0,0 +1,157 @@ +package buffer + +import ( + "context" + "fmt" + "io" + "sync" +) + +type PipeBuffer struct { + limit int //expected size + ctx context.Context + offR int + offW int + rw sync.Mutex + block Block + + readSignal chan struct{} + readPending bool +} + +// NewPipeBuffer is a buffer that can have 1 read & 1 write at the same time. +// when read is faster write, immediately feed data to read after written +func NewPipeBuffer(ctx context.Context, block Block) *PipeBuffer { + br := &PipeBuffer{ + ctx: ctx, + limit: int(block.Size()), + readSignal: make(chan struct{}, 1), + block: block, + } + return br +} + +func (br *PipeBuffer) Read(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + if br.offR >= br.limit { + return 0, io.EOF + } + + for { + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + if br.offW == br.offR { + br.readPending = true + br.rw.Unlock() + select { + case <-br.ctx.Done(): + return 0, br.ctx.Err() + case _, ok := <-br.readSignal: + if !ok { + return 0, io.ErrClosedPipe + } + continue + } + } + break + } + + canRead := br.offW - br.offR + if canRead < 0 { + br.rw.Unlock() + return 0, io.ErrUnexpectedEOF + } + + off := br.offR + block := br.block + br.rw.Unlock() + + n, err := block.ReadAt(p[:min(len(p), canRead)], int64(off)) + + br.rw.Lock() + br.offR += n + br.rw.Unlock() + + if n < len(p) && br.offR >= br.limit { + return n, io.EOF + } + return n, err +} + +func (br *PipeBuffer) Write(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + canWrite := br.limit - br.offW + if canWrite <= 0 { + br.rw.Unlock() + return 0, io.ErrShortWrite + } + + off := br.offW + block := br.block + br.rw.Unlock() + + n, err := block.WriteAt(p[:min(canWrite, len(p))], int64(off)) + + br.rw.Lock() + br.offW += n + if br.readPending { + br.readPending = false + select { + case br.readSignal <- struct{}{}: + default: + } + } + br.rw.Unlock() + + if n < len(p) && err == nil { + return n, io.ErrShortWrite + } + return n, err +} + +func (br *PipeBuffer) Reset(limit int) error { + br.rw.Lock() + defer br.rw.Unlock() + if br.block == nil { + return io.ErrClosedPipe + } + if int64(limit) > br.block.Size() { + return fmt.Errorf("reset limit %d exceeds max size %d", limit, br.block.Size()) + } + br.limit = limit + br.offR = 0 + br.offW = 0 + return nil +} + +func (br *PipeBuffer) Close() error { + br.rw.Lock() + defer br.rw.Unlock() + if br.block != nil { + br.block = nil + br.readPending = false + close(br.readSignal) + } + return nil +} diff --git a/pkg/buffer/type.go b/pkg/buffer/type.go new file mode 100644 index 0000000000..ce0d78b2e9 --- /dev/null +++ b/pkg/buffer/type.go @@ -0,0 +1,24 @@ +package buffer + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type Block interface { + io.ReaderAt + io.WriterAt + Size() int64 +} + +type WriteAtSeeker = model.FileWriter +type WriteAtSeekerProvider interface{ GetWriteAtSeeker() WriteAtSeeker } + +type ReadAtSeeker = model.File +type ReadAtSeekerProvider interface{ GetReadAtSeeker() ReadAtSeeker } + +type SizedReadAtSeeker interface { + ReadAtSeeker + Size() int64 +} diff --git a/pkg/buffer/utils.go b/pkg/buffer/utils.go new file mode 100644 index 0000000000..4783f4b704 --- /dev/null +++ b/pkg/buffer/utils.go @@ -0,0 +1,93 @@ +package buffer + +import ( + "errors" + "io" +) + +func WriteAtSeekerOf(b Block) WriteAtSeeker { + if p, ok := b.(WriteAtSeekerProvider); ok { + return p.GetWriteAtSeeker() + } + return io.NewOffsetWriter(b, 0) +} + +// 将一个Block包装为ReadAtSeeker。 +// 固定大小：当前Block的Size()。 +func ReadAtSeekerOf(b Block) ReadAtSeeker { + if p, ok := b.(ReadAtSeekerProvider); ok { + return p.GetReadAtSeeker() + } + return io.NewSectionReader(b, 0, b.Size()) +} + +type blockAdapter struct { + WriteAtSeeker + SizedReadAtSeeker +} + +func (b *blockAdapter) GetWriteAtSeeker() WriteAtSeeker { + return b.WriteAtSeeker +} + +func (b *blockAdapter) GetReadAtSeeker() ReadAtSeeker { + return b.SizedReadAtSeeker +} +func NewBlockAdapter(w WriteAtSeeker, r SizedReadAtSeeker) Block { + return &blockAdapter{ + WriteAtSeeker: w, + SizedReadAtSeeker: r, + } +} + +var _ Block = (*blockAdapter)(nil) + +// 将一个Block包装为ReadAtSeeker。 +// 动态大小：Size() 是动态跟随底层 Block。 +type DynamicReadAtSeeker struct { + block Block + offset int64 +} + +func (r *DynamicReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { + return r.block.ReadAt(p, off) +} + +func (r *DynamicReadAtSeeker) Read(p []byte) (n int, err error) { + n, err = r.block.ReadAt(p, r.offset) + if n > 0 { + r.offset += int64(n) + } + return n, err +} + +func (r *DynamicReadAtSeeker) Size() int64 { + return r.block.Size() +} + +func (r *DynamicReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + if offset == 0 { + return r.offset, nil + } + offset = r.offset + offset + case io.SeekEnd: + offset = r.block.Size() + offset + default: + return 0, errors.New("Seek: invalid whence") + } + + if offset < 0 || offset > r.block.Size() { + return 0, errors.New("Seek: invalid offset") + } + r.offset = offset + return offset, nil +} + +func NewDynamicReadAtSeeker(block Block) *DynamicReadAtSeeker { + return &DynamicReadAtSeeker{ + block: block, + } +} diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index ce92cd1fc3..01cd736d32 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -3,9 +3,7 @@ package pool import "sync" type Pool[T any] struct { - New func() T - MaxCap int - + New func() T cache []T mu sync.Mutex } @@ -24,9 +22,7 @@ func (p *Pool[T]) Get() T { func (p *Pool[T]) Put(item T) { p.mu.Lock() defer p.mu.Unlock() - if p.MaxCap == 0 || len(p.cache) < int(p.MaxCap) { - p.cache = append(p.cache, item) - } + p.cache = append(p.cache, item) } func (p *Pool[T]) Reset() { @@ -35,3 +31,8 @@ func (p *Pool[T]) Reset() { clear(p.cache) p.cache = nil } + +func (p *Pool[T]) Close() error { + p.Reset() + return nil +} diff --git a/pkg/qbittorrent/client.go b/pkg/qbittorrent/client.go index cc8be87071..e4c12db6fc 100644 --- a/pkg/qbittorrent/client.go +++ b/pkg/qbittorrent/client.go @@ -99,12 +99,14 @@ func (c *client) login() error { defer resp.Body.Close() // avoid long waiting time if being upgraded to websocket connections (e.g. 101 responses) - // as per API documentation, qBittorrent returns only 200 on successful login - // so we safely treat any non-200 response as a failure - if resp.StatusCode != http.StatusOK { + // as per API documentation, qBittorrent returns only 200 on successful login (qBittorrent < 5.2.0) + // qBittorrent 5.2.0 /api/v2/auth/login returns HTTP 204 on success + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return errors.New("failed to login into qBittorrent webui with status code: " + resp.Status) } - + if resp.StatusCode == http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) _, err = resp.Body.Read(body) @@ -180,6 +182,10 @@ func (c *client) AddFromLink(link string, savePath string, id string) error { return err } defer resp.Body.Close() + // qBittorrent 5.2.0 returns 204 on success. + if resp.StatusCode != http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) diff --git a/pkg/torrent/bencode.go b/pkg/torrent/bencode.go new file mode 100644 index 0000000000..2d4fd782c6 --- /dev/null +++ b/pkg/torrent/bencode.go @@ -0,0 +1,261 @@ +package torrent + +import ( + "bytes" + "fmt" + "io" + "sort" + "strconv" +) + +// bencode 编码 + +// BencodeEncode 将值编码为 bencode 格式 +func BencodeEncode(v interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := bencodeEncodeValue(&buf, v); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func bencodeEncodeValue(w io.Writer, v interface{}) error { + switch val := v.(type) { + case int: + return bencodeEncodeInt(w, int64(val)) + case int64: + return bencodeEncodeInt(w, val) + case string: + return bencodeEncodeString(w, val) + case []byte: + return bencodeEncodeBytes(w, val) + case []interface{}: + return bencodeEncodeList(w, val) + case map[string]interface{}: + return bencodeEncodeDict(w, val) + case OrderedDict: + return bencodeEncodeOrderedDict(w, val) + default: + return fmt.Errorf("bencode: unsupported type %T", v) + } +} + +func bencodeEncodeInt(w io.Writer, v int64) error { + _, err := fmt.Fprintf(w, "i%de", v) + return err +} + +func bencodeEncodeString(w io.Writer, v string) error { + _, err := fmt.Fprintf(w, "%d:%s", len(v), v) + return err +} + +func bencodeEncodeBytes(w io.Writer, v []byte) error { + _, err := fmt.Fprintf(w, "%d:", len(v)) + if err != nil { + return err + } + _, err = w.Write(v) + return err +} + +func bencodeEncodeList(w io.Writer, v []interface{}) error { + if _, err := w.Write([]byte("l")); err != nil { + return err + } + for _, item := range v { + if err := bencodeEncodeValue(w, item); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +func bencodeEncodeDict(w io.Writer, v map[string]interface{}) error { + // bencode 字典要求 key 按字典序排列 + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, v[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// OrderedDict 有序字典，保持插入顺序 +type OrderedDict struct { + Keys []string + Values map[string]interface{} +} + +func NewOrderedDict() OrderedDict { + return OrderedDict{ + Keys: make([]string, 0), + Values: make(map[string]interface{}), + } +} + +func (d *OrderedDict) Set(key string, value interface{}) { + if _, exists := d.Values[key]; !exists { + d.Keys = append(d.Keys, key) + } + d.Values[key] = value +} + +func (d *OrderedDict) Get(key string) (interface{}, bool) { + v, ok := d.Values[key] + return v, ok +} + +func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error { + // 按字典序排列 key（bencode 规范要求） + keys := make([]string, len(d.Keys)) + copy(keys, d.Keys) + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, d.Values[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// bencode 解码 + +// BencodeDecode 从字节数组解码 bencode 数据 +func BencodeDecode(data []byte) (interface{}, error) { + reader := bytes.NewReader(data) + val, err := bencodeDecodeValue(reader) + if err != nil { + return nil, err + } + return val, nil +} + +func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + + switch { + case b == 'i': + return bencodeDecodeInt(r) + case b == 'l': + return bencodeDecodeList(r) + case b == 'd': + return bencodeDecodeDict(r) + case b >= '0' && b <= '9': + r.UnreadByte() + return bencodeDecodeString(r) + default: + return nil, fmt.Errorf("bencode: unexpected byte '%c' at position %d", b, int64(r.Len())) + } +} + +func bencodeDecodeInt(r *bytes.Reader) (int64, error) { + var buf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return 0, err + } + if b == 'e' { + break + } + buf.WriteByte(b) + } + return strconv.ParseInt(buf.String(), 10, 64) +} + +func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { + // 读取长度 + var lenBuf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == ':' { + break + } + lenBuf.WriteByte(b) + } + length, err := strconv.ParseInt(lenBuf.String(), 10, 64) + if err != nil { + return nil, fmt.Errorf("bencode: invalid string length: %v", err) + } + if length < 0 || length > 100*1024*1024 { + return nil, fmt.Errorf("bencode: string length out of bounds: %d", length) + } + // Safe to convert to int: bounds check above ensures length <= 100MB which fits in int32 + data := make([]byte, int(length)) + _, err = io.ReadFull(r, data) + if err != nil { + return nil, err + } + return data, nil +} + +func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { + var list []interface{} + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return list, nil + } + r.UnreadByte() + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + list = append(list, val) + } +} + +func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { + dict := make(map[string]interface{}) + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return dict, nil + } + r.UnreadByte() + keyBytes, err := bencodeDecodeString(r) + if err != nil { + return nil, err + } + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + dict[string(keyBytes)] = val + } +} diff --git a/pkg/torrent/generate.go b/pkg/torrent/generate.go new file mode 100644 index 0000000000..566cad86f7 --- /dev/null +++ b/pkg/torrent/generate.go @@ -0,0 +1,123 @@ +package torrent + +import ( + "io" + "os" + "strings" +) + +// GenerateFromFile 从文件路径生成通用的 torrent 文件（不含 CAS 扩展） +// 这是一个通用函数，适用于所有驱动 +func GenerateFromFile(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReader(f, info.Name(), info.Size(), DefaultPieceSize) +} + +// GenerateFromReader 从 io.Reader 生成通用的 torrent 文件（不含 CAS 扩展） +// 返回 torrent 字节数据 +func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + pieceHashes := hw.GetPieceHashes() + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + + return t.Encode() +} + +// GenerateFromReaderWithCAS 从 io.Reader 生成包含 CAS 扩展的 torrent 文件 +// 适用于天翼云等支持秒传的网盘 +func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: pieceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// GenerateFromFileWithCAS 从文件路径生成包含 CAS 扩展的 torrent 文件 +func GenerateFromFileWithCAS(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize) +} diff --git a/pkg/torrent/hash_writer.go b/pkg/torrent/hash_writer.go new file mode 100644 index 0000000000..a62f42c648 --- /dev/null +++ b/pkg/torrent/hash_writer.go @@ -0,0 +1,229 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "strings" +) + +// HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash +// 用于在上传过程中一次性计算所有需要的哈希值 +type HashWriter struct { + // 整文件 MD5 + fileMD5 hash.Hash + // 当前分片 MD5 + sliceMD5 hash.Hash + // 当前 piece 的 SHA-1 + pieceSHA1 hash.Hash + + // 分片大小（默认 10MB） + sliceSize int64 + // piece 大小（与 sliceSize 相同，保持对齐） + pieceSize int64 + + // 当前分片已写入字节数 + sliceWritten int64 + // 当前 piece 已写入字节数 + pieceWritten int64 + // 总写入字节数 + totalWritten int64 + + // 每个分片的 MD5（大写十六进制） + sliceMD5Hexs []string + // 所有 piece 的 SHA-1 哈希拼接 + pieceHashes []byte +} + +// NewHashWriter 创建一个新的 HashWriter +// sliceSize: CAS 分片大小（通常 10MB） +// pieceSize: BT piece 大小（设为与 sliceSize 相同以保持对齐） +func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { + if sliceSize <= 0 { + sliceSize = DefaultPieceSize + } + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + return &HashWriter{ + fileMD5: md5.New(), + sliceMD5: md5.New(), + pieceSHA1: sha1.New(), + sliceSize: sliceSize, + pieceSize: pieceSize, + } +} + +// NewDefaultHashWriter 创建默认的 HashWriter（10MB 分片） +func NewDefaultHashWriter() *HashWriter { + return NewHashWriter(DefaultPieceSize, DefaultPieceSize) +} + +// Write 实现 io.Writer 接口 +func (hw *HashWriter) Write(p []byte) (n int, err error) { + total := len(p) + offset := 0 + + for offset < total { + // 计算当前可以写入的字节数（取分片和 piece 剩余空间的最小值） + sliceRemain := hw.sliceSize - hw.sliceWritten + pieceRemain := hw.pieceSize - hw.pieceWritten + canWrite := min64(sliceRemain, pieceRemain) + canWrite = min64(canWrite, int64(total-offset)) + + chunk := p[offset : offset+int(canWrite)] + + // 写入整文件 MD5 + hw.fileMD5.Write(chunk) + // 写入当前分片 MD5 + hw.sliceMD5.Write(chunk) + // 写入当前 piece SHA-1 + hw.pieceSHA1.Write(chunk) + + hw.sliceWritten += canWrite + hw.pieceWritten += canWrite + hw.totalWritten += canWrite + offset += int(canWrite) + + // 检查分片是否完成 + if hw.sliceWritten >= hw.sliceSize { + hw.finishSlice() + } + + // 检查 piece 是否完成 + if hw.pieceWritten >= hw.pieceSize { + hw.finishPiece() + } + } + + return total, nil +} + +// finishSlice 完成当前分片的 MD5 计算 +func (hw *HashWriter) finishSlice() { + md5Hex := strings.ToUpper(hex.EncodeToString(hw.sliceMD5.Sum(nil))) + hw.sliceMD5Hexs = append(hw.sliceMD5Hexs, md5Hex) + hw.sliceMD5.Reset() + hw.sliceWritten = 0 +} + +// finishPiece 完成当前 piece 的 SHA-1 计算 +func (hw *HashWriter) finishPiece() { + hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...) + hw.pieceSHA1.Reset() + hw.pieceWritten = 0 +} + +// Finish 完成所有哈希计算（处理最后不完整的分片/piece） +func (hw *HashWriter) Finish() { + // 处理最后一个不完整的分片 + if hw.sliceWritten > 0 { + hw.finishSlice() + } + // 处理最后一个不完整的 piece + if hw.pieceWritten > 0 { + hw.finishPiece() + } +} + +// GetFileMD5 获取整文件 MD5（大写十六进制） +func (hw *HashWriter) GetFileMD5() string { + return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil))) +} + +// GetSliceMD5s 获取所有分片的 MD5 列表 +func (hw *HashWriter) GetSliceMD5s() []string { + return hw.sliceMD5Hexs +} + +// GetSliceMD5 获取最终的 sliceMD5（用于秒传） +func (hw *HashWriter) GetSliceMD5(fileMD5 string) string { + if len(hw.sliceMD5Hexs) <= 1 { + return fileMD5 + } + joined := strings.Join(hw.sliceMD5Hexs, "\n") + return strings.ToUpper(GetMD5Str(joined)) +} + +// GetPieceHashes 获取所有 piece 的 SHA-1 哈希拼接 +func (hw *HashWriter) GetPieceHashes() []byte { + return hw.pieceHashes +} + +// GetTotalWritten 获取总写入字节数 +func (hw *HashWriter) GetTotalWritten() int64 { + return hw.totalWritten +} + +// BuildTorrent 根据计算结果构建 Torrent 结构 +func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent { + fileMD5 := hw.GetFileMD5() + sliceMD5 := hw.GetSliceMD5(fileMD5) + + t := NewTorrent(fileName, fileSize, fileMD5) + t.SetPieces(hw.GetPieceHashes()) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: hw.GetSliceMD5s(), + SliceSize: hw.sliceSize, + Cloud: "189", + }) + + return t +} + +// BuildTorrentBytes 构建并编码 torrent 文件 +func (hw *HashWriter) BuildTorrentBytes(fileName string, fileSize int64) ([]byte, error) { + t := hw.BuildTorrent(fileName, fileSize) + return t.Encode() +} + +// CopyAndHash 从 reader 读取数据，同时写入 writer 和 HashWriter +func CopyAndHash(dst io.Writer, src io.Reader, hw *HashWriter) (int64, error) { + buf := make([]byte, 32*1024) // 32KB buffer + var written int64 + for { + nr, er := src.Read(buf) + if nr > 0 { + // 写入 HashWriter + hw.Write(buf[:nr]) + // 写入目标 + if dst != nil { + nw, ew := dst.Write(buf[:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = fmt.Errorf("invalid write result") + } + } + written += int64(nw) + if ew != nil { + return written, ew + } + if nr != nw { + return written, io.ErrShortWrite + } + } else { + written += int64(nr) + } + } + if er != nil { + if er == io.EOF { + break + } + return written, er + } + } + return written, nil +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/pkg/torrent/torrent.go b/pkg/torrent/torrent.go new file mode 100644 index 0000000000..8744e6362e --- /dev/null +++ b/pkg/torrent/torrent.go @@ -0,0 +1,439 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "time" +) + +const ( + // DefaultPieceSize 默认分片大小 10MB，与天翼云 CAS 分片大小一致 + DefaultPieceSize int64 = 10 * 1024 * 1024 + + // CASExtensionKey torrent 根字典中的 CAS 扩展 key + CASExtensionKey = "x-cas" + + // CASSliceSizeKey CAS 分片大小 key + CASSliceSizeKey = "slice_size" + // CASSliceMD5sKey 每片 MD5 列表 key + CASSliceMD5sKey = "slice_md5s" + // CASSliceMD5Key 最终 sliceMd5 key + CASSliceMD5Key = "slice_md5" + // CASFileMD5Key 整文件 MD5 key + CASFileMD5Key = "file_md5" + // CASCloudKey 云盘类型 key + CASCloudKey = "cloud" +) + +// CASInfo 天翼云 CAS 秒传所需信息 +type CASInfo struct { + // FileMD5 整文件 MD5（大写十六进制） + FileMD5 string + // SliceMD5 分片 MD5 的摘要（大写十六进制） + SliceMD5 string + // SliceMD5s 每个 10MB 分片的 MD5（大写十六进制） + SliceMD5s []string + // SliceSize 分片大小（字节） + SliceSize int64 + // Cloud 云盘类型标识 + Cloud string +} + +// TorrentFile 表示 torrent 中的单个文件 +type TorrentFile struct { + // Length 文件大小（字节） + Length int64 + // Path 文件路径（多文件模式下的相对路径各段） + Path []string + // MD5Sum 文件的 MD5（可选，BT 标准字段） + MD5Sum string +} + +// TorrentInfo torrent 的 info 字典 +type TorrentInfo struct { + // PieceLength 分片大小 + PieceLength int64 + // Pieces 所有分片的 SHA-1 哈希拼接（每 20 字节一个） + Pieces []byte + // Name 种子名称（单文件模式为文件名，多文件模式为目录名） + Name string + // Length 单文件模式下的文件大小 + Length int64 + // Files 多文件模式下的文件列表 + Files []TorrentFile + // MD5Sum 单文件模式下的文件 MD5（可选） + MD5Sum string +} + +// Torrent 完整的 torrent 文件结构 +type Torrent struct { + // Info info 字典 + Info TorrentInfo + // InfoHash info 字典的 SHA-1 哈希（20 字节） + InfoHash []byte + // Announce tracker URL + Announce string + // AnnounceList tracker 列表 + AnnounceList [][]string + // CreationDate 创建时间 + CreationDate int64 + // Comment 注释 + Comment string + // CreatedBy 创建者 + CreatedBy string + // CAS 天翼云 CAS 扩展信息（存储在 info 字典外部，不影响 info_hash） + CAS *CASInfo +} + +// NewTorrent 创建一个新的 torrent 结构 +func NewTorrent(name string, fileSize int64, fileMD5 string) *Torrent { + return &Torrent{ + Info: TorrentInfo{ + PieceLength: DefaultPieceSize, + Name: name, + Length: fileSize, + MD5Sum: fileMD5, + }, + CreationDate: time.Now().Unix(), + CreatedBy: "OpenList", + Comment: "Generated by OpenList with CAS extension", + } +} + +// SetPieces 设置 SHA-1 分片哈希 +func (t *Torrent) SetPieces(pieces []byte) { + t.Info.Pieces = pieces +} + +// SetCASInfo 设置 CAS 扩展信息 +func (t *Torrent) SetCASInfo(cas *CASInfo) { + t.CAS = cas +} + +// Encode 将 torrent 编码为 bencode 格式的字节数组 +func (t *Torrent) Encode() ([]byte, error) { + // 构建 info 字典 + infoDict := make(map[string]interface{}) + infoDict["piece length"] = int64(t.Info.PieceLength) + infoDict["pieces"] = t.Info.Pieces + infoDict["name"] = t.Info.Name + + if len(t.Info.Files) > 0 { + // 多文件模式 + files := make([]interface{}, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + fileDict := make(map[string]interface{}) + fileDict["length"] = int64(f.Length) + path := make([]interface{}, 0, len(f.Path)) + for _, p := range f.Path { + path = append(path, p) + } + fileDict["path"] = path + if f.MD5Sum != "" { + fileDict["md5sum"] = f.MD5Sum + } + files = append(files, fileDict) + } + infoDict["files"] = files + } else { + // 单文件模式 + infoDict["length"] = int64(t.Info.Length) + if t.Info.MD5Sum != "" { + infoDict["md5sum"] = t.Info.MD5Sum + } + } + + // 编码 info 字典并计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info dict: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 构建根字典 + rootDict := make(map[string]interface{}) + if t.Announce != "" { + rootDict["announce"] = t.Announce + } + if len(t.AnnounceList) > 0 { + announceList := make([]interface{}, 0, len(t.AnnounceList)) + for _, tier := range t.AnnounceList { + tierList := make([]interface{}, 0, len(tier)) + for _, url := range tier { + tierList = append(tierList, url) + } + announceList = append(announceList, tierList) + } + rootDict["announce-list"] = announceList + } + if t.Comment != "" { + rootDict["comment"] = t.Comment + } + if t.CreatedBy != "" { + rootDict["created by"] = t.CreatedBy + } + if t.CreationDate > 0 { + rootDict["creation date"] = t.CreationDate + } + + // info 字典使用原始编码的字节（保证 info_hash 一致） + rootDict["info"] = infoDict + + // CAS 扩展信息（放在 info 外部，不影响 info_hash） + if t.CAS != nil { + casDict := make(map[string]interface{}) + casDict[CASCloudKey] = t.CAS.Cloud + casDict[CASFileMD5Key] = t.CAS.FileMD5 + casDict[CASSliceMD5Key] = t.CAS.SliceMD5 + casDict[CASSliceSizeKey] = t.CAS.SliceSize + + if len(t.CAS.SliceMD5s) > 0 { + md5List := make([]interface{}, 0, len(t.CAS.SliceMD5s)) + for _, md5 := range t.CAS.SliceMD5s { + md5List = append(md5List, md5) + } + casDict[CASSliceMD5sKey] = md5List + } + rootDict[CASExtensionKey] = casDict + } + + return BencodeEncode(rootDict) +} + +// Decode 从 bencode 字节数组解析 torrent +func Decode(data []byte) (*Torrent, error) { + val, err := BencodeDecode(data) + if err != nil { + return nil, fmt.Errorf("bencode decode: %w", err) + } + + rootDict, ok := val.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: root is not a dict") + } + + t := &Torrent{} + + // 解析 announce + if v, ok := rootDict["announce"]; ok { + if b, ok := v.([]byte); ok { + t.Announce = string(b) + } + } + + // 解析 announce-list + if v, ok := rootDict["announce-list"]; ok { + if list, ok := v.([]interface{}); ok { + for _, tier := range list { + if tierList, ok := tier.([]interface{}); ok { + var urls []string + for _, u := range tierList { + if b, ok := u.([]byte); ok { + urls = append(urls, string(b)) + } + } + if len(urls) > 0 { + t.AnnounceList = append(t.AnnounceList, urls) + } + } + } + } + } + + // 解析 comment + if v, ok := rootDict["comment"]; ok { + if b, ok := v.([]byte); ok { + t.Comment = string(b) + } + } + + // 解析 created by + if v, ok := rootDict["created by"]; ok { + if b, ok := v.([]byte); ok { + t.CreatedBy = string(b) + } + } + + // 解析 creation date + if v, ok := rootDict["creation date"]; ok { + if n, ok := v.(int64); ok { + t.CreationDate = n + } + } + + // 解析 info 字典 + if infoVal, ok := rootDict["info"]; ok { + infoDict, ok := infoVal.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: info is not a dict") + } + + // 计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info for hash: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 解析 info 字段 + if v, ok := infoDict["piece length"]; ok { + if n, ok := v.(int64); ok { + t.Info.PieceLength = n + } + } + if v, ok := infoDict["pieces"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Pieces = b + } + } + if v, ok := infoDict["name"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Name = string(b) + } + } + if v, ok := infoDict["length"]; ok { + if n, ok := v.(int64); ok { + t.Info.Length = n + } + } + if v, ok := infoDict["md5sum"]; ok { + if b, ok := v.([]byte); ok { + t.Info.MD5Sum = string(b) + } + } + + // 解析多文件模式 + if v, ok := infoDict["files"]; ok { + if files, ok := v.([]interface{}); ok { + for _, f := range files { + if fileDict, ok := f.(map[string]interface{}); ok { + tf := TorrentFile{} + if l, ok := fileDict["length"]; ok { + if n, ok := l.(int64); ok { + tf.Length = n + } + } + if p, ok := fileDict["path"]; ok { + if pathList, ok := p.([]interface{}); ok { + for _, pp := range pathList { + if b, ok := pp.([]byte); ok { + tf.Path = append(tf.Path, string(b)) + } + } + } + } + if m, ok := fileDict["md5sum"]; ok { + if b, ok := m.([]byte); ok { + tf.MD5Sum = string(b) + } + } + t.Info.Files = append(t.Info.Files, tf) + } + } + } + } + } + + // 解析 CAS 扩展 + if casVal, ok := rootDict[CASExtensionKey]; ok { + if casDict, ok := casVal.(map[string]interface{}); ok { + cas := &CASInfo{} + if v, ok := casDict[CASCloudKey]; ok { + if b, ok := v.([]byte); ok { + cas.Cloud = string(b) + } + } + if v, ok := casDict[CASFileMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.FileMD5 = string(b) + } + } + if v, ok := casDict[CASSliceMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.SliceMD5 = string(b) + } + } + if v, ok := casDict[CASSliceSizeKey]; ok { + if n, ok := v.(int64); ok { + cas.SliceSize = n + } + } + if v, ok := casDict[CASSliceMD5sKey]; ok { + if list, ok := v.([]interface{}); ok { + for _, item := range list { + if b, ok := item.([]byte); ok { + cas.SliceMD5s = append(cas.SliceMD5s, string(b)) + } + } + } + } + t.CAS = cas + } + } + + return t, nil +} + +// GetInfoHashHex 获取 info_hash 的十六进制字符串 +func (t *Torrent) GetInfoHashHex() string { + return hex.EncodeToString(t.InfoHash) +} + +// GetPieceHashes 获取所有分片的 SHA-1 哈希（每个 20 字节） +func (t *Torrent) GetPieceHashes() [][]byte { + if len(t.Info.Pieces) == 0 { + return nil + } + count := len(t.Info.Pieces) / 20 + hashes := make([][]byte, count) + for i := 0; i < count; i++ { + hashes[i] = t.Info.Pieces[i*20 : (i+1)*20] + } + return hashes +} + +// GetTotalSize 获取 torrent 中所有文件的总大小 +func (t *Torrent) GetTotalSize() int64 { + if len(t.Info.Files) > 0 { + var total int64 + for _, f := range t.Info.Files { + total += f.Length + } + return total + } + return t.Info.Length +} + +// HasCASInfo 检查 torrent 是否包含 CAS 扩展信息 +func (t *Torrent) HasCASInfo() bool { + return t.CAS != nil && t.CAS.FileMD5 != "" && t.CAS.SliceMD5 != "" +} + +// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息 +func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo { + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + // 所有分片 MD5 用 \n 拼接后再取 MD5 + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + return &CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + } +} + +// GetMD5Str 计算字符串的 MD5（大写十六进制） +func GetMD5Str(data string) string { + h := md5.New() + h.Write([]byte(data)) + return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) +} diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 596e61e541..c4b4e735f2 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -90,6 +90,12 @@ var ( // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) + + // SHA1_128K is SHA1 of first 128KB, used by 115 driver for rapid upload + SHA1_128K = RegisterHash("sha1_128k", "SHA1-128K", 40, sha1.New) + + // PRE_HASH is SHA1 of first 1024 bytes, used by Aliyundrive for rapid upload + PRE_HASH = RegisterHash("pre_hash", "PRE-HASH", 40, sha1.New) ) // HashData get hash of one hashType diff --git a/public/dist/README.md b/public/dist/README.md deleted file mode 100644 index d8709fb571..0000000000 --- a/public/dist/README.md +++ /dev/null @@ -1 +0,0 @@ -## Put dist of frontend here. \ No newline at end of file diff --git a/server/common/common.go b/server/common/common.go index d780512684..a1a9a6319e 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -128,13 +128,24 @@ func Pluralize(count int, singular, plural string) string { return plural } -func GinWithValue(c *gin.Context, keyAndValue ...any) { +type requestContext struct { + context.Context +} + +// GinAppendValues 向当前请求上下文追加键值，提供类似 gin.Context Set/Get 的可变语义。 +// 同一请求内，已持有的上下文引用会同步看到后续更新。 +func GinAppendValues(c *gin.Context, keyAndValue ...any) { + ctx := c.Request.Context() + if r, ok := ctx.(*requestContext); ok { + r.Context = ContentWithValues(r.Context, keyAndValue...) + return + } c.Request = c.Request.WithContext( - ContentWithValue(c.Request.Context(), keyAndValue...), + &requestContext{ContentWithValues(ctx, keyAndValue...)}, ) } -func ContentWithValue(ctx context.Context, keyAndValue ...any) context.Context { +func ContentWithValues(ctx context.Context, keyAndValue ...any) context.Context { if len(keyAndValue) < 1 || len(keyAndValue)%2 != 0 { panic("keyAndValue must be an even number of arguments (key, value, ...)") } diff --git a/server/common/proxy.go b/server/common/proxy.go index c7c975d255..06f8c21d58 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -70,6 +70,28 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. }) return err } +// ApplyProxyUserAgent overrides the User-Agent on the client request header used to +// resolve and proxy a download when the storage configured a custom one. It mutates +// the (request-scoped) request header BEFORE the link is resolved — deliberately not +// the cached *model.Link — so it: +// - feeds drivers that sign the download URL with the request UA (e.g. 115 calls +// DownURL with args.Header's User-Agent), and is keyed correctly by drivers whose +// LinkCacheMode includes the UA; +// - covers transparent-proxy drivers too, since ProcessHeader copies the request +// header (the override) when link.Header carries no User-Agent; +// - never races on or pollutes the shared link cache (the previous approach mutated +// link.Header after caching, leaking the UA across concurrent/later downloads). +// +// Empty config keeps the default behavior (passthrough the client's User-Agent). A +// driver that pins a fixed UA of its own still wins, which is intended — that UA is +// usually required for the link to resolve. +func ApplyProxyUserAgent(r *http.Request, storage *model.Storage) { + if r == nil || storage == nil || storage.ProxyUserAgent == "" { + return + } + r.Header.Set("User-Agent", storage.ProxyUserAgent) +} + func attachHeader(w http.ResponseWriter, file model.Obj, link *model.Link) { fileName := file.GetName() w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(fileName)) diff --git a/server/handles/archive.go b/server/handles/archive.go index d46f83c868..364e93edc5 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -105,7 +105,7 @@ func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -188,7 +188,7 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/handles/auth.go b/server/handles/auth.go index 7800690918..0baeb63db8 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -8,6 +8,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -79,6 +80,7 @@ func loginHash(c *gin.Context, req *LoginReq) { } common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) + plugin.FireHook(plugin.HookUserLoginAfter, map[string]any{"username": user.Username}) } type UserResp struct { @@ -111,6 +113,13 @@ func UpdateCurrent(c *gin.Context) { common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } + // Changing the username or setting a new password requires the manage-user-info + // permission (admins always have it). SSO link/unlink — which posts the + // unchanged username with an empty password — stays open to every non-guest. + if (req.Username != user.Username || req.Password != "") && !user.CanManageUserInfo() { + common.ErrorStrResp(c, model.NoPermissionUpdateProfile, 403) + return + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) diff --git a/server/handles/cluster.go b/server/handles/cluster.go new file mode 100644 index 0000000000..d07c573185 --- /dev/null +++ b/server/handles/cluster.go @@ -0,0 +1,110 @@ +package handles + +import ( + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/cluster" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// ClusterWS is the peer-to-peer endpoint nodes use to establish a persistent, +// stateful sync connection (WebSocket). A persistent connection is required so a +// node behind NAT can participate: it dials OUT to this endpoint and keeps the +// link open, since it cannot be dialed itself. The HTTP upgrade is open; +// authentication and confidentiality come from the cluster pre-shared key (only +// PSK holders can seal/open the per-frame AEAD envelope). +func ClusterWS(c *gin.Context) { + m := cluster.Default + if m == nil { + c.Status(http.StatusServiceUnavailable) + return + } + m.ServeWS(c.Writer, c.Request) +} + +// ---- Admin config/status ---- + +func clusterMgr(c *gin.Context) *cluster.Manager { + if cluster.Default == nil { + common.ErrorStrResp(c, "cluster sync is unavailable", 500) + return nil + } + return cluster.Default +} + +// ClusterGetConfig returns the current cluster config (key redacted) plus a live +// status overview. Admin only. +func ClusterGetConfig(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, gin.H{ + "config": m.GetConfig(true), + "status": m.Status(), + }) +} + +// ClusterRevealKey returns the cluster pre-shared key in PLAINTEXT for the admin +// UI's "show key" toggle. ClusterGetConfig redacts the key (it is polled live), +// so this dedicated endpoint serves the real value only on explicit request. It +// lives in the admin-only cluster group, same as the other config routes. +func ClusterRevealKey(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, gin.H{"key": m.GetConfig(false).Key}) +} + +// ClusterSetConfig persists a new cluster config. A blank or "********" key keeps +// the existing key. Admin only. +func ClusterSetConfig(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + var cfg cluster.Config + if err := c.ShouldBind(&cfg); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := m.SetConfig(cfg); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, gin.H{ + "config": m.GetConfig(true), + "status": m.Status(), + }) +} + +// ClusterStatus returns just the live status overview. Admin only. +func ClusterStatus(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, m.Status()) +} + +// ClusterSetGroups replaces the cluster-shared sync-group document. Admin only. +func ClusterSetGroups(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + var req struct { + Groups []cluster.GroupSpec `json:"groups"` + } + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := m.SetGroups(req.Groups); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, m.Status()) +} diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go index 69cfd2fa81..d77c3044cb 100644 --- a/server/handles/direct_upload.go +++ b/server/handles/direct_upload.go @@ -2,8 +2,10 @@ package handles import ( "net/url" + stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/server/common" @@ -38,15 +40,29 @@ func FsGetDirectUploadInfo(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if err := checkRelativePath(req.FileName); err != nil { + common.ErrorResp(c, err, 403) + return + } overwrite := c.GetHeader("Overwrite") != "false" + dstPath := stdpath.Join(path, req.FileName) if !overwrite { - if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + res, err := fs.Get(c.Request.Context(), dstPath, &fs.GetArgs{NoLog: true}) + if err != nil && !errs.IsObjectNotFound(err) { + common.ErrorResp(c, err, 500) + return + } + if res != nil { common.ErrorStrResp(c, "file exists", 403) return } } - directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) + directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite) if err != nil { + if !overwrite && errs.IsObjectAlreadyExists(err) { + common.ErrorStrResp(c, "file exists", 403) + return + } common.ErrorResp(c, err, 500) return } diff --git a/server/handles/down.go b/server/handles/down.go index 50025a0b69..828fc7033b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -58,6 +58,10 @@ func Proxy(c *gin.Context) { return } } + // Override the upstream User-Agent before resolving the link so drivers that + // sign the download URL with it (e.g. 115) get the configured UA, not the + // passthrough client UA. No-op when proxy_user_agent is unset. + common.ApplyProxyUserAgent(c.Request, storage.GetStorage()) link, file, err := fs.Link(c.Request.Context(), rawPath, model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), diff --git a/server/handles/favorite.go b/server/handles/favorite.go new file mode 100644 index 0000000000..68f6a6f765 --- /dev/null +++ b/server/handles/favorite.go @@ -0,0 +1,117 @@ +package handles + +import ( + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// normalizeFavoritePath cleans a favorite path so that dedup/toggle compares +// equal paths consistently (backslashes, missing leading slash, ".."/"." are +// all normalized away). +func normalizeFavoritePath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "/" + } + return utils.FixAndCleanPath(path) +} + +type FavoriteAddReq struct { + Path string `json:"path" binding:"required"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Tag string `json:"tag"` +} + +func currentUser(c *gin.Context) (*model.User, bool) { + userObj, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || userObj == nil { + return nil, false + } + return userObj, true +} + +func ListFavorites(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + favorites, err := db.GetFavoritesByUser(userObj.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, favorites) +} + +func AddFavorite(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var req FavoriteAddReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorStrResp(c, "request invalid", 400) + return + } + path := normalizeFavoritePath(req.Path) + // dedup/toggle: if a favorite with the same (userID, path) exists, just + // update its tag instead of creating a duplicate. + existing, err := db.GetFavoriteByUserAndPath(userObj.ID, path) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if existing != nil { + existing.Tag = req.Tag + existing.Name = req.Name + existing.IsDir = req.IsDir + if err := db.UpdateFavorite(existing); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, existing) + return + } + favorite := &model.Favorite{ + UserID: userObj.ID, + Path: path, + Name: req.Name, + IsDir: req.IsDir, + Tag: req.Tag, + CreatedAt: time.Now().Unix(), + } + if err := db.CreateFavorite(favorite); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, favorite) +} + +func DeleteFavorite(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + id, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + if err := db.DeleteFavorite(userObj.ID, uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/favorite_test.go b/server/handles/favorite_test.go new file mode 100644 index 0000000000..f85a21e413 --- /dev/null +++ b/server/handles/favorite_test.go @@ -0,0 +1,38 @@ +package handles + +import "testing" + +func TestNormalizeFavoritePath(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty", "", "/"}, + {"whitespace only", " ", "/"}, + {"already clean", "/a/b", "/a/b"}, + {"missing leading slash", "a/b", "/a/b"}, + {"backslashes", "\\a\\b", "/a/b"}, + {"trailing slash", "/a/b/", "/a/b"}, + {"dot segments", "/a/../b", "/b"}, + {"root", "/", "/"}, + {"trim spaces", " /a/b ", "/a/b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeFavoritePath(tc.in); got != tc.want { + t.Fatalf("normalizeFavoritePath(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestNormalizeFavoritePathDedup verifies that two inputs that should refer to +// the same favorite normalize to the same key (so toggle/dedup matches them). +func TestNormalizeFavoritePathDedup(t *testing.T) { + a := normalizeFavoritePath("a/b/") + b := normalizeFavoritePath("\\a\\b") + if a != b { + t.Fatalf("expected equal normalized paths, got %q and %q", a, b) + } +} diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 28588d6683..e672165cd8 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -49,7 +49,7 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, srcMeta) + common.GinAppendValues(c, conf.MetaKey, srcMeta) dstDir, err := user.JoinPath(req.DstDir) if err != nil { @@ -183,7 +183,7 @@ func FsBatchRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { if renameObject.SrcName == "" || renameObject.NewName == "" { continue @@ -236,7 +236,7 @@ func FsRegexRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) srcRegexp, err := regexp.Compile(req.SrcNameRegex) if err != nil { diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 94e7c3fec5..b3c3581780 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -10,6 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" @@ -119,6 +120,7 @@ func FsMove(c *gin.Context) { req.Names[i] = "" continue } + req.Names[i] = srcPath if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { @@ -129,12 +131,11 @@ func FsMove(c *gin.Context) { if !req.SkipExisting { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return - } else { - continue } + req.Names[i] = "" + continue } } - req.Names[i] = srcPath } // Create all tasks immediately without any synchronous validation @@ -222,6 +223,7 @@ func FsCopy(c *gin.Context) { req.Names[i] = "" continue } + req.Names[i] = srcPath if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { @@ -233,11 +235,11 @@ func FsCopy(c *gin.Context) { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } else if !req.Merge || !res.IsDir() { + req.Names[i] = "" continue } } } - req.Names[i] = srcPath } // Create all tasks immediately without any synchronous validation @@ -425,7 +427,7 @@ func FsRemoveEmptyDirectory(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { @@ -527,4 +529,5 @@ func Link(c *gin.Context) { } defer link.Close() common.SuccessResp(c, link) + plugin.FireHook(plugin.HookFsLinkAfter, map[string]any{"path": rawPath}) } diff --git a/server/handles/fsmanage_test.go b/server/handles/fsmanage_test.go new file mode 100644 index 0000000000..e2de6508c6 --- /dev/null +++ b/server/handles/fsmanage_test.go @@ -0,0 +1,100 @@ +package handles + +import ( + "testing" +) + +// TestClearSkippedNames verifies the contract from upstream fix #2520: +// When SkipExisting is true and a file already exists at the destination, +// that name must be cleared (set to "") so the task loop skips it. +// Valid (non-skipped) names must be set to full srcPath. +// +// This is a contract test for the name-filtering loop in FsMove/FsCopy. +// It tests the logic in isolation without needing gin/fs dependencies. +func TestClearSkippedNames(t *testing.T) { + // Simulate the name-filtering loop logic from FsMove/FsCopy. + // existsAtDst simulates whether a file already exists at the destination. + filterNames := func(srcDir string, names []string, overwrite, skipExisting bool, existsAtDst func(name string) bool) []string { + result := make([]string, len(names)) + copy(result, names) + for i, name := range result { + srcPath := srcDir + "/" + name + // First: set to srcPath (the fix moves this before skip check) + result[i] = srcPath + if !overwrite { + if existsAtDst(name) { + if !skipExisting { + // Would return error in real code + result[i] = "ERROR" + return result + } + // Skip: must clear to "" + result[i] = "" + continue + } + } + } + return result + } + + tests := []struct { + name string + srcDir string + names []string + overwrite bool + skipExisting bool + existsAtDst func(string) bool + want []string + }{ + { + name: "skip existing files clears their names", + srcDir: "/src", + names: []string{"a.mkv", "b.mkv", "c.mkv"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(n string) bool { return n == "b.mkv" }, + want: []string{"/src/a.mkv", "", "/src/c.mkv"}, + }, + { + name: "no skipping when overwrite is true", + srcDir: "/src", + names: []string{"a.mkv", "b.mkv"}, + overwrite: true, + skipExisting: false, + existsAtDst: func(string) bool { return true }, + want: []string{"/src/a.mkv", "/src/b.mkv"}, + }, + { + name: "all files skipped results in all empty", + srcDir: "/src", + names: []string{"x.mp4", "y.mp4"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(string) bool { return true }, + want: []string{"", ""}, + }, + { + name: "no existing files keeps all paths", + srcDir: "/data", + names: []string{"movie.mkv"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(string) bool { return false }, + want: []string{"/data/movie.mkv"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := filterNames(tt.srcDir, tt.names, tt.overwrite, tt.skipExisting, tt.existsAtDst) + if len(got) != len(tt.want) { + t.Fatalf("length mismatch: got %d, want %d", len(got), len(tt.want)) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("names[%d] = %q, want %q", i, got[i], tt.want[i]) + } + } + }) + } +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index a90fc1082b..e9c9ae0e35 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -7,10 +7,12 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -44,6 +46,9 @@ type ObjResp struct { HashInfoStr string `json:"hashinfo"` HashInfo map[*utils.HashType]string `json:"hash_info"` MountDetails *model.StorageDetails `json:"mount_details,omitempty"` + // Extra carries optional driver-specific metadata (e.g. media duration, + // video resolution, starred). Clients render known keys and ignore the rest. + Extra map[string]any `json:"extra,omitempty"` } type FsListResp struct { @@ -55,6 +60,11 @@ type FsListResp struct { WriteContentBypass bool `json:"write_content_bypass"` Provider string `json:"provider"` DirectUploadTools []string `json:"direct_upload_tools,omitempty"` + // MountDetails is the disk usage of the storage that the *current directory* + // belongs to (nil at the virtual storages-root or when hidden). It lets the + // web header show the current mount's usage dynamically on folder navigation + // without firing an extra fs/get per click. + MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } func FsListSplit(c *gin.Context) { @@ -88,7 +98,7 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -109,10 +119,19 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string - if canWriteContentAtPath { - if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + var mountDetails *model.StorageDetails + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + if canWriteContentAtPath { directUploadTools = op.GetDirectUploadTools(storage) } + // Current directory's storage usage for the web header. Cache-backed + + // singleflight in op.GetStorageDetails, so concurrent multi-user access + // collapses to at most one provider call per storage per TTL — no 429. + if !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails) { + if d, e := op.GetStorageDetails(c.Request.Context(), storage); e == nil { + mountDetails = d + } + } } common.SuccessResp(c, FsListResp{ Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), @@ -123,7 +142,14 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { WriteContentBypass: common.CanWriteContentBypassUserPerms(meta, reqPath), Provider: provider, DirectUploadTools: directUploadTools, + MountDetails: mountDetails, }) + if plugin.HasSubscribers(plugin.HookFsListAfter) { + plugin.FireHook(plugin.HookFsListAfter, map[string]any{ + "path": reqPath, + "count": total, + }) + } } func FsDirs(c *gin.Context) { @@ -152,7 +178,7 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -230,6 +256,11 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + extra, _ := model.GetExtra(obj) + hashInfo := obj.GetHash().Export() + if hashInfo == nil { + hashInfo = make(map[*utils.HashType]string) + } resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), @@ -237,11 +268,12 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), + HashInfo: hashInfo, Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), MountDetails: mountDetails, + Extra: extra, }) } return resp @@ -291,7 +323,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -356,6 +388,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + extra, _ := model.GetExtra(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ Name: obj.GetName(), @@ -369,6 +402,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { Type: utils.GetFileType(obj.GetName()), Thumb: thumb, MountDetails: mountDetails, + Extra: extra, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), @@ -378,6 +412,200 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { }) } +type FsVideoPlayReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` +} + +// FsVideoPlay exposes the storage provider's official online-play (transcoded +// streaming) sources for a video at multiple resolutions, for drivers that +// implement driver.VideoPlayer (e.g. 115_open). +func FsVideoPlay(c *gin.Context) { + var req FsVideoPlayReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if user.IsGuest() && user.Disabled { + common.ErrorStrResp(c, "Guest user is disabled, login please", 401) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + common.GinAppendValues(c, conf.MetaKey, meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + vp, ok := storage.(driver.VideoPlayer) + if !ok { + common.ErrorStrResp(c, "driver does not support official play sources", 400) + return + } + obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + sources, err := vp.VideoPlay(c.Request.Context(), obj) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // Route every transcoded source through OpenList's signed video proxy so the + // browser fetches them same-origin instead of hitting the provider CDN + // directly (which fails CORS for 115's online-play HLS). + apiURL := common.GetApiUrl(c) + for i := range sources { + if sources[i].URL != "" { + sources[i].URL = BuildVideoProxyURL(apiURL, sources[i].URL) + } + } + common.SuccessResp(c, sources) +} + +// FsVideoSubtitle exposes the provider's subtitle tracks for a video, for +// drivers that implement driver.VideoSubtitleProvider (e.g. 115_open). These +// subtitles are independent of the play source, so the frontend can render them +// on every quality tier — including transcoded HLS streams that drop the +// original container's embedded subtitle tracks. +func FsVideoSubtitle(c *gin.Context) { + var req FsVideoPlayReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if user.IsGuest() && user.Disabled { + common.ErrorStrResp(c, "Guest user is disabled, login please", 401) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + common.GinAppendValues(c, conf.MetaKey, meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // Same-name sidecar subtitle files (any driver): list the folder once and + // match basename siblings carrying a subtitle extension. This is the unified + // subtitle source so the frontend doesn't have to compute it from `related`. + subs := resolveSidecarSubtitles(c, reqPath, obj) + // Provider subtitle tracks (e.g. 115 extracts a container's embedded subs + // during transcoding) — appended when the driver supports them. Best-effort: + // a provider error still returns the sidecar subs we already resolved. + if vsp, ok := storage.(driver.VideoSubtitleProvider); ok { + if providerSubs, perr := vsp.VideoSubtitle(c.Request.Context(), obj); perr == nil { + // Route provider subtitle files through the signed video proxy so the + // browser fetches them same-origin (provider CDNs reject cross-origin + // subtitle fetches). + apiURL := common.GetApiUrl(c) + for i := range providerSubs { + if providerSubs[i].URL != "" { + providerSubs[i].URL = BuildVideoProxyURL(apiURL, providerSubs[i].URL) + } + } + subs = append(subs, providerSubs...) + } + } + common.SuccessResp(c, subs) +} + +// subtitleExts are the sidecar subtitle file types matched as same-name siblings +// of a video. Matching is case-insensitive. +var subtitleExts = map[string]bool{ + "srt": true, "vtt": true, "ass": true, "ssa": true, "sup": true, +} + +// matchSidecarSubtitle reports whether sibling is a same-name sidecar subtitle of +// the video (basename.<...>. with ext ∈ subtitleExts), returning the +// lowercased subtitle extension. It requires a '.' right after the video's +// basename so "Movie2.srt" doesn't match video "Movie.mkv", and ignores the file +// that is the video itself. +func matchSidecarSubtitle(videoName, siblingName string) (string, bool) { + if strings.EqualFold(videoName, siblingName) { + return "", false + } + ext := strings.TrimPrefix(strings.ToLower(stdpath.Ext(siblingName)), ".") + if !subtitleExts[ext] { + return "", false + } + base := strings.ToLower(strings.TrimSuffix(videoName, stdpath.Ext(videoName))) + sib := strings.ToLower(siblingName) + if !strings.HasPrefix(sib, base) || !strings.HasPrefix(sib[len(base):], ".") { + return "", false + } + return ext, true +} + +// resolveSidecarSubtitles lists the video's parent folder and returns the +// same-name sidecar subtitle files (multi-type / multi-language) as +// VideoSubtitleInfo with signed same-origin /p proxy URLs (mirroring the +// frontend proxyLink + FsGet rawURL construction). +func resolveSidecarSubtitles(c *gin.Context, reqPath string, video model.Obj) []driver.VideoSubtitleInfo { + parentPath := stdpath.Dir(reqPath) + siblings, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) + if err != nil { + return nil + } + parentMeta, _ := op.GetNearestMeta(parentPath) + encrypt := isEncrypt(parentMeta, parentPath) + apiURL := common.GetApiUrl(c) + var subs []driver.VideoSubtitleInfo + for _, o := range siblings { + if o.IsDir() { + continue + } + ext, ok := matchSidecarSubtitle(video.GetName(), o.GetName()) + if !ok { + continue + } + query := "" + if s := common.Sign(o, parentPath, encrypt); s != "" { + query = "?sign=" + s + } + siblingPath := stdpath.Join(parentPath, o.GetName()) + subs = append(subs, driver.VideoSubtitleInfo{ + Title: strings.TrimSuffix(o.GetName(), stdpath.Ext(o.GetName())), + Type: ext, + URL: fmt.Sprintf("%s/p%s%s", apiURL, utils.EncodePath(siblingPath, true), query), + }) + } + return subs +} + func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { var related []model.Obj nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) @@ -415,7 +643,7 @@ func FsOther(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/handles/fsread_test.go b/server/handles/fsread_test.go index 3947ae27f8..7158677067 100644 --- a/server/handles/fsread_test.go +++ b/server/handles/fsread_test.go @@ -172,6 +172,37 @@ func TestGetHeader(t *testing.T) { } } +func TestMatchSidecarSubtitle(t *testing.T) { + tests := []struct { + name string + video string + sibling string + wantExt string + wantOK bool + }{ + {"plain srt", "Movie.mkv", "Movie.srt", "srt", true}, + {"lang-tagged ass", "Movie.mkv", "Movie.zh.ass", "ass", true}, + {"multi-tag vtt", "Show.S01E01.mkv", "Show.S01E01.eng.forced.vtt", "vtt", true}, + {"sup pgs", "Movie.mkv", "Movie.sup", "sup", true}, + {"ssa", "Movie.mkv", "Movie.SSA", "ssa", true}, + {"case-insensitive both", "MOVIE.MKV", "movie.SRT", "srt", true}, + {"prefix collision excluded", "Movie.mkv", "Movie2.srt", "", false}, + {"different basename", "Movie.mkv", "Other.srt", "", false}, + {"non-subtitle ext", "Movie.mkv", "Movie-poster.jpg", "", false}, + {"video itself", "Movie.mkv", "Movie.mkv", "", false}, + {"basename substring not boundary", "Movie.mkv", "Moviegoers.srt", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExt, gotOK := matchSidecarSubtitle(tt.video, tt.sibling) + if gotExt != tt.wantExt || gotOK != tt.wantOK { + t.Errorf("matchSidecarSubtitle(%q, %q) = (%q, %v), want (%q, %v)", + tt.video, tt.sibling, gotExt, gotOK, tt.wantExt, tt.wantOK) + } + }) + } +} + func TestIsEncrypt(t *testing.T) { tests := []struct { name string diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cdf..54cdb4feeb 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -93,6 +93,12 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) @@ -190,6 +196,12 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) diff --git a/server/handles/index.go b/server/handles/index.go index cd270f6f9f..78096ddb7f 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -2,6 +2,7 @@ package handles import ( "context" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -55,18 +56,38 @@ func UpdateIndex(c *gin.Context) { } go func() { ctx := context.Background() + progress, err := search.Progress() + if err != nil { + log.Errorf("get index progress error: %+v", err) + progress = &model.IndexProgress{} + } for _, path := range req.Paths { - err := search.Del(ctx, path) + err = search.Del(ctx, path) if err != nil { log.Errorf("delete index on %s error: %+v", path, err) return } } - err := search.BuildIndex(context.Background(), req.Paths, + err = search.BuildIndex(context.Background(), req.Paths, conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, false) if err != nil { log.Errorf("update index error: %+v", err) + now := time.Now() + search.WriteProgress(&model.IndexProgress{ + ObjCount: progress.ObjCount, + IsDone: true, + LastDoneTime: &now, + Error: err.Error(), + }) + return } + now := time.Now() + search.WriteProgress(&model.IndexProgress{ + ObjCount: progress.ObjCount, + IsDone: true, + LastDoneTime: &now, + Error: "", + }) }() common.SuccessResp(c) } diff --git a/server/handles/plugin.go b/server/handles/plugin.go new file mode 100644 index 0000000000..ea3d2d4497 --- /dev/null +++ b/server/handles/plugin.go @@ -0,0 +1,148 @@ +package handles + +import ( + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/plugin" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// PluginManifest serves the list of frontend (JS) plugins for the web UI to +// hot-load. Always returns a (possibly empty) manifest. +func PluginManifest(c *gin.Context) { + if plugin.Default == nil { + common.SuccessResp(c, plugin.Manifest{Plugins: []plugin.ManifestEntry{}}) + return + } + common.SuccessResp(c, plugin.Default.FrontendManifest()) +} + +// PluginAsset serves a frontend plugin's JS file by name, guarding against path +// traversal. +func PluginAsset(c *gin.Context) { + if plugin.Default == nil { + c.Status(http.StatusNotFound) + return + } + path, ok := plugin.Default.AssetPath(c.Param("name")) + if !ok { + c.Status(http.StatusBadRequest) + return + } + c.File(path) +} + +// ---- Admin management of backend (Go-source) plugins ---- + +// PluginListResp is the payload for the management UI: backend Go plugins plus +// the frontend JS plugins served to the web UI. +type PluginListResp struct { + Go []plugin.PluginInfo `json:"go"` + Frontend []plugin.ManifestEntry `json:"frontend"` +} + +func pluginMgr(c *gin.Context) *plugin.Manager { + if plugin.Default == nil { + common.ErrorStrResp(c, "plugin support is disabled", 500) + return nil + } + return plugin.Default +} + +// PluginList returns all backend Go plugins (with load status) and frontend JS +// plugins. Admin only. +func PluginList(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + common.SuccessResp(c, PluginListResp{ + Go: mgr.ListGoPlugins(), + Frontend: mgr.FrontendManifest().Plugins, + }) +} + +// PluginGet returns a backend plugin's source for editing. Admin only. +func PluginGet(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + src, err := mgr.GetGoPluginSource(c.Query("name")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + common.SuccessResp(c, gin.H{"name": c.Query("name"), "source": src}) +} + +type PluginSaveReq struct { + Name string `json:"name" binding:"required"` + Source string `json:"source" binding:"required"` +} + +// PluginSave creates or overwrites a backend plugin and loads it immediately. +// A successful write whose code fails to load still returns 200 — the load +// error is reported by PluginList. Admin only. +func PluginSave(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginSaveReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.SaveGoPlugin(req.Name, req.Source); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} + +type PluginNameReq struct { + Name string `json:"name" binding:"required"` +} + +// PluginDelete removes a backend plugin and unloads it. Admin only. +func PluginDelete(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginNameReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.DeleteGoPlugin(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} + +type PluginEnableReq struct { + Name string `json:"name" binding:"required"` + Enabled bool `json:"enabled"` +} + +// PluginSetEnabled enables/disables a backend plugin. Admin only. +func PluginSetEnabled(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginEnableReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.SetGoPluginEnabled(req.Name, req.Enabled); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} diff --git a/server/handles/storage.go b/server/handles/storage.go index 687c93551b..0342b54a0b 100644 --- a/server/handles/storage.go +++ b/server/handles/storage.go @@ -90,6 +90,12 @@ func ListStorages(c *gin.Context) { }) } +// StorageLoadingStatus reports the per-storage startup load progress so the +// frontend can render an async loading status bar. +func StorageLoadingStatus(c *gin.Context) { + common.SuccessResp(c, op.StorageLoadProgress.Snapshot()) +} + func CreateStorage(c *gin.Context) { var req model.Storage if err := c.ShouldBind(&req); err != nil { diff --git a/server/handles/torrent.go b/server/handles/torrent.go new file mode 100644 index 0000000000..8b6ee1b6bf --- /dev/null +++ b/server/handles/torrent.go @@ -0,0 +1,433 @@ +package handles + +import ( + "encoding/base64" + "fmt" + "io" + "strings" + + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +// maxTorrentBase64Len is the max allowed Base64-encoded torrent size (~10MB decoded) +const maxTorrentBase64Len = 14 * 1024 * 1024 + +// maxTorrentGenFileSize is the max file size allowed for synchronous torrent generation (1GB) +const maxTorrentGenFileSize = 1 * 1024 * 1024 * 1024 + +// validateParsedTorrent checks that basic torrent invariants hold. +func validateParsedTorrent(t *torrent.Torrent) error { + if len(t.Info.Pieces)%20 != 0 { + return fmt.Errorf("torrent pieces 数据无效：长度必须为 20 的整数倍") + } + return nil +} + +// ParseTorrentReq 解析 torrent 文件请求 +type ParseTorrentReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` +} + +// ParseTorrentResp 解析 torrent 文件响应 +type ParseTorrentResp struct { + // Name 种子名称 + Name string `json:"name"` + // TotalSize 总大小 + TotalSize int64 `json:"total_size"` + // PieceLength 分片大小 + PieceLength int64 `json:"piece_length"` + // PieceCount 分片数量 + PieceCount int `json:"piece_count"` + // InfoHash info_hash（十六进制） + InfoHash string `json:"info_hash"` + // Files 文件列表（多文件模式） + Files []TorrentFileInfo `json:"files"` + // HasCAS 是否包含 CAS 扩展信息 + HasCAS bool `json:"has_cas"` + // CAS CAS 扩展信息 + CAS *CASInfoResp `json:"cas,omitempty"` +} + +// TorrentFileInfo torrent 中的文件信息 +type TorrentFileInfo struct { + Path string `json:"path"` + Size int64 `json:"size"` +} + +// CASInfoResp CAS 信息响应 +type CASInfoResp struct { + FileMD5 string `json:"file_md5"` + SliceMD5 string `json:"slice_md5"` + SliceSize int64 `json:"slice_size"` + Cloud string `json:"cloud"` +} + +// ParseTorrent 解析 torrent 文件，返回文件列表等信息 +func ParseTorrent(c *gin.Context) { + var req ParseTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + // 限制 Base64 输入大小（最大 ~10MB decoded） + if len(req.TorrentData) > maxTorrentBase64Len { + common.ErrorResp(c, fmt.Errorf("torrent 数据过大（最大 10MB）"), 400) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + // 单文件模式 + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + common.SuccessResp(c, resp) +} + +// TorrentRapidUploadReq 从 torrent 秒传请求 +type TorrentRapidUploadReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` + // Path 目标路径 + Path string `json:"path" binding:"required"` +} + +// TorrentRapidUpload 从 torrent 文件中提取 CAS 信息尝试秒传到天翼云 +func TorrentRapidUpload(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req TorrentRapidUploadReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + + if !t.HasCASInfo() { + common.ErrorResp(c, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传"), 400) + return + } + + // 获取目标存储 + storage, dstDirActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // 获取目标目录对象 + dstDir, err := op.Get(c.Request.Context(), storage, dstDirActualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取目标目录失败: %w", err), 500) + return + } + if !dstDir.IsDir() { + common.ErrorResp(c, errs.NotFolder, 400) + return + } + + // 检查是否是天翼云 PC 驱动 + cloud189PC, ok := storage.(*_189pc.Cloud189PC) + if !ok { + common.ErrorResp(c, fmt.Errorf("目标存储不是天翼云PC驱动，不支持 CAS 秒传"), 400) + return + } + + // 尝试秒传 + obj, err := cloud189PC.RapidUploadFromTorrent(c.Request.Context(), dstDir, torrentData, true) + if err != nil { + common.ErrorResp(c, fmt.Errorf("秒传失败: %w", err), 400) + return + } + + common.SuccessResp(c, gin.H{ + "message": "秒传成功", + "file_name": obj.GetName(), + "file_size": obj.GetSize(), + }) +} + +// UploadTorrentAndParse 通过文件上传方式解析 torrent +func UploadTorrentAndParse(c *gin.Context) { + file, err := c.FormFile("torrent") + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取上传文件失败: %w", err), 400) + return + } + + // 限制文件大小（最大 10MB） + if file.Size > 10*1024*1024 { + common.ErrorResp(c, fmt.Errorf("torrent 文件过大（最大 10MB）"), 400) + return + } + + f, err := file.Open() + if err != nil { + common.ErrorResp(c, fmt.Errorf("打开文件失败: %w", err), 500) + return + } + defer f.Close() + + torrentData, err := io.ReadAll(f) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + // 同时返回 Base64 编码的 torrent 数据，方便后续使用 + common.SuccessResp(c, gin.H{ + "info": resp, + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + }) +} + +// GenerateTorrentReq 为指定路径的文件生成 torrent 请求 +type GenerateTorrentReq struct { + // Path 文件在 OpenList 中的路径 + Path string `json:"path" binding:"required"` + // WithCAS 是否注入 CAS 扩展信息（仅天翼云需要） + WithCAS bool `json:"with_cas"` +} + +// GenerateTorrentForPath 为指定路径的文件生成 torrent +// 这是一个通用接口，适用于所有驱动 +// 会获取文件内容计算哈希，然后生成 torrent +func GenerateTorrentForPath(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req GenerateTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查读取权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanRead(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // 获取存储和文件信息 + storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // with_cas 仅支持天翼云PC驱动 + if req.WithCAS { + if _, is189pc := storage.(*_189pc.Cloud189PC); !is189pc { + common.ErrorResp(c, fmt.Errorf("CAS 秒传扩展仅支持天翼云PC驱动"), 400) + return + } + } + + // 获取文件对象 + obj, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件失败: %w", err), 500) + return + } + if obj.IsDir() { + common.ErrorResp(c, fmt.Errorf("不支持为目录生成 torrent"), 400) + return + } + + // 限制可生成 torrent 的文件大小 + if obj.GetSize() > maxTorrentGenFileSize { + common.ErrorResp(c, fmt.Errorf("文件过大，无法生成 torrent（最大 1GB）"), 400) + return + } + + // 获取文件下载链接 + link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件链接失败: %w", err), 500) + return + } + defer link.Close() + + // 通过 RangeReader 获取文件内容并计算哈希生成 torrent + if link.RangeReader == nil { + common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取，无法生成 torrent（请先下载文件到本地）"), 400) + return + } + + // 读取整个文件 + rc, err := link.RangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + defer rc.Close() + + var torrentData []byte + if req.WithCAS { + torrentData, err = torrent.GenerateFromReaderWithCAS(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } else { + torrentData, err = torrent.GenerateFromReader(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } + if err != nil { + common.ErrorResp(c, fmt.Errorf("生成 torrent 失败: %w", err), 500) + return + } + + // 解析生成的 torrent 获取 info_hash + t, _ := torrent.Decode(torrentData) + var infoHash string + if t != nil { + infoHash = t.GetInfoHashHex() + } + + common.SuccessResp(c, gin.H{ + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + "info_hash": infoHash, + "file_name": obj.GetName() + ".torrent", + "size": len(torrentData), + "with_cas": req.WithCAS, + }) +} diff --git a/server/handles/user.go b/server/handles/user.go index 095089d707..4ebbde1686 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,8 +1,10 @@ package handles import ( + "errors" "strconv" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" @@ -10,6 +12,29 @@ import ( log "github.com/sirupsen/logrus" ) +// applyUserUpdatePolicy enforces who may change what during a user update. +// Role can never change here. A non-admin editor (a user delegated the +// "manage user info" permission) may only edit a normal user's username and +// password; every privileged field (permission, role, base path, disabled, +// otp) is forced back to the existing value, and admins can't be edited by them. +func applyUserUpdatePolicy(existing, req *model.User, byAdmin bool) error { + if existing.Role != req.Role { + return errors.New("role can not be changed") + } + if byAdmin { + return nil + } + if existing.IsAdmin() { + return errors.New("you are not allowed to edit an admin user") + } + // Profile-only edit: keep all privileged fields, allow username/password. + username, password := req.Username, req.Password + *req = *existing + req.Username = username + req.Password = password + return nil +} + func ListUsers(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { @@ -60,8 +85,10 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) + editor, _ := c.Request.Context().Value(conf.UserKey).(*model.User) + byAdmin := editor != nil && editor.IsAdmin() + if err := applyUserUpdatePolicy(user, &req, byAdmin); err != nil { + common.ErrorStrResp(c, err.Error(), 403) return } if req.Password == "" { diff --git a/server/handles/user_test.go b/server/handles/user_test.go new file mode 100644 index 0000000000..0cce111e3d --- /dev/null +++ b/server/handles/user_test.go @@ -0,0 +1,80 @@ +package handles + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestApplyUserUpdatePolicyRejectsRoleChange(t *testing.T) { + existing := &model.User{ID: 2, Role: model.GENERAL, Username: "bob"} + req := &model.User{ID: 2, Role: model.ADMIN, Username: "bob"} + if err := applyUserUpdatePolicy(existing, req, true); err == nil { + t.Fatal("expected role change to be rejected even for admin") + } +} + +func TestApplyUserUpdatePolicyAdminCanEditPrivilegedFields(t *testing.T) { + existing := &model.User{ID: 2, Role: model.GENERAL, Username: "bob", Permission: 0, BasePath: "/old"} + req := &model.User{ID: 2, Role: model.GENERAL, Username: "bob2", Permission: 0xFF, BasePath: "/new", Disabled: true} + if err := applyUserUpdatePolicy(existing, req, true); err != nil { + t.Fatalf("admin edit should succeed: %v", err) + } + if req.Permission != 0xFF || req.BasePath != "/new" || !req.Disabled { + t.Fatalf("admin's privileged changes must be preserved, got %+v", req) + } +} + +func TestApplyUserUpdatePolicyDelegatedCannotEditAdmin(t *testing.T) { + existing := &model.User{ID: 1, Role: model.ADMIN, Username: "root"} + req := &model.User{ID: 1, Role: model.ADMIN, Username: "root2"} + if err := applyUserUpdatePolicy(existing, req, false); err == nil { + t.Fatal("a delegated (non-admin) editor must not edit an admin user") + } +} + +func TestApplyUserUpdatePolicyDelegatedProfileOnly(t *testing.T) { + existing := &model.User{ + ID: 2, + Role: model.GENERAL, + Username: "bob", + Permission: 0b101, + BasePath: "/restricted", + Disabled: false, + OtpSecret: "secret", + } + // A delegated editor tries to escalate permission, change base path, disable. + req := &model.User{ + ID: 2, + Role: model.GENERAL, + Username: "bobby", + Password: "newpass", + Permission: 0xFFFF, + BasePath: "/", + Disabled: true, + OtpSecret: "", + } + if err := applyUserUpdatePolicy(existing, req, false); err != nil { + t.Fatalf("delegated profile edit should succeed: %v", err) + } + // Username + password allowed through. + if req.Username != "bobby" { + t.Errorf("username should be editable, got %q", req.Username) + } + if req.Password != "newpass" { + t.Errorf("password should be editable, got %q", req.Password) + } + // Every privileged field must be forced back to the existing value. + if req.Permission != 0b101 { + t.Errorf("permission must not change, got %d", req.Permission) + } + if req.BasePath != "/restricted" { + t.Errorf("base path must not change, got %q", req.BasePath) + } + if req.Disabled { + t.Errorf("disabled must not change") + } + if req.OtpSecret != "secret" { + t.Errorf("otp secret must be preserved, got %q", req.OtpSecret) + } +} diff --git a/server/handles/video_proxy.go b/server/handles/video_proxy.go new file mode 100644 index 0000000000..731b4718c9 --- /dev/null +++ b/server/handles/video_proxy.go @@ -0,0 +1,180 @@ +package handles + +import ( + "io" + "net/http" + "net/url" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// videoProxyPath is the route (relative to conf.URL.Path, sibling to /p) that +// streams an arbitrary signed upstream media URL through OpenList. It exists so +// the browser never fetches a provider's transcoded CDN directly (which fails +// CORS, e.g. 115's online-play HLS); everything is fetched same-origin and the +// provider request happens server-side. +const videoProxyPath = "/video_proxy" + +// BuildVideoProxyURL wraps an upstream media URL in a signed OpenList proxy URL. +// The signature (over the raw upstream URL) prevents the endpoint from acting as +// an open relay: only URLs OpenList itself produced are accepted. +func BuildVideoProxyURL(apiURL, rawURL string) string { + return apiURL + videoProxyPath + + "?url=" + url.QueryEscape(rawURL) + + "&sign=" + url.QueryEscape(sign.Sign(rawURL)) +} + +// isM3U8 reports whether the upstream response is an HLS manifest, by +// content-type or by the upstream URL's path extension. +func isM3U8(contentType, rawURL string) bool { + ct := strings.ToLower(contentType) + if strings.Contains(ct, "mpegurl") { + return true + } + if u, err := url.Parse(rawURL); err == nil { + if strings.HasSuffix(strings.ToLower(u.Path), ".m3u8") { + return true + } + } + return false +} + +// rewriteM3U8 rewrites every segment / sub-playlist / key URI in an HLS manifest +// so they are also fetched through the OpenList video proxy. baseRawURL is the +// upstream URL the manifest was fetched from (used to resolve relative URIs); +// apiURL is the OpenList base used to build the proxy links. Without this, the +// segment URLs inside the manifest would still point at the provider CDN and +// re-introduce the CORS failure we are trying to avoid. +func rewriteM3U8(manifest, baseRawURL, apiURL string) string { + baseURL, _ := url.Parse(baseRawURL) + var b strings.Builder + lines := strings.Split(manifest, "\n") + for i, line := range lines { + if i > 0 { + b.WriteByte('\n') + } + trimmed := strings.TrimSpace(line) + if trimmed == "" { + b.WriteString(line) + continue + } + if strings.HasPrefix(trimmed, "#") { + // Tag line: rewrite any URI="..." attribute (EXT-X-KEY, EXT-X-MAP, + // EXT-X-MEDIA, EXT-X-I-FRAME-STREAM-INF, EXT-X-SESSION-KEY, ...). + b.WriteString(rewriteTagURIs(line, baseURL, apiURL)) + continue + } + // Resource line: a segment or sub-playlist URI on its own line. + b.WriteString(proxyResolve(trimmed, baseURL, apiURL)) + } + return b.String() +} + +// rewriteTagURIs rewrites every URI="..." occurrence on an HLS tag line. +func rewriteTagURIs(line string, baseURL *url.URL, apiURL string) string { + const marker = `URI="` + var b strings.Builder + rest := line + for { + idx := strings.Index(rest, marker) + if idx < 0 { + b.WriteString(rest) + break + } + b.WriteString(rest[:idx+len(marker)]) + rest = rest[idx+len(marker):] + end := strings.IndexByte(rest, '"') + if end < 0 { + b.WriteString(rest) + break + } + b.WriteString(proxyResolve(rest[:end], baseURL, apiURL)) + b.WriteByte('"') + rest = rest[end+1:] + } + return b.String() +} + +// proxyResolve resolves ref against the manifest's base URL (no-op if ref is +// already absolute) and wraps the result in a signed proxy URL. +func proxyResolve(ref string, baseURL *url.URL, apiURL string) string { + abs := ref + if baseURL != nil { + if u, err := url.Parse(ref); err == nil { + abs = baseURL.ResolveReference(u).String() + } + } + return BuildVideoProxyURL(apiURL, abs) +} + +// hopByHopOrRangeHeaders are upstream response headers worth forwarding to the +// client for non-manifest passthrough (segments / progressive mp4). +var passthroughHeaders = []string{ + "Content-Type", + "Content-Length", + "Content-Range", + "Accept-Ranges", + "Last-Modified", + "ETag", +} + +// VideoProxy streams a signed upstream media URL through OpenList. For HLS +// manifests it rewrites the inner URIs to route back through this same endpoint; +// everything else is streamed verbatim (with Range support for seeking). +func VideoProxy(c *gin.Context) { + rawURL := c.Query("url") + signParam := c.Query("sign") + if rawURL == "" { + common.ErrorStrResp(c, "missing url", 400) + return + } + if err := sign.Verify(rawURL, signParam); err != nil { + common.ErrorResp(c, err, 401) + return + } + + req, err := http.NewRequestWithContext(c.Request.Context(), c.Request.Method, rawURL, nil) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // Forward Range so the player can byte-range seek within segments / mp4. + if rng := c.GetHeader("Range"); rng != "" { + req.Header.Set("Range", rng) + } + req.Header.Set("User-Agent", base.UserAgent) + + resp, err := base.HttpClient.Do(req) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + defer resp.Body.Close() + + if isM3U8(resp.Header.Get("Content-Type"), rawURL) { + body, err := io.ReadAll(resp.Body) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + out := rewriteM3U8(string(body), rawURL, common.GetApiUrl(c)) + c.Header("Content-Type", "application/vnd.apple.mpegurl") + c.Header("Cache-Control", "no-cache") + c.String(resp.StatusCode, out) + return + } + + for _, h := range passthroughHeaders { + if v := resp.Header.Get(h); v != "" { + c.Header(h, v) + } + } + c.Status(resp.StatusCode) + if c.Request.Method != http.MethodHead { + _, _ = io.Copy(c.Writer, resp.Body) + } +} diff --git a/server/handles/video_proxy_test.go b/server/handles/video_proxy_test.go new file mode 100644 index 0000000000..735356c701 --- /dev/null +++ b/server/handles/video_proxy_test.go @@ -0,0 +1,141 @@ +package handles + +import ( + "net/url" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/sign" +) + +const testAPIURL = "http://openlist.test" + +// seedSign installs a deterministic signing token in the setting cache so +// sign.Sign / sign.Verify round-trip without touching the database. +func seedSign(t *testing.T) { + t.Helper() + op.Cache.SetSetting(conf.LinkExpiration, &model.SettingItem{Key: conf.LinkExpiration, Value: "0"}) + op.Cache.SetSetting(conf.Token, &model.SettingItem{Key: conf.Token, Value: "test-token"}) + t.Cleanup(func() { op.Cache.ClearAll() }) +} + +// parseProxied pulls the upstream url + sign back out of a proxy link and +// asserts the signature is valid. Returns the decoded upstream URL. +func parseProxied(t *testing.T, proxied string) string { + t.Helper() + if !strings.HasPrefix(proxied, testAPIURL+videoProxyPath+"?") { + t.Fatalf("proxied url has unexpected prefix: %s", proxied) + } + u, err := url.Parse(proxied) + if err != nil { + t.Fatalf("parse proxied url: %v", err) + } + raw := u.Query().Get("url") + s := u.Query().Get("sign") + if raw == "" || s == "" { + t.Fatalf("proxied url missing url/sign params: %s", proxied) + } + if err := sign.Verify(raw, s); err != nil { + t.Fatalf("sign verify failed for %q: %v", raw, err) + } + return raw +} + +func TestBuildVideoProxyURL(t *testing.T) { + seedSign(t) + raw := "https://cdn.115.com/v/movie.m3u8?t=123&k=abc/def" + got := BuildVideoProxyURL(testAPIURL, raw) + + // The upstream URL must be query-escaped (no bare '?'/'&' leaking past the + // first '?'), and must round-trip + verify. + if decoded := parseProxied(t, got); decoded != raw { + t.Fatalf("round-trip mismatch:\n have %q\n want %q", decoded, raw) + } + if strings.Count(got, "?") != 1 { + t.Fatalf("upstream query not escaped, multiple '?': %s", got) + } +} + +func TestIsM3U8(t *testing.T) { + cases := []struct { + ct, rawURL string + want bool + }{ + {"application/vnd.apple.mpegurl", "https://x/y", true}, + {"application/x-mpegURL", "https://x/y", true}, + {"", "https://cdn.115.com/v/movie.m3u8", true}, + {"", "https://cdn.115.com/v/movie.m3u8?token=1", true}, + {"video/mp4", "https://cdn.115.com/v/movie.mp4", false}, + {"binary/octet-stream", "https://cdn.115.com/v/seg001.ts", false}, + } + for _, c := range cases { + if got := isM3U8(c.ct, c.rawURL); got != c.want { + t.Errorf("isM3U8(%q,%q) = %v, want %v", c.ct, c.rawURL, got, c.want) + } + } +} + +func TestRewriteM3U8(t *testing.T) { + seedSign(t) + base := "https://cdn.115.com/hls/720/index.m3u8?auth=xyz" + manifest := strings.Join([]string{ + "#EXTM3U", + "#EXT-X-VERSION:3", + "#EXT-X-KEY:METHOD=AES-128,URI=\"https://key.115.com/k?id=1\",IV=0x00", + "#EXT-X-TARGETDURATION:10", + "#EXTINF:9.009,", + "seg001.ts", + "#EXTINF:9.009,", + "https://cdn.115.com/hls/720/seg002.ts?auth=xyz", + "", + "#EXT-X-ENDLIST", + }, "\n") + + out := rewriteM3U8(manifest, base, testAPIURL) + lines := strings.Split(out, "\n") + + // Structural tags preserved verbatim. + for _, want := range []string{"#EXTM3U", "#EXT-X-VERSION:3", "#EXT-X-TARGETDURATION:10", "#EXTINF:9.009,", "#EXT-X-ENDLIST"} { + if !strings.Contains(out, want) { + t.Errorf("missing preserved tag %q in output", want) + } + } + // Blank line preserved (same line count as input). + if len(lines) != len(strings.Split(manifest, "\n")) { + t.Errorf("line count changed: got %d want %d", len(lines), len(strings.Split(manifest, "\n"))) + } + + // Relative segment resolves against the manifest base, then proxied. + relLine := lines[5] + if got := parseProxied(t, relLine); got != "https://cdn.115.com/hls/720/seg001.ts" { + t.Errorf("relative segment resolved to %q", got) + } + // Absolute segment kept as-is, proxied. + absLine := lines[7] + if got := parseProxied(t, absLine); got != "https://cdn.115.com/hls/720/seg002.ts?auth=xyz" { + t.Errorf("absolute segment resolved to %q", got) + } + + // EXT-X-KEY URI rewritten in place (tag prefix kept, URI proxied). + keyLine := lines[2] + if !strings.HasPrefix(keyLine, "#EXT-X-KEY:METHOD=AES-128,URI=\"") || !strings.HasSuffix(keyLine, ",IV=0x00") { + t.Fatalf("key line structure broken: %s", keyLine) + } + inner := keyLine[strings.Index(keyLine, "URI=\"")+len("URI=\"") : strings.LastIndex(keyLine, "\"")] + if got := parseProxied(t, inner); got != "https://key.115.com/k?id=1" { + t.Errorf("key URI proxied to %q", got) + } + + // No raw provider CDN URL should survive outside an escaped 'url=' param. + for i, l := range lines { + if strings.HasPrefix(l, "#") { + continue + } + if strings.Contains(l, "cdn.115.com") && !strings.Contains(l, "url=") { + t.Errorf("line %d leaks raw CDN url: %s", i, l) + } + } +} diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index b2a0fbfb47..6848020fd0 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -133,7 +133,10 @@ func BeginAuthnRegistration(c *gin.Context) { return } - options, sessionData, err := authnInstance.BeginRegistration(user) + options, sessionData, err := authnInstance.BeginRegistration( + user, + webauthn.WithResidentKeyRequirement(protocol.ResidentKeyRequirementRequired), + ) if err != nil { common.ErrorResp(c, err, 400) diff --git a/server/mcp.go b/server/mcp.go new file mode 100644 index 0000000000..c036ce389a --- /dev/null +++ b/server/mcp.go @@ -0,0 +1,18 @@ +package server + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/mcp" + "github.com/gin-gonic/gin" +) + +func MCP(g *gin.RouterGroup) { + if !conf.Conf.MCP.Enable { + g.Any("/mcp", func(c *gin.Context) { + common.ErrorStrResp(c, "MCP server is not enabled", 403) + }) + return + } + mcp.Register(g) +} diff --git a/server/mcp/call.go b/server/mcp/call.go new file mode 100644 index 0000000000..5a19b9b4b2 --- /dev/null +++ b/server/mcp/call.go @@ -0,0 +1,88 @@ +package mcp + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" +) + +type toolCallParams struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` +} + +type toolResultContent struct { + Type string `json:"type"` + Text string `json:"text"` +} + +func (s *Server) handleToolsCall(c *gin.Context, req request) (int, response) { + var params toolCallParams + if len(req.Params) == 0 { + return http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/call params"}, + } + } + if err := json.Unmarshal(req.Params, ¶ms); err != nil || params.Name == "" { + return http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/call params"}, + } + } + + var ( + result any + err *rpcError + ) + switch params.Name { + case "openlist.fs.list": + result, err = s.callFSList(c, params.Arguments) + case "openlist.fs.get": + result, err = s.callFSGet(c, params.Arguments) + case "openlist.fs.link": + result, err = s.callFSLink(c, params.Arguments) + default: + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32601, Message: "unknown tool"}, + } + } + + if err != nil { + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "content": []toolResultContent{ + {Type: "text", Text: err.Message}, + }, + "isError": true, + }, + } + } + + resultJSON, marshalErr := json.Marshal(result) + if marshalErr != nil { + return http.StatusInternalServerError, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32603, Message: "failed to encode tool result"}, + } + } + + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "content": []toolResultContent{ + {Type: "text", Text: string(resultJSON)}, + }, + "structuredContent": result, + }, + } +} diff --git a/server/mcp/call_test.go b/server/mcp/call_test.go new file mode 100644 index 0000000000..b922ffd52c --- /dev/null +++ b/server/mcp/call_test.go @@ -0,0 +1,183 @@ +package mcp + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +func TestToolsListRequiresInitializedSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32002 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestToolsListSuccess(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s2": {id: "s2", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":2, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s2") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + tools, ok := result["tools"].([]any) + if !ok || len(tools) != 3 { + t.Fatalf("unexpected tools payload: %#v", result["tools"]) + } + names := map[string]bool{} + for _, rawTool := range tools { + currentTool, ok := rawTool.(map[string]any) + if !ok { + t.Fatalf("unexpected tool payload: %#v", rawTool) + } + name, _ := currentTool["name"].(string) + names[name] = true + } + if !names["openlist.fs.list"] || !names["openlist.fs.get"] || !names["openlist.fs.link"] { + t.Fatalf("unexpected tool names: %#v", names) + } +} + +func TestToolsCallUnknownTool(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s3": {id: "s3", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":3, + "method":"tools/call", + "params":{"name":"openlist.fs.unknown","arguments":{"path":"/"}} + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s3") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32601 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestToolsCallInvalidParams(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s4": {id: "s4", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":4, + "method":"tools/call", + "params":{"name":"openlist.fs.list","arguments":"bad"} + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s4") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("expected tool error result, got protocol error: %+v", resp.Error) + } +} + +func decodeResponse(t *testing.T, w *httptest.ResponseRecorder) response { + t.Helper() + + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + return resp +} diff --git a/server/mcp/fs_get.go b/server/mcp/fs_get.go new file mode 100644 index 0000000000..550cc036fc --- /dev/null +++ b/server/mcp/fs_get.go @@ -0,0 +1,168 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/handles" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsGetArgs struct { + Path string `json:"path"` + Password string `json:"password"` +} + +func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSGetArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{ + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + rawURL, provider, err := buildFSGetRawURL(ctx, c, reqPath, obj, meta) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + parentPath := stdpath.Dir(reqPath) + var related []model.Obj + sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelatedObjs(sameLevelFiles, obj) + } + + parentMeta, _ := op.GetNearestMeta(parentPath) + thumb, _ := model.GetThumb(obj) + mountDetails, _ := model.GetStorageDetails(obj) + return handles.FsGetResp{ + ObjResp: handles.ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Thumb: thumb, + Type: utils.GetFileType(obj.GetName()), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + MountDetails: mountDetails, + }, + RawURL: rawURL, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Provider: provider, + Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)), + }, nil +} + +func parseFSGetArgs(raw json.RawMessage) (*fsGetArgs, *rpcError) { + args := &fsGetArgs{} + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.get arguments"} + } + + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.get arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + return args, nil +} + +func buildFSGetRawURL(ctx context.Context, c *gin.Context, reqPath string, obj model.Obj, meta *model.Meta) (string, string, error) { + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + provider, ok := model.GetProvider(obj) + if !ok && storageErr == nil { + provider = storage.Config().Name + } + if obj.IsDir() { + return "", provider, nil + } + if storageErr != nil { + return "", provider, storageErr + } + + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + rawURL := common.GenerateDownProxyURL(storage.GetStorage(), reqPath) + if rawURL != "" { + return rawURL, provider, nil + } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + return fmt.Sprintf("%s/p%s%s", common.GetApiUrl(ctx), utils.EncodePath(reqPath, true), query), provider, nil + } + + if url, ok := model.GetUrl(obj); ok { + return url, provider, nil + } + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Redirect: true, + }) + if err != nil { + return "", provider, err + } + defer link.Close() + return link.URL, provider, nil +} + +func filterRelatedObjs(objs []model.Obj, obj model.Obj) []model.Obj { + related := make([]model.Obj, 0) + nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + for _, current := range objs { + if current.GetName() == obj.GetName() { + continue + } + if strings.HasPrefix(current.GetName(), nameWithoutExt) { + related = append(related, current) + } + } + return related +} diff --git a/server/mcp/fs_link.go b/server/mcp/fs_link.go new file mode 100644 index 0000000000..43ddad8522 --- /dev/null +++ b/server/mcp/fs_link.go @@ -0,0 +1,200 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + stdpath "path" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsLinkArgs struct { + Path string `json:"path"` + Password string `json:"password"` + Type string `json:"type"` +} + +type fsLinkResp struct { + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Provider string `json:"provider"` + URL string `json:"url"` + URLType string `json:"url_type"` + DirectURL string `json:"direct_url,omitempty"` + ProxyURL string `json:"proxy_url,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + Header http.Header `json:"header,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + PartSize int `json:"part_size,omitempty"` +} + +func (s *Server) callFSLink(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSLinkArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{ + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if obj.IsDir() { + return nil, &rpcError{Code: -32003, Message: "path is a directory"} + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + linkInfo, err := buildFSLinkInfo(ctx, c, reqPath, args, obj, meta, storage) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + return linkInfo, nil +} + +func parseFSLinkArgs(raw json.RawMessage) (*fsLinkArgs, *rpcError) { + args := &fsLinkArgs{} + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.link arguments"} + } + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.link arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + return args, nil +} + +func buildFSLinkInfo(ctx context.Context, c *gin.Context, reqPath string, args *fsLinkArgs, obj model.Obj, meta *model.Meta, storage driver.Driver) (*fsLinkResp, error) { + provider, ok := model.GetProvider(obj) + if !ok { + provider = storage.Config().Name + } + + resp := &fsLinkResp{ + Path: reqPath, + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Provider: provider, + DownloadURL: signedFileURL(ctx, "/d", reqPath, meta, args.Type), + } + + if canProxyFile(storage, stdpath.Base(reqPath)) { + proxyURL := proxyFileURL(ctx, reqPath, meta, storage.GetStorage(), args.Type) + resp.ProxyURL = proxyURL + } + + if common.ShouldProxy(storage, stdpath.Base(reqPath)) { + resp.URL = resp.ProxyURL + resp.URLType = "proxy" + return resp, nil + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Type: args.Type, + Redirect: true, + }) + if err != nil { + return nil, err + } + defer link.Close() + + resp.DirectURL = link.URL + resp.URL = link.URL + resp.URLType = "direct" + resp.Header = link.Header + resp.ContentLength = link.ContentLength + resp.Concurrency = link.Concurrency + resp.PartSize = link.PartSize + return resp, nil +} + +func canProxyFile(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) { + return true + } + return false +} + +func proxyFileURL(ctx context.Context, reqPath string, meta *model.Meta, storage *model.Storage, linkType string) string { + if url := common.GenerateDownProxyURL(storage, reqPath); url != "" { + return url + } + return signedFileURL(ctx, "/p", reqPath, meta, linkType) +} + +func signedFileURL(ctx context.Context, prefix, reqPath string, meta *model.Meta, linkType string) string { + query := url.Values{} + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query.Set("sign", sign.Sign(reqPath)) + } + if linkType != "" { + query.Set("type", linkType) + } + rawQuery := "" + if encoded := query.Encode(); encoded != "" { + rawQuery = "?" + encoded + } + return fmt.Sprintf("%s%s%s%s", + common.GetApiUrl(ctx), + prefix, + utils.EncodePath(reqPath, true), + rawQuery, + ) +} diff --git a/server/mcp/fs_list.go b/server/mcp/fs_list.go new file mode 100644 index 0000000000..ac28cf7561 --- /dev/null +++ b/server/mcp/fs_list.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "encoding/json" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/handles" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsListArgs struct { + Path string `json:"path"` + Password string `json:"password"` + Refresh bool `json:"refresh"` + Page int `json:"page"` + PerPage int `json:"per_page"` +} + +func (s *Server) callFSList(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSListArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + write := common.CanWrite(user, meta, reqPath) + writeContentBypass := common.CanWriteContentBypassUserPerms(meta, reqPath) + canWriteContentAtPath := write && (user.CanWriteContent() || writeContentBypass) + if args.Refresh && !canWriteContentAtPath { + return nil, &rpcError{Code: -32003, Message: "refresh without permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{ + Refresh: args.Refresh, + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + total, paged := paginateObjs(objs, args.Page, args.PerPage) + return handles.FsListResp{ + Content: toObjResp(paged, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Write: write, + WriteContentBypass: writeContentBypass, + Provider: "unknown", + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + }, nil +} + +func parseFSListArgs(raw json.RawMessage) (*fsListArgs, *rpcError) { + args := &fsListArgs{ + Page: 1, + PerPage: model.MaxInt, + } + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.list arguments"} + } + + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.list arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + normalizeFSListArgs(args) + return args, nil +} + +func normalizeFSListArgs(args *fsListArgs) { + pageReq := model.PageReq{ + Page: args.Page, + PerPage: args.PerPage, + } + pageReq.Validate() + args.Page = pageReq.Page + args.PerPage = pageReq.PerPage +} + +func paginateObjs(objs []model.Obj, page, perPage int) (int, []model.Obj) { + total := len(objs) + if page < 1 { + page = 1 + } + if perPage < 1 { + perPage = model.MaxInt + } + offset := page - 1 + if offset > total/perPage { + return total, []model.Obj{} + } + start := offset * perPage + if start > total { + return total, []model.Obj{} + } + end := total + if perPage <= total-start { + end = start + perPage + } + if end > total { + end = total + } + return total, objs[start:end] +} + +func toObjResp(objs []model.Obj, parent string, encrypt bool) []handles.ObjResp { + resp := make([]handles.ObjResp, 0, len(objs)) + for _, obj := range objs { + thumb, _ := model.GetThumb(obj) + mountDetails, _ := model.GetStorageDetails(obj) + resp = append(resp, handles.ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + MountDetails: mountDetails, + }) + } + return resp +} + +func getReadme(meta *model.Meta, path string) string { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.RSub) { + return meta.Readme + } + return "" +} + +func getHeader(meta *model.Meta, path string) string { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.HeaderSub) { + return meta.Header + } + return "" +} + +func isEncrypt(meta *model.Meta, path string) bool { + if common.IsStorageSignEnabled(path) { + return true + } + if meta == nil || meta.Password == "" { + return false + } + return common.MetaCoversPath(meta.Path, path, meta.PSub) +} diff --git a/server/mcp/get_test.go b/server/mcp/get_test.go new file mode 100644 index 0000000000..81fbe39006 --- /dev/null +++ b/server/mcp/get_test.go @@ -0,0 +1,39 @@ +package mcp + +import ( + "encoding/json" + "testing" +) + +func TestParseFSGetArgsRequiresPath(t *testing.T) { + _, err := parseFSGetArgs(json.RawMessage(`{"password":"secret"}`)) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} + +func TestParseFSGetArgs(t *testing.T) { + args, err := parseFSGetArgs(json.RawMessage(`{"path":"/movie.mkv","password":"secret"}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/movie.mkv" { + t.Fatalf("unexpected path: %q", args.Path) + } + if args.Password != "secret" { + t.Fatalf("unexpected password: %q", args.Password) + } +} + +func TestParseFSGetArgsRejectsInvalidJSON(t *testing.T) { + _, err := parseFSGetArgs(json.RawMessage(`"bad"`)) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} diff --git a/server/mcp/handler.go b/server/mcp/handler.go new file mode 100644 index 0000000000..191977eb43 --- /dev/null +++ b/server/mcp/handler.go @@ -0,0 +1,536 @@ +package mcp + +import ( + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/middlewares" + "github.com/gin-gonic/gin" +) + +const ( + ProtocolVersion = "2025-11-25" + ProtocolVersionHeader = "MCP-Protocol-Version" + SessionHeader = "MCP-Session-Id" + sessionTTL = 30 * time.Minute + maxSessions = 128 + maxUserSessions = 16 +) + +type session struct { + id string + userID uint + protocolVersion string + initialized bool + createdAt time.Time + lastUsedAt time.Time +} + +type Server struct { + mu sync.Mutex + sessions map[string]*session +} + +type request struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id,omitempty"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type response struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type initializeParams struct { + ProtocolVersion string `json:"protocolVersion"` + Capabilities map[string]any `json:"capabilities"` + ClientInfo map[string]any `json:"clientInfo"` +} + +var defaultServer = &Server{ + sessions: map[string]*session{}, +} + +var supportedProtocolVersions = map[string]struct{}{ + "2025-11-25": {}, + "2025-06-18": {}, +} + +func Register(g *gin.RouterGroup) { + mcpGroup := g.Group("/mcp", middlewares.Auth(false), middlewares.AuthAdmin) + mcpGroup.GET("", defaultServer.handleGet) + mcpGroup.POST("", defaultServer.handlePost) + mcpGroup.DELETE("", defaultServer.handleDelete) +} + +func (s *Server) handleGet(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + c.Header("Allow", "POST, DELETE") + c.Status(http.StatusMethodNotAllowed) +} + +func (s *Server) handlePost(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + if !acceptsStreamableHTTP(c.GetHeader("Accept")) { + c.JSON(http.StatusNotAcceptable, response{ + JSONRPC: "2.0", + Error: &rpcError{ + Code: -32000, + Message: "Not Acceptable: client must accept both application/json and text/event-stream", + }, + }) + return + } + + body, err := io.ReadAll(io.LimitReader(c.Request.Body, 1<<20)) + if err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + Error: &rpcError{Code: -32700, Message: "failed to read request body"}, + }) + return + } + + var req request + if err := json.Unmarshal(body, &req); err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + Error: &rpcError{Code: -32700, Message: "parse error"}, + }) + return + } + if req.JSONRPC != "2.0" { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32600, Message: "invalid request"}, + }) + return + } + + if req.Method == "initialize" { + s.handleInitialize(c, req) + return + } + sessionID := c.GetHeader(SessionHeader) + if sessionID == "" { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32000, Message: "missing MCP session"}, + }) + return + } + + currentSession, ok := s.getSession(sessionID) + if !ok { + c.JSON(http.StatusNotFound, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32001, Message: "session not found"}, + }) + return + } + + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if currentSession.userID != user.ID { + c.JSON(http.StatusNotFound, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32001, Message: "session not found"}, + }) + return + } + + if !s.validateRequestProtocolVersion(c.GetHeader(ProtocolVersionHeader), currentSession.protocolVersion) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32000, Message: "missing or unsupported MCP protocol version"}, + }) + return + } + + switch req.Method { + case "ping": + c.JSON(http.StatusOK, response{JSONRPC: "2.0", ID: req.ID, Result: map[string]any{}}) + case "notifications/initialized": + s.markSessionInitialized(sessionID) + c.Status(http.StatusAccepted) + case "tools/list": + if !s.sessionInitialized(sessionID) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32002, Message: "MCP session not initialized"}, + }) + return + } + c.JSON(http.StatusOK, s.handleToolsList(req)) + case "tools/call": + if !s.sessionInitialized(sessionID) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32002, Message: "MCP session not initialized"}, + }) + return + } + status, resp := s.handleToolsCall(c, req) + c.JSON(status, resp) + default: + c.JSON(http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32601, Message: fmt.Sprintf("method %q not implemented yet", req.Method)}, + }) + } +} + +func (s *Server) handleInitialize(c *gin.Context, req request) { + var params initializeParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid initialize params"}, + }) + return + } + } + + protocolVersion := negotiateProtocolVersion(params.ProtocolVersion) + currentSession := s.initializeSession( + c.Request.Context().Value(conf.UserKey).(*model.User).ID, + c.GetHeader(SessionHeader), + protocolVersion, + ) + c.Header(SessionHeader, currentSession.id) + c.JSON(http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "protocolVersion": protocolVersion, + "capabilities": map[string]any{ + "tools": map[string]any{ + "listChanged": false, + }, + }, + "serverInfo": map[string]any{ + "name": "OpenList MCP", + "version": conf.Version, + }, + "instructions": "Complete initialization with notifications/initialized, then use tools/list and tools/call. Available tools include openlist.fs.list, openlist.fs.get, and openlist.fs.link.", + }, + }) +} + +func (s *Server) initializeSession(userID uint, requestedID string, protocolVersion string) *session { + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + s.pruneExpiredSessionsLocked(now) + if requestedID != "" { + currentSession, ok := s.sessions[requestedID] + if ok && currentSession != nil && currentSession.userID == userID { + currentSession.initialized = false + currentSession.protocolVersion = protocolVersion + currentSession.lastUsedAt = now + return currentSession + } + } + + return s.createSessionLocked(userID, protocolVersion, now) +} + +func (s *Server) handleDelete(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + + currentSession, ok := s.getSession(c.GetHeader(SessionHeader)) + if !ok { + c.Status(http.StatusNotFound) + return + } + + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if currentSession.userID != user.ID { + c.Status(http.StatusNotFound) + return + } + + s.deleteSession(currentSession.id) + c.Status(http.StatusNoContent) +} + +func (s *Server) createSession(userID uint) *session { + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + s.pruneExpiredSessionsLocked(now) + return s.createSessionLocked(userID, ProtocolVersion, now) +} + +func (s *Server) createSessionLocked(userID uint, protocolVersion string, now time.Time) *session { + s.pruneLeastRecentlyUsedUserSessionsLocked(userID, max(0, s.countUserSessionsLocked(userID)-maxUserSessions+1)) + s.pruneLeastRecentlyUsedSessionsLocked(max(0, len(s.sessions)-maxSessions+1)) + currentSession := &session{ + id: random.Token(), + userID: userID, + protocolVersion: protocolVersion, + createdAt: now, + lastUsedAt: now, + } + s.sessions[currentSession.id] = currentSession + return currentSession +} + +func (s *Server) validateRequestProtocolVersion(requestedVersion string, negotiatedVersion string) bool { + if requestedVersion == "" { + _, ok := supportedProtocolVersions[negotiatedVersion] + return ok + } + if _, ok := supportedProtocolVersions[requestedVersion]; !ok { + return false + } + return negotiatedVersion == "" || requestedVersion == negotiatedVersion +} + +func negotiateProtocolVersion(requestedVersion string) string { + if _, ok := supportedProtocolVersions[requestedVersion]; ok { + return requestedVersion + } + return ProtocolVersion +} + +func (s *Server) getSession(id string) (session, bool) { + if id == "" { + return session{}, false + } + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return session{}, false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return session{}, false + } + currentSession.lastUsedAt = now + return *currentSession, true +} + +func (s *Server) markSessionInitialized(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return false + } + currentSession.initialized = true + currentSession.lastUsedAt = now + return true +} + +func (s *Server) sessionInitialized(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return false + } + currentSession.lastUsedAt = now + return currentSession.initialized +} + +func (s *Server) deleteSession(id string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.sessions, id) +} + +func (s *Server) pruneExpiredSessionsLocked(now time.Time) { + for id, currentSession := range s.sessions { + if currentSession == nil || sessionExpired(currentSession, now) { + delete(s.sessions, id) + } + } +} + +func (s *Server) pruneLeastRecentlyUsedSessionsLocked(count int) { + for range count { + var ( + oldestID string + oldest time.Time + ) + for id, currentSession := range s.sessions { + if currentSession == nil { + oldestID = id + break + } + lastUsedAt := sessionLastUsedAt(currentSession) + if oldestID == "" || lastUsedAt.Before(oldest) { + oldestID = id + oldest = lastUsedAt + } + } + if oldestID == "" { + return + } + delete(s.sessions, oldestID) + } +} + +func (s *Server) countUserSessionsLocked(userID uint) int { + count := 0 + for _, currentSession := range s.sessions { + if currentSession != nil && currentSession.userID == userID { + count++ + } + } + return count +} + +func (s *Server) pruneLeastRecentlyUsedUserSessionsLocked(userID uint, count int) { + for range count { + var ( + oldestID string + oldest time.Time + ) + for id, currentSession := range s.sessions { + if currentSession == nil { + continue + } + if currentSession.userID != userID { + continue + } + lastUsedAt := sessionLastUsedAt(currentSession) + if oldestID == "" || lastUsedAt.Before(oldest) { + oldestID = id + oldest = lastUsedAt + } + } + if oldestID == "" { + return + } + delete(s.sessions, oldestID) + } +} + +func sessionExpired(currentSession *session, now time.Time) bool { + lastUsedAt := sessionLastUsedAt(currentSession) + return !lastUsedAt.IsZero() && now.Sub(lastUsedAt) > sessionTTL +} + +func sessionLastUsedAt(currentSession *session) time.Time { + if currentSession.lastUsedAt.IsZero() { + return currentSession.createdAt + } + return currentSession.lastUsedAt +} + +func acceptsStreamableHTTP(accept string) bool { + if accept == "" { + return false + } + hasJSON := false + hasSSE := false + for part := range strings.SplitSeq(accept, ",") { + mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(part)) + if err != nil { + continue + } + if q, ok := params["q"]; ok { + quality, err := strconv.ParseFloat(q, 64) + if err == nil && quality == 0 { + continue + } + } + switch mediaType { + case "*/*": + hasJSON = true + hasSSE = true + case "application/*", "application/json": + hasJSON = true + case "text/*", "text/event-stream": + hasSSE = true + } + } + return hasJSON || hasSSE +} + +func validateOrigin(r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin == "" { + return true + } + + originURL, err := url.Parse(origin) + if err != nil || originURL.Host == "" { + return false + } + if strings.EqualFold(originURL.Host, r.Host) { + return strings.EqualFold(originURL.Scheme, requestScheme(r)) + } + + siteURL := common.GetApiUrlFromRequest(r) + if siteURL == "" { + return false + } + siteParsed, err := url.Parse(siteURL) + if err != nil { + return false + } + return strings.EqualFold(originURL.Host, siteParsed.Host) && strings.EqualFold(originURL.Scheme, siteParsed.Scheme) +} + +func requestScheme(r *http.Request) string { + if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { + return "https" + } + return "http" +} diff --git a/server/mcp/handler_test.go b/server/mcp/handler_test.go new file mode 100644 index 0000000000..d3369576b2 --- /dev/null +++ b/server/mcp/handler_test.go @@ -0,0 +1,569 @@ +package mcp + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +func TestInitializeCreatesSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-11-25", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(SessionHeader); got == "" { + t.Fatal("expected session header to be set") + } + + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != ProtocolVersion { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestInitializeNegotiatesSupportedOlderProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-06-18", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != "2025-06-18" { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestInitializeReusesExistingSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + currentSession := srv.createSession(1) + currentSession.initialized = true + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-06-18", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, currentSession.id) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(SessionHeader); got != currentSession.id { + t.Fatalf("unexpected session header: got %q want %q", got, currentSession.id) + } + if len(srv.sessions) != 1 { + t.Fatalf("unexpected session count: got %d want %d", len(srv.sessions), 1) + } + reusedSession, ok := srv.getSession(currentSession.id) + if !ok { + t.Fatal("expected existing session to be reused") + } + if reusedSession.initialized { + t.Fatal("expected reused session to require initialized notification again") + } + if reusedSession.protocolVersion != "2025-06-18" { + t.Fatalf("unexpected protocol version: got %q want %q", reusedSession.protocolVersion, "2025-06-18") + } +} + +func TestInitializeNegotiatesUnsupportedProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2026-01-01", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != ProtocolVersion { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { + srv := newTestServer(nil) + var firstSessionID string + for i := range maxUserSessions { + currentSession := srv.createSession(1) + if i == 0 { + firstSessionID = currentSession.id + } + } + srv.createSession(2) + srv.createSession(1) + + if _, ok := srv.sessions[firstSessionID]; ok { + t.Fatal("expected oldest user session to be pruned") + } + if got := countSessionsForUser(srv, 1); got != maxUserSessions { + t.Fatalf("unexpected user session count: got %d want %d", got, maxUserSessions) + } + if got := len(srv.sessions); got != maxUserSessions+1 { + t.Fatalf("unexpected total session count: got %d want %d", got, maxUserSessions+1) + } +} + +func TestCreateSessionPrunesGlobalSessionLimit(t *testing.T) { + srv := newTestServer(nil) + var firstSessionID string + for i := range maxSessions { + currentSession := srv.createSession(uint(i + 1)) + if i == 0 { + firstSessionID = currentSession.id + } + } + srv.createSession(uint(maxSessions + 1)) + + if _, ok := srv.sessions[firstSessionID]; ok { + t.Fatal("expected oldest global session to be pruned") + } + if got := len(srv.sessions); got != maxSessions { + t.Fatalf("unexpected session count: got %d want %d", got, maxSessions) + } +} + +func TestPostAcceptsJSONOnly(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "application/json") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostAcceptsWildcardAccept(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "*/*") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostInvalidAcceptReturnsJSONRPCError(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "image/png") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotAcceptable { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNotAcceptable) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostUsesNegotiatedProtocolVersionWhenHeaderMissing(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, protocolVersion: "2025-06-18", initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsUnsupportedProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, "2025-03-26") + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsProtocolVersionMismatch(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, protocolVersion: "2025-06-18", initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsMissingSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsUnknownSessionWithNotFound(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "unknown") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNotFound) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32001 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestDeleteRemovesSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + currentSession := srv.createSession(1) + r := gin.New() + r.DELETE("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handleDelete(c) + }) + + req := httptest.NewRequest(http.MethodDelete, "http://example.com/mcp", nil) + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, currentSession.id) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNoContent { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNoContent) + } + if _, ok := srv.getSession(currentSession.id); ok { + t.Fatal("expected session to be deleted") + } +} + +func TestGetReturnsMethodNotAllowed(t *testing.T) { + gin.SetMode(gin.TestMode) + + r := gin.New() + r.GET("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + defaultServer.handleGet(c) + }) + + req := httptest.NewRequest(http.MethodGet, "http://example.com/mcp", nil) + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusMethodNotAllowed { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusMethodNotAllowed) + } + if allow := w.Header().Get("Allow"); allow != "POST, DELETE" { + t.Fatalf("unexpected Allow header: got %q want %q", allow, "POST, DELETE") + } +} + +func newTestServer(sessions map[string]*session) *Server { + if sessions == nil { + sessions = map[string]*session{} + } + now := time.Now() + for _, currentSession := range sessions { + if currentSession.createdAt.IsZero() { + currentSession.createdAt = now + } + if currentSession.lastUsedAt.IsZero() { + currentSession.lastUsedAt = now + } + } + return &Server{sessions: sessions} +} + +func countSessionsForUser(srv *Server, userID uint) int { + count := 0 + for _, currentSession := range srv.sessions { + if currentSession != nil && currentSession.userID == userID { + count++ + } + } + return count +} diff --git a/server/mcp/link_test.go b/server/mcp/link_test.go new file mode 100644 index 0000000000..10896d80ca --- /dev/null +++ b/server/mcp/link_test.go @@ -0,0 +1,143 @@ +package mcp + +import ( + "context" + "encoding/json" + "net/http/httptest" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/gin-gonic/gin" +) + +var settingCacheMu sync.Mutex + +func TestParseFSLinkArgs(t *testing.T) { + args, err := parseFSLinkArgs(json.RawMessage(`{"path":"/file.txt","password":"pw","type":"thumb"}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/file.txt" || args.Password != "pw" || args.Type != "thumb" { + t.Fatalf("unexpected args: %+v", args) + } +} + +func TestParseFSLinkArgsRequiresPath(t *testing.T) { + _, err := parseFSLinkArgs(json.RawMessage(`{"type":"thumb"}`)) + if err == nil || err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} + +func TestCanProxyFile(t *testing.T) { + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test"}, + storage: model.Storage{ + MountPath: "/", + }, + } + if canProxyFile(storage, "file.bin") { + t.Fatal("unexpected proxy support") + } + storage.config.OnlyProxy = true + if !canProxyFile(storage, "file.bin") { + t.Fatal("expected proxy support") + } +} + +func TestCanProxyFileIgnoresWebdavProxyURLPolicy(t *testing.T) { + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test"}, + storage: model.Storage{ + MountPath: "/", + Proxy: model.Proxy{ + WebdavPolicy: "use_proxy_url", + }, + }, + } + + if canProxyFile(storage, "file.bin") { + t.Fatal("webdav proxy url policy should not enable MCP proxy support") + } +} + +func TestBuildFSLinkInfoUsesProxyWhenStorageRequiresProxy(t *testing.T) { + gin.SetMode(gin.TestMode) + settingCacheMu.Lock() + t.Cleanup(func() { + op.Cache.ClearAll() + settingCacheMu.Unlock() + }) + op.Cache.SetSetting(conf.LinkExpiration, &model.SettingItem{Key: conf.LinkExpiration, Value: "0"}) + op.Cache.SetSetting(conf.Token, &model.SettingItem{Key: conf.Token, Value: "test-token"}) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest("POST", "http://example.com/mcp", nil) + ctx := context.WithValue(c.Request.Context(), conf.ApiUrlKey, "http://openlist.test") + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test", OnlyProxy: true}, + storage: model.Storage{ + MountPath: "/", + Proxy: model.Proxy{ + DownProxyURL: "http://proxy.test", + DisableProxySign: true, + }, + }, + } + obj := &model.ObjectURL{ + Object: model.Object{Name: "file.txt", Size: 12}, + Url: model.Url{Url: "http://direct.test/file.txt"}, + } + + meta := &model.Meta{Path: "/file.txt", Password: "secret"} + resp, err := buildFSLinkInfo(ctx, c, "/file.txt", &fsLinkArgs{Path: "/file.txt"}, obj, meta, storage) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if resp.URL != "http://proxy.test/file.txt" || resp.URLType != "proxy" { + t.Fatalf("unexpected selected link: %+v", resp) + } + if resp.DirectURL != "" { + t.Fatalf("direct link should not be resolved for proxy storage: %+v", resp) + } +} + +type fsLinkTestDriver struct { + config driver.Config + storage model.Storage +} + +func (d *fsLinkTestDriver) Config() driver.Config { + return d.config +} + +func (d *fsLinkTestDriver) GetStorage() *model.Storage { + return &d.storage +} + +func (d *fsLinkTestDriver) SetStorage(storage model.Storage) { + d.storage = storage +} + +func (d *fsLinkTestDriver) GetAddition() driver.Additional { + return nil +} + +func (d *fsLinkTestDriver) Init(context.Context) error { + return nil +} + +func (d *fsLinkTestDriver) Drop(context.Context) error { + return nil +} + +func (d *fsLinkTestDriver) List(context.Context, model.Obj, model.ListArgs) ([]model.Obj, error) { + return nil, nil +} + +func (d *fsLinkTestDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + return nil, nil +} diff --git a/server/mcp/list_test.go b/server/mcp/list_test.go new file mode 100644 index 0000000000..8079245aa5 --- /dev/null +++ b/server/mcp/list_test.go @@ -0,0 +1,54 @@ +package mcp + +import ( + "encoding/json" + "testing" +) + +func TestParseFSListArgsRequiresPath(t *testing.T) { + assertFSListPathRequired(t, json.RawMessage(`{"refresh":true}`)) +} + +func TestParseFSListArgsRejectsEmptyArguments(t *testing.T) { + for _, raw := range []json.RawMessage{nil, json.RawMessage(`null`)} { + assertFSListInvalidArguments(t, raw) + } +} + +func TestParseFSListArgsRejectsInvalidJSON(t *testing.T) { + assertFSListInvalidArguments(t, json.RawMessage(`"bad"`)) +} + +func TestParseFSListArgs(t *testing.T) { + args, err := parseFSListArgs(json.RawMessage(`{"path":"/movies","password":"secret","refresh":true,"page":2,"per_page":10}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/movies" || args.Password != "secret" || !args.Refresh || args.Page != 2 || args.PerPage != 10 { + t.Fatalf("unexpected args: %+v", args) + } +} + +func assertFSListPathRequired(t *testing.T, raw json.RawMessage) { + t.Helper() + + _, err := parseFSListArgs(raw) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 || err.Message != "path is required" { + t.Fatalf("unexpected error: %+v", err) + } +} + +func assertFSListInvalidArguments(t *testing.T, raw json.RawMessage) { + t.Helper() + + _, err := parseFSListArgs(raw) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 || err.Message != "invalid openlist.fs.list arguments" { + t.Fatalf("unexpected error: %+v", err) + } +} diff --git a/server/mcp/tools.go b/server/mcp/tools.go new file mode 100644 index 0000000000..188c674ce6 --- /dev/null +++ b/server/mcp/tools.go @@ -0,0 +1,122 @@ +package mcp + +import "encoding/json" + +type tool struct { + Name string `json:"name"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + InputSchema toolInputSchema `json:"inputSchema"` +} + +type toolInputSchema struct { + Type string `json:"type"` + Properties map[string]schemaProperty `json:"properties,omitempty"` + Required []string `json:"required,omitempty"` +} + +type schemaProperty struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` +} + +type toolsListParams struct { + Cursor string `json:"cursor,omitempty"` +} + +var openListTools = []tool{ + { + Name: "openlist.fs.list", + Title: "OpenList FS List", + Description: "List files and directories under a mount path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "Mount path to list, for example \"/\" or \"/movies\".", + }, + "refresh": { + Type: "boolean", + Description: "Refresh the directory listing before returning results.", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + "page": { + Type: "integer", + Description: "1-based page number.", + }, + "per_page": { + Type: "integer", + Description: "Page size.", + }, + }, + Required: []string{"path"}, + }, + }, + { + Name: "openlist.fs.get", + Title: "OpenList FS Get", + Description: "Get file or directory details for a mount path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "Mount path to inspect, for example \"/movies/demo.mp4\".", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + }, + Required: []string{"path"}, + }, + }, + { + Name: "openlist.fs.link", + Title: "OpenList FS Link", + Description: "Return usable link information for a file path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "File mount path, for example \"/movies/demo.mp4\".", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + "type": { + Type: "string", + Description: "Optional link type forwarded to storage drivers.", + }, + }, + Required: []string{"path"}, + }, + }, +} + +func (s *Server) handleToolsList(req request) response { + var params toolsListParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/list params"}, + } + } + } + + return response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "tools": openListTools, + }, + } +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 0fc243617f..965994d5f8 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -24,7 +24,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -41,7 +41,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -69,7 +69,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } @@ -84,7 +84,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -96,7 +96,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -124,7 +124,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } @@ -148,3 +148,15 @@ func AuthAdmin(c *gin.Context) { c.Next() } } + +// AuthUserInfoManage allows admins, or non-admins to whom an admin has delegated +// the "manage user info" permission, to access user-profile management endpoints. +func AuthUserInfoManage(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if !user.CanManageUserInfo() { + common.ErrorStrResp(c, "You are not allowed to manage users", 403) + c.Abort() + } else { + c.Next() + } +} diff --git a/server/middlewares/check.go b/server/middlewares/check.go index c7203a4907..c5e07874cd 100644 --- a/server/middlewares/check.go +++ b/server/middlewares/check.go @@ -29,7 +29,7 @@ func StoragesLoaded(c *gin.Context) { return } } - common.GinWithValue(c, + common.GinAppendValues(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), ) c.Next() diff --git a/server/middlewares/down.go b/server/middlewares/down.go index c1f81b54b3..d71be00b36 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -17,7 +17,7 @@ import ( func PathParse(c *gin.Context) { rawPath := parsePath(c.Param("path")) - common.GinWithValue(c, conf.PathKey, rawPath) + common.GinAppendValues(c, conf.PathKey, rawPath) c.Next() } @@ -29,7 +29,7 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { common.ErrorPage(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) // verify sign if needSign(meta, rawPath) { s := c.Query("sign") diff --git a/server/middlewares/sharing.go b/server/middlewares/sharing.go index d7549202f3..aa0cab0c63 100644 --- a/server/middlewares/sharing.go +++ b/server/middlewares/sharing.go @@ -8,11 +8,11 @@ import ( func SharingIdParse(c *gin.Context) { sid := c.Param("sid") - common.GinWithValue(c, conf.SharingIDKey, sid) + common.GinAppendValues(c, conf.SharingIDKey, sid) c.Next() } func EmptyPathParse(c *gin.Context) { - common.GinWithValue(c, conf.PathKey, "/") + common.GinAppendValues(c, conf.PathKey, "/") c.Next() } diff --git a/server/router.go b/server/router.go index 57d1166ae7..157410c508 100644 --- a/server/router.go +++ b/server/router.go @@ -41,6 +41,7 @@ func Init(e *gin.Engine) { } WebDav(g.Group("/dav")) S3(g.Group("/s3")) + MCP(g) downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) signCheck := middlewares.Down(sign.Verify) @@ -48,6 +49,11 @@ func Init(e *gin.Engine) { g.GET("/p/*path", middlewares.PathParse, signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", middlewares.PathParse, signCheck, handles.Down) g.HEAD("/p/*path", middlewares.PathParse, signCheck, handles.Proxy) + // /video_proxy streams a signed upstream media URL (query-param sign verified + // in-handler) so the browser never fetches a provider's transcoded CDN + // directly, e.g. 115 online-play HLS (avoids CORS). Sibling of /p. + g.GET("/video_proxy", downloadLimiter, handles.VideoProxy) + g.HEAD("/video_proxy", handles.VideoProxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveProxy) @@ -77,6 +83,9 @@ func Init(e *gin.Engine) { auth.GET("/me/sshkey/list", handles.ListMyPublicKey) auth.POST("/me/sshkey/add", handles.AddMyPublicKey) auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) + auth.GET("/me/favorites", handles.ListFavorites) + auth.POST("/me/favorites/add", handles.AddFavorite) + auth.POST("/me/favorites/delete", handles.DeleteFavorite) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) @@ -101,11 +110,23 @@ func Init(e *gin.Engine) { public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + // Plugin system: frontend manifest + hot-loadable JS assets (public so the + // web UI can fetch them before auth). + api.GET("/plugin/manifest", handles.PluginManifest) + api.GET("/plugin/asset/:name", handles.PluginAsset) + + // Cluster storage-sync peer endpoint: a persistent WebSocket so NAT'd nodes + // can dial out and stay connected. Authentication/encryption come from the + // cluster pre-shared key (per-frame AEAD), not the HTTP layer. + api.GET("/cluster/ws", handles.ClusterWS) + _fs(auth.Group("/fs")) fsAndShare(api.Group("/fs", middlewares.Auth(true))) _task(auth.Group("/task", middlewares.AuthNotGuest)) _sharing(auth.Group("/share", middlewares.AuthNotGuest)) admin(auth.Group("/admin", middlewares.AuthAdmin)) + // User-profile management: admins, or non-admins delegated the permission. + userManage(auth.Group("/admin/user", middlewares.AuthUserInfoManage)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) } @@ -114,6 +135,15 @@ func Init(e *gin.Engine) { }) } +// userManage registers the user-profile endpoints that an admin may delegate to +// a non-admin via the "manage user info" permission. The UpdateUser handler +// itself restricts a non-admin editor to username/password only. +func userManage(g *gin.RouterGroup) { + g.GET("/list", handles.ListUsers) + g.GET("/get", handles.GetUser) + g.POST("/update", handles.UpdateUser) +} + func admin(g *gin.RouterGroup) { meta := g.Group("/meta") meta.GET("/list", handles.ListMetas) @@ -122,11 +152,27 @@ func admin(g *gin.RouterGroup) { meta.POST("/update", handles.UpdateMeta) meta.POST("/delete", handles.DeleteMeta) + // Backend (Go-source, yaegi) plugin management. Admin only. + plug := g.Group("/plugin") + plug.GET("/list", handles.PluginList) + plug.GET("/get", handles.PluginGet) + plug.POST("/save", handles.PluginSave) + plug.POST("/delete", handles.PluginDelete) + plug.POST("/enable", handles.PluginSetEnabled) + + // Cluster storage-sharing config/status. Admin only. + clusterGrp := g.Group("/cluster") + clusterGrp.GET("/config", handles.ClusterGetConfig) + clusterGrp.GET("/key", handles.ClusterRevealKey) + clusterGrp.POST("/config", handles.ClusterSetConfig) + clusterGrp.GET("/status", handles.ClusterStatus) + clusterGrp.POST("/groups", handles.ClusterSetGroups) + + // Admin-only user operations (create/delete/role-affecting). The profile-edit + // endpoints (list/get/update) live in a separate group that also accepts a + // non-admin who has been delegated the "manage user info" permission. user := g.Group("/user") - user.GET("/list", handles.ListUsers) - user.GET("/get", handles.GetUser) user.POST("/create", handles.CreateUser) - user.POST("/update", handles.UpdateUser) user.POST("/cancel_2fa", handles.Cancel2FAById) user.POST("/delete", handles.DeleteUser) user.POST("/del_cache", handles.DelUserCache) @@ -135,6 +181,7 @@ func admin(g *gin.RouterGroup) { storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) + storage.GET("/loading", handles.StorageLoadingStatus) storage.GET("/get", handles.GetStorage) storage.POST("/create", handles.CreateStorage) storage.POST("/update", handles.UpdateStorage) @@ -190,6 +237,8 @@ func admin(g *gin.RouterGroup) { func fsAndShare(g *gin.RouterGroup) { g.Any("/list", handles.FsListSplit) g.Any("/get", handles.FsGetSplit) + g.POST("/video_play", handles.FsVideoPlay) + g.POST("/video_subtitle", handles.FsVideoSubtitle) a := g.Group("/archive") a.Any("/meta", handles.FsArchiveMetaSplit) a.Any("/list", handles.FsArchiveListSplit) @@ -217,6 +266,11 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Torrent 相关接口 + g.POST("/torrent/parse", handles.ParseTorrent) + g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse) + g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) + g.POST("/torrent/generate", handles.GenerateTorrentForPath) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } @@ -242,6 +296,10 @@ func Cors(r *gin.Engine) { config.AllowHeaders = conf.Conf.Cors.AllowHeaders config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) + r.Use(func(c *gin.Context) { + c.Header("Cross-Origin-Opener-Policy", "same-origin") + c.Header("Cross-Origin-Embedder-Policy", "credentialless") + }) } func InitS3(e *gin.Engine) { diff --git a/server/s3/redirect.go b/server/s3/redirect.go new file mode 100644 index 0000000000..4031e171d9 --- /dev/null +++ b/server/s3/redirect.go @@ -0,0 +1,167 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for openlist +package s3 + +import ( + "context" + "net/http" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/itsHenry35/gofakes3/signature" +) + +func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if u, ok := directObjectURL(r, authPairs); ok { + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + w.Header().Set("Location", u) + w.WriteHeader(http.StatusFound) + return + } + if u, ok := directUploadURL(r, authPairs); ok { + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("Location", u) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusTemporaryRedirect) + return + } + next.ServeHTTP(w, r) + }) +} + +func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool) { + if r.Method != http.MethodGet { + return "", false + } + if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) { + return "", false + } + bucketName, objectName, ok := parseObjectPath(r.URL.Path) + if !ok { + return "", false + } + bucket, err := getBucketByName(bucketName) + if err != nil { + return "", false + } + reqPath := path.Join(bucket.Path, objectName) + meta, _ := op.GetNearestMeta(reqPath) + ctx := context.WithValue(r.Context(), conf.MetaKey, meta) + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil || common.ShouldProxy(storage, path.Base(reqPath)) { + return "", false + } + link, file, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: utils.ClientIP(r), + Header: r.Header, + Redirect: true, + }) + if err != nil { + return "", false + } + defer link.Close() + if file == nil || file.IsDir() || link.URL == "" || link.RangeReader != nil { + return "", false + } + return link.URL, true +} + +func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool) { + if r.Method != http.MethodPut || r.ContentLength < 0 { + return "", false + } + if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) { + return "", false + } + if r.Header.Get("X-Amz-Copy-Source") != "" || + r.Header.Get("X-Amz-Content-Sha256") == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" { + return "", false + } + bucketName, objectName, ok := parseObjectPath(r.URL.Path) + if !ok || strings.HasSuffix(objectName, "/") { + return "", false + } + bucket, err := getBucketByName(bucketName) + if err != nil { + return "", false + } + reqPath := path.Join(bucket.Path, objectName) + storage, dstDirActualPath, err := op.GetStorageAndActualPath(path.Dir(reqPath)) + if err != nil || storage.Config().NoUpload { + return "", false + } + info, err := op.GetDirectUploadInfo(r.Context(), "HttpDirect", storage, dstDirActualPath, path.Base(reqPath), r.ContentLength, true) + if err != nil { + return "", false + } + httpInfo, ok := asHTTPDirectUploadInfo(info) + if !ok { + return "", false + } + method := httpInfo.Method + if method == "" { + method = http.MethodPut + } + if !strings.EqualFold(method, http.MethodPut) || httpInfo.UploadURL == "" || + httpInfo.ChunkSize > 0 || len(httpInfo.Headers) > 0 { + return "", false + } + return httpInfo.UploadURL, true +} + +func asHTTPDirectUploadInfo(info any) (*model.HttpDirectUploadInfo, bool) { + switch v := info.(type) { + case *model.HttpDirectUploadInfo: + return v, v != nil + case model.HttpDirectUploadInfo: + return &v, true + default: + return nil, false + } +} + +func parseObjectPath(rawPath string) (bucket, object string, ok bool) { + parts := strings.SplitN(strings.TrimPrefix(rawPath, "/"), "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + return parts[0], parts[1], true +} + +// hasNonObjectQuery reports whether the request carries a query parameter that +// makes gofakes3 route it to a sub-resource handler instead of plain object +// download (see gofakes3 routing.go). Only those keys force server-side +// handling; every other query parameter (response-content-disposition, +// response-content-type, etc.) is irrelevant to route selection and must not +// block a direct redirect. +func hasNonObjectQuery(r *http.Request) bool { + query := r.URL.Query() + for _, key := range []string{"uploadId", "uploads", "versioning", "versions", "location"} { + if _, ok := query[key]; ok { + return true + } + } + if versionID := query.Get("versionId"); versionID != "" && versionID != "null" { + return true + } + return false +} + +func s3RequestAuthorized(r *http.Request, authPairs map[string]string) bool { + if len(authPairs) == 0 { + return true + } + result := signature.V4SignVerify(r) + if result == signature.ErrUnsupportAlgorithm { + result = signature.V2SignVerify(r) + } + return result == signature.ErrNone +} diff --git a/server/s3/redirect_test.go b/server/s3/redirect_test.go new file mode 100644 index 0000000000..9584c136a0 --- /dev/null +++ b/server/s3/redirect_test.go @@ -0,0 +1,87 @@ +package s3 + +import ( + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestParseObjectPath(t *testing.T) { + tests := []struct { + name string + path string + wantBucket string + wantObject string + wantOK bool + }{ + {name: "object", path: "/bucket/path/to/file.txt", wantBucket: "bucket", wantObject: "path/to/file.txt", wantOK: true}, + {name: "missing object", path: "/bucket", wantOK: false}, + {name: "empty bucket", path: "//file.txt", wantOK: false}, + {name: "empty object", path: "/bucket/", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bucket, object, ok := parseObjectPath(tt.path) + if ok != tt.wantOK || bucket != tt.wantBucket || object != tt.wantObject { + t.Fatalf("parseObjectPath(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.path, bucket, object, ok, tt.wantBucket, tt.wantObject, tt.wantOK) + } + }) + } +} + +func TestHasNonObjectQuery(t *testing.T) { + tests := []struct { + name string + raw string + want bool + }{ + {name: "no query", raw: "/bucket/object", want: false}, + {name: "aws auth query", raw: "/bucket/object?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc", want: false}, + {name: "legacy auth query", raw: "/bucket/object?AWSAccessKeyId=ak&Signature=sig&Expires=1", want: false}, + {name: "sdk operation marker", raw: "/bucket/object?x-id=GetObject", want: false}, + {name: "response content disposition", raw: "/bucket/object?response-content-disposition=attachment%3Bfilename%3Dx", want: false}, + {name: "response content type", raw: "/bucket/object?response-content-type=text%2Fplain", want: false}, + // list-type is not a routing key in gofakes3: a path with an object + // segment is dispatched to getObject regardless, so it must not block + // a direct redirect. + {name: "list query", raw: "/bucket/object?list-type=2", want: false}, + {name: "multipart uploads", raw: "/bucket/object?uploads", want: true}, + {name: "multipart upload id", raw: "/bucket/object?uploadId=123", want: true}, + {name: "versioning", raw: "/bucket/object?versioning", want: true}, + {name: "versions", raw: "/bucket/object?versions", want: true}, + {name: "bucket location", raw: "/bucket/object?location", want: true}, + {name: "version id", raw: "/bucket/object?versionId=abc", want: true}, + {name: "version id null", raw: "/bucket/object?versionId=null", want: false}, + {name: "version id empty", raw: "/bucket/object?versionId=", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest("GET", tt.raw, nil) + if got := hasNonObjectQuery(req); got != tt.want { + t.Fatalf("hasNonObjectQuery(%q) = %v, want %v", tt.raw, got, tt.want) + } + }) + } +} + +func TestAsHTTPDirectUploadInfo(t *testing.T) { + info := model.HttpDirectUploadInfo{UploadURL: "https://example.com/upload"} + got, ok := asHTTPDirectUploadInfo(info) + if !ok || got == nil || got.UploadURL != info.UploadURL { + t.Fatalf("asHTTPDirectUploadInfo(value) = (%v, %v), want upload info", got, ok) + } + + got, ok = asHTTPDirectUploadInfo(&info) + if !ok || got == nil || got.UploadURL != info.UploadURL { + t.Fatalf("asHTTPDirectUploadInfo(pointer) = (%v, %v), want upload info", got, ok) + } + + got, ok = asHTTPDirectUploadInfo(struct{}{}) + if ok || got != nil { + t.Fatalf("asHTTPDirectUploadInfo(unsupported) = (%v, %v), want nil false", got, ok) + } +} diff --git a/server/s3/server.go b/server/s3/server.go index 360eb8752e..d33b63ddde 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -13,15 +13,16 @@ import ( // Make a new S3 Server to serve the remote func NewServer(ctx context.Context) (h http.Handler, err error) { var newLogger logger + authPairs := authlistResolver() faker := gofakes3.New( newBackend(), // gofakes3.WithHostBucket(!opt.pathBucketMode), gofakes3.WithLogger(newLogger), gofakes3.WithRequestID(rand.Uint64()), gofakes3.WithoutVersioning(), - gofakes3.WithV4Auth(authlistResolver()), + gofakes3.WithV4Auth(authPairs), gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) - return faker.Server(), nil + return redirectHandler(faker.Server(), authPairs), nil } diff --git a/server/static/static.go b/server/static/static.go index 29f97ff746..81695c9fb7 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -8,10 +8,13 @@ import ( "io/fs" "net/http" "os" + "path/filepath" "strings" + "sync" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/public" @@ -32,21 +35,41 @@ type Manifest struct { Icons []ManifestIcon `json:"icons"` } -var static fs.FS +// reloadableFS wraps fs.FS with thread-safe swapping. +// This allows gin StaticFS routes (which capture the fs.FS at registration time) +// to serve updated files after a watcher-triggered reload. +type reloadableFS struct { + mu sync.RWMutex + current fs.FS +} + +func (r *reloadableFS) Open(name string) (fs.File, error) { + r.mu.RLock() + current := r.current + r.mu.RUnlock() + return current.Open(name) +} + +func (r *reloadableFS) swap(f fs.FS) { + r.mu.Lock() + r.current = f + r.mu.Unlock() +} + +var staticFS = &reloadableFS{} func initStatic() { - utils.Log.Debug("Initializing static file system...") - if conf.Conf.DistDir == "" { - dist, err := fs.Sub(public.Public, "dist") - if err != nil { - utils.Log.Fatalf("failed to read dist dir: %v", err) - } - static = dist - utils.Log.Debug("Using embedded dist directory") + if conf.Conf.DistDir != "" { + staticFS.swap(os.DirFS(conf.Conf.DistDir)) + utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) return } - static = os.DirFS(conf.Conf.DistDir) - utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + dist, err := fs.Sub(public.Public, "dist") + if err != nil { + utils.Log.Fatalf("failed to read embedded dist dir: %v", err) + } + staticFS.swap(dist) + utils.Log.Infof("Using embedded dist directory") } func replaceStrings(content string, replacements map[string]string) string { @@ -74,7 +97,7 @@ func initIndex(siteConfig SiteConfig) { utils.Log.Info("Successfully fetched index.html from CDN") } else { utils.Log.Debug("Reading index.html from static files system...") - indexFile, err := static.Open("index.html") + indexFile, err := staticFS.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") @@ -131,13 +154,23 @@ func UpdateIndex() { utils.Log.Debug("Index.html update completed") } +// ReloadStatic reloads the static files from disk (called by the watcher after an update) +func ReloadStatic() { + utils.Log.Info("[static] reloading static files after frontend update...") + distPath := filepath.Join(frontend.GetDistPath(), "dist") + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Switched to dynamically fetched dist: %s", distPath) + siteConfig := getSiteConfig() + initIndex(siteConfig) +} + func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() - + // Get site title from settings siteTitle := setting.GetStr(conf.SiteTitle) - + // Get logo from settings, use the first line (light theme logo) logoSetting := setting.GetStr(conf.Logo) logoUrl := strings.Split(logoSetting, "\n")[0] @@ -167,7 +200,7 @@ func ManifestJSON(c *gin.Context) { c.Header("Content-Type", "application/json") c.Header("Cache-Control", "public, max-age=3600") // cache for 1 hour - + if err := json.NewEncoder(c.Writer).Encode(manifest); err != nil { utils.Log.Errorf("Failed to encode manifest.json: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to generate manifest"}) @@ -180,8 +213,12 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { siteConfig := getSiteConfig() initStatic() initIndex(siteConfig) + + // Start the frontend watcher for periodic updates + frontend.StartWatcher(ReloadStatic) + folders := []string{"assets", "images", "streamer", "static"} - + if conf.Conf.Cdn == "" { utils.Log.Debug("Setting up static file serving...") r.Use(func(c *gin.Context) { @@ -192,7 +229,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for _, folder := range folders { - sub, err := fs.Sub(static, folder) + sub, err := fs.Sub(staticFS, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } diff --git a/server/webdav.go b/server/webdav.go index a949068f01..74523b0b30 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -54,7 +54,7 @@ func WebDAVAuth(c *gin.Context) { count, cok := model.LoginCache.Get(ip) if cok && count >= model.DefaultMaxAuthRetries { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -78,13 +78,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) c.Next() return } } if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -96,7 +96,7 @@ func WebDAVAuth(c *gin.Context) { user, ok := tryLogin(username, password) if !ok { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -109,7 +109,7 @@ func WebDAVAuth(c *gin.Context) { model.LoginCache.Del(ip) if user.Disabled || !user.CanWebdavRead() { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -142,11 +142,11 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) if user.IsGuest() { - common.GinWithValue(c, conf.MetaPassKey, password) + common.GinAppendValues(c, conf.MetaPassKey, password) } else { - common.GinWithValue(c, conf.MetaPassKey, "") + common.GinAppendValues(c, conf.MetaPassKey, "") } c.Next() } diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 06d1431ac3..318ff78d38 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -271,6 +271,9 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } } + // Override the upstream User-Agent before resolving the link (see down.go); + // no-op when proxy_user_agent is unset. + common.ApplyProxyUserAgent(r, storage.GetStorage()) link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header}) if err != nil { return http.StatusInternalServerError, err