diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..fa4b0b91b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + # Runtime deps (requirements.txt is the single source; once a lockfile lands, keep raising version PRs here) + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.gitignore b/.gitignore index ebe2f8769..c18189018 100644 --- a/.gitignore +++ b/.gitignore @@ -80,8 +80,8 @@ docs/_build/ .pybuilder/ target/ *.db -./filecodebox.db-shm -./filecodebox.db-wal +*.db-shm +*.db-wal # Jupyter Notebook .ipynb_checkpoints @@ -147,13 +147,8 @@ cython_debug/ # Project .vscode .DS_Store -for_test.py .html -/evaluate/temp.py -/evaluation/back.json data/.env -.backup/ -/cloc-1.64.exe # Ignore node_modules node_modules/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..ca117ca6b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +# First-time setup: pre-commit install +# Behind a proxy: HTTPS_PROXY=http://: pre-commit run --all-files +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.6 + hooks: + - id: ruff-check + args: [--fix] diff --git a/apps/admin/dependencies.py b/apps/admin/dependencies.py index c4ca8541f..e17324a80 100644 --- a/apps/admin/dependencies.py +++ b/apps/admin/dependencies.py @@ -2,7 +2,7 @@ # @Author : Lan # @File : depends.py # @Software: PyCharm -from fastapi import Header, HTTPException, Depends +from fastapi import Header, HTTPException from fastapi.requests import Request import base64 import hmac diff --git a/apps/base/schemas.py b/apps/base/schemas.py index efe8025fd..f4cf4a405 100644 --- a/apps/base/schemas.py +++ b/apps/base/schemas.py @@ -1,5 +1,4 @@ from pydantic import BaseModel -from typing import Optional class SelectFileModel(BaseModel): diff --git a/apps/base/utils.py b/apps/base/utils.py index 2f1dba7df..968518fa7 100644 --- a/apps/base/utils.py +++ b/apps/base/utils.py @@ -1,5 +1,4 @@ import datetime -import hashlib import os import uuid from urllib.parse import unquote @@ -26,29 +25,32 @@ def validate_expire_style(expire_style: str) -> str: return expire_style -async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]: +async def build_file_path( + file_name: str, file_uuid: str +) -> Tuple[str, str, str, str, str]: + """Single source of storage path generation (date dir + UUID), shared by + regular, chunked, and presigned uploads. + + Always use get_now() (UTC+8); do not switch to server-local time. + """ today = await get_now() storage_path = settings.storage_path.strip("/") - file_uuid = uuid.uuid4().hex - filename = await sanitize_filename(unquote(file.filename or "")) + filename = await sanitize_filename(unquote(file_name or "")) base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{file_uuid}" path = f"{storage_path}/{base_path}" if storage_path else base_path prefix, suffix = os.path.splitext(filename) - save_path = f"{path}/{filename}" + save_path = f"{path}/{prefix}{suffix}" return path, suffix, prefix, filename, save_path +async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]: + return await build_file_path(file.filename or "", uuid.uuid4().hex) + + async def get_chunk_file_path_name( file_name: str, upload_id: str ) -> Tuple[str, str, str, str, str]: - today = await get_now() - storage_path = settings.storage_path.strip("/") - file_name = await sanitize_filename(unquote(file_name or "")) - base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{upload_id}" - path = f"{storage_path}/{base_path}" if storage_path else base_path - prefix, suffix = os.path.splitext(file_name) - save_path = f"{path}/{prefix}{suffix}" - return path, suffix, prefix, file_name, save_path + return await build_file_path(file_name, upload_id) async def get_expire_info( @@ -121,18 +123,6 @@ async def get_random_code(style: str | None = None) -> str: return str(code) -async def calculate_file_hash(file: UploadFile, chunk_size=1024 * 1024) -> str: - sha = hashlib.sha256() - await file.seek(0) - while True: - chunk = await file.read(chunk_size) - if not chunk: - break - sha.update(chunk) - await file.seek(0) - return sha.hexdigest() - - ip_limit = { "error": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute), "metadata": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute), diff --git a/apps/base/views.py b/apps/base/views.py index aad50a9a5..a16792507 100644 --- a/apps/base/views.py +++ b/apps/base/views.py @@ -1,4 +1,3 @@ -import datetime import hashlib import os import uuid @@ -16,6 +15,7 @@ from apps.admin.dependencies import share_required_login from apps.base.models import FileCodes, UploadChunk, PresignUploadSession from apps.base.quota import release_storage, reserve_storage +from core.logger import logger from apps.base.schemas import ( SelectFileModel, InitChunkUploadModel, @@ -24,6 +24,7 @@ ) from apps.base.file_validation import validate_file_type, validate_upload_file, validate_header_bytes from apps.base.utils import ( + build_file_path, get_expire_info, get_file_path_name, ip_limit, @@ -51,16 +52,8 @@ class FileUploadService: async def generate_file_path( file_name: str, upload_id: Optional[str] = None ) -> tuple[str, str, str, str, str]: - """统一的路径生成""" - today = datetime.datetime.now() - storage_path = settings.storage_path.strip("/") - file_uuid = upload_id or uuid.uuid4().hex - filename = await sanitize_filename(unquote(file_name)) - base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{file_uuid}" - path = f"{storage_path}/{base_path}" if storage_path else base_path - prefix, suffix = os.path.splitext(filename) - save_path = f"{path}/{filename}" - return path, suffix, prefix, filename, save_path + """Delegates path generation to apps.base.utils.build_file_path.""" + return await build_file_path(file_name, upload_id or uuid.uuid4().hex) @staticmethod async def create_file_record( @@ -186,7 +179,7 @@ async def share_file( FileCodes(file_path=path, uuid_file_name=uuid_file_name) ) except Exception: - pass + logger.warning("分享上传：记录创建失败，回滚删除已保存文件失败", exc_info=True) raise finally: await release_storage(reservation_token) @@ -404,21 +397,6 @@ async def init_chunk_upload(data: InitChunkUploadModel = Depends(parse_init_chun status_code=403, detail=f"文件大小超过限制，最大为 {max_size_mb:.2f} MB" ) - # # 秒传检查 - # existing = await FileCodes.filter(file_hash=data.file_hash).first() - # if existing: - # if await existing.is_expired(): - # file_storage: FileStorageInterface = storages[settings.file_storage]( - # ) - # await file_storage.delete_file(existing) - # await existing.delete() - # else: - # return APIResponse(detail={ - # "code": existing.code, - # "existed": True, - # "name": f'{existing.prefix}{existing.suffix}' - # }) - # 断点续传：检查是否存在相同文件的未完成上传会话 existing_session = await UploadChunk.filter( chunk_hash=data.file_hash, @@ -591,8 +569,8 @@ async def cancel_upload(upload_id: str): if save_path: try: await storage.clean_chunks(upload_id, save_path) - except Exception as e: - pass + except Exception: + logger.warning("取消分片上传：清理分片文件失败 upload_id=%s", upload_id, exc_info=True) # 清理数据库记录 await UploadChunk.filter(upload_id=upload_id).delete() @@ -663,7 +641,7 @@ async def complete_upload( try: await storage.clean_chunks(upload_id, save_path) except Exception: - pass + logger.warning("分片超限中止：清理分片文件失败 upload_id=%s", upload_id, exc_info=True) await UploadChunk.filter(upload_id=upload_id).delete() await release_storage(f"chunk:{upload_id}") max_size_mb = settings.uploadSize / (1024 * 1024) @@ -711,7 +689,7 @@ async def complete_upload( try: await storage.clean_chunks(upload_id, save_path) except Exception: - pass + logger.warning("分片合并失败：清理临时分片文件失败 upload_id=%s", upload_id, exc_info=True) raise HTTPException( status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"文件合并失败: {str(e)}" ) @@ -847,7 +825,7 @@ async def presign_upload_proxy( ) ) except Exception: - pass + logger.warning("预签名代理上传：记录创建失败，回滚删除已保存文件失败 upload_id=%s", session.upload_id, exc_info=True) raise await session.delete() @@ -904,7 +882,7 @@ async def presign_upload_confirm(upload_id: str, ip: str = Depends(ip_limit["upl ) ) except Exception: - pass + logger.warning("预签名确认：记录创建失败，回滚删除已保存文件失败 upload_id=%s", session.upload_id, exc_info=True) raise await session.delete() @@ -952,7 +930,7 @@ async def presign_upload_cancel(upload_id: str): ) await storage.delete_file(temp_file_code) except Exception: - pass + logger.warning("取消预签名会话：清理临时文件失败 upload_id=%s", upload_id, exc_info=True) await session.delete() await release_storage(f"presign:{upload_id}") diff --git a/docs/guide/storage-onedrive.md b/docs/guide/storage-onedrive.md deleted file mode 100644 index 796a3c0c9..000000000 --- a/docs/guide/storage-onedrive.md +++ /dev/null @@ -1,135 +0,0 @@ -# OneDrive作为存储的配置方法 - -**仅支持工作或学校账户，并且需要有管理员权限以授权API** - -## 1. 需要配置的参数 - -``` -file_storage=onedrive -onedrive_domain=XXXXXX -onedrive_client_id=XXXXXX-XXXXXX-XXXXXX-XXXXXX -onedrive_username=XXXXXX@XXXXXX -onedrive_password=XXXXXX -``` - -`onedrive_username`和`onedrive_password`是你的账户名（邮箱）和密码，另外两个参数需要在[微软Azure门户](https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade)中注册应用后获取。 - -## 2. 应用注册 - -1. 登录[https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade](https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade)，鼠标置于右上角账号处，浮窗将显示的`域`即为`onedrive_domain`的值。 -![onedrive_domain](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGCiErO85doq9Tcu/root/content) - -2. 点击左上角的`+新注册`，输入名称， - * 受支持的帐户类型：选择任何组织目录(任何 Azure AD 目录 - 多租户)中的帐户和个人 Microsoft 帐户(例如，Skype、Xbox) - * 重定向 URI (可选)：选择`Web`，并输入`http://localhost` - -3. 完成注册后进入概述页面，在概要中找到`应用程序(客户端)ID`，即为`onedrive_client_id`的值。 -![onedrive_client_id](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGHD4CNyJxm_QBb8/root/content) - -4. 此时还需要配置允许公共客户端流和API权限 - * 在左侧选择`身份验证`，找到`允许的客户端流`，选择`是`，并**点击`保存`**。 - ![允许的客户端流](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGJQMOlOCb2-L0Lh/root/content) - * 在左侧选择`API权限`，点击`+添加权限`，选择`Microsoft Graph`->`委托的权限`，并勾选下述权限：openid、Files中所有权限、User.Read，如下图所示。最后**点击下方的`添加权限`**。 - ![添加权限](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGOZzz7sIrdXkD4w/root/content) - * 最后点击`授予管理员同意`，并**点击`是`**，最终状态变为`已授予`。 - ![授予管理员同意](https://api.onedrive.com/v1.0/shares/s!Au-BDzXcM6_VmGSOAnjnHUlbirbU/root/content) - -## 3. 使用下述代码测试是否配置成功 - -安装依赖：`pip install Office365-REST-Python-Client` - -```python -# common.py -import msal -domain = 'XXXXXX' -client_id = 'XXXXXX' -username = 'XXXXXX' -password = 'XXXXXX' - -def acquire_token_pwd(): - authority_url = f'https://login.microsoftonline.com/{domain}' - app = msal.PublicClientApplication( - authority=authority_url, - client_id=client_id - ) - result = app.acquire_token_by_username_password( - username=username, - password=password, - scopes=['https://graph.microsoft.com/.default'] - ) - return result -``` - -测试登录，如果成功打印出账户名，说明配置成功。 - -```python -from common import acquire_token_pwd - -from office365.graph_client import GraphClient -try: - client = GraphClient(acquire_token_pwd) - me = client.me.get().execute_query() - print(me.user_principal_name) -except Exception as e: - print(e) -``` - -测试文件上传 - -```python -import os -from office365.graph_client import GraphClient -from common import acquire_token_pwd - -remote_path = 'tmp' -local_path = '.tmp/1689843925000.png' - -def convert_link_to_download_link(link): - import re - p1 = re.search(r'https:\/\/(.+)\.sharepoint\.com', link).group(1) - p2 = re.search(r'personal\/(.+)\/', link).group(1) - p3 = re.search(rf'{p2}\/(.+)', link).group(1) - return f'https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}' - -client = GraphClient(acquire_token_pwd) -folder = client.me.drive.root.get_by_path(remote_path) -# 1. upload -file = folder.upload_file(local_path).execute_query() -print(f'File {file.web_url} has been uploaded') -# 2. create sharing link -remote_file = folder.get_by_path(os.path.basename(local_path)) -permission = remote_file.create_link("view", "anonymous").execute_query() -print(f"sharing link: {convert_link_to_download_link(permission.link.webUrl)}") -``` - -测试文件下载 - -```python -import os -from office365.graph_client import GraphClient -from common import acquire_token_pwd - -remote_path = 'tmp/1689843925000.png' -local_path = '.tmp' -if not os.path.exists(local_path): - os.makedirs(local_path) - -client = GraphClient(acquire_token_pwd) -remote_file = client.me.drive.root.get_by_path(remote_path).get().execute_query() -with open(os.path.join(local_path, os.path.basename(remote_path)), 'wb') as local_file: - remote_file.download(local_file).execute_query() - print(f'{remote_file.name} has been downloaded into {local_file.name}') -``` - -测试删除文件 - -```python -from office365.graph_client import GraphClient -from common import acquire_token_pwd - -remote_path = 'tmp/1689843925000.png' - -client = GraphClient(acquire_token_pwd) -file = client.me.drive.root.get_by_path(remote_path) -file.delete_object().execute_query() -``` diff --git a/docs/guide/storage-opendal.md b/docs/guide/storage-opendal.md deleted file mode 100644 index c4d383f96..000000000 --- a/docs/guide/storage-opendal.md +++ /dev/null @@ -1,30 +0,0 @@ -# 通过 OpenDAL 集成存储的配置方法 - -## 需要配置的参数 - -```dotenv -file_storage=opendal -opendal_scheme= -opendal__=... -``` - -以 Gcs 为例，需要配置的参数如下： -```dotenv -file_storage=opendal -opendal_scheme=gcs -opendal_gcs_root= -opendal_gcs_bucket= -opendal_gcs_credential= -``` - -所有支持的服务可以在[此处](https://opendal.apache.org/docs/rust/opendal/services/index.html)查看。 -具体服务的配置参数与 OpenDAL 文档一致。 - -## 补充说明 - -通过 OpenDAL 集成的服务均通过服务器中转下载。因此，每次下载既消耗存储服务的流量，也消耗服务器的流量。 - -OpenDAL 和该项目本身都支持本地存储、`s3`、`onedrive`。不同之处有以下几点: -1. 项目的支持通过预签名实现，不消耗服务器流量。而 OpenDAL 通过服务器中转下载，消耗服务器流量。（本地存储除外） -2. 项目的支持对于异常情况可能会有更多的调试信息，方便排查问题。 -3. OpenDAL 项目本身采用 Rust 编写，性能更好。 \ No newline at end of file diff --git a/main.py b/main.py index f916683df..b6b605c66 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,6 @@ from apps.admin.views import admin_api from apps.base.models import KeyValue -from apps.base.utils import ip_limit from apps.base.views import share_api, chunk_api, presign_api from core.config import ( ensure_security_settings, @@ -755,16 +754,12 @@ async def load_config(): await KeyValue.update_or_create( key="sys_start", defaults={"value": int(time.time() * 1000)} ) + # refresh_settings already syncs every rate limiter (error/metadata/upload/login) + # via _sync_ip_limits; do not hand-sync a subset here — that once drifted by + # missing the metadata limiter. await refresh_settings() await ensure_security_settings() - ip_limit["error"].minutes = settings.errorMinute - ip_limit["error"].count = settings.errorCount - ip_limit["upload"].minutes = settings.uploadMinute - ip_limit["upload"].count = settings.uploadCount - ip_limit["login"].minutes = settings.loginMinute - ip_limit["login"].count = settings.loginCount - app = FastAPI(lifespan=lifespan, version=APP_VERSION) @app.middleware("http") @@ -879,14 +874,17 @@ async def theme_asset(asset_path: str): @app.exception_handler(404) @app.get("/") async def index(request=None, exc=None): + # Site config is admin input (and during the setup window anyone can claim it); + # always escape before injecting into the theme template to prevent stored XSS + # (mirrors the setup page). return HTMLResponse( content=resolve_theme_file("index.html") .read_text(encoding="utf-8") - .replace("{{title}}", str(settings.name)) - .replace("{{description}}", str(settings.description)) - .replace("{{keywords}}", str(settings.keywords)) - .replace("{{opacity}}", str(settings.opacity)) - .replace("{{background}}", str(settings.background)), + .replace("{{title}}", html.escape(str(settings.name))) + .replace("{{description}}", html.escape(str(settings.description))) + .replace("{{keywords}}", html.escape(str(settings.keywords))) + .replace("{{opacity}}", html.escape(str(settings.opacity))) + .replace("{{background}}", html.escape(str(settings.background))), media_type="text/html", headers={"Cache-Control": "no-cache"}, ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..5087b126d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +# FileCodeBox tooling config. +# Application repo (not a library): no [project] packaging metadata here, and +# requirements.txt remains the single source of runtime deps. This file only +# holds tool configs plus the dev dependency group (PEP 735, `uv sync --group dev`). + +[tool.pytest.ini_options] +testpaths = ["tests"] +# Tests import top-level packages (core/apps) directly; put the repo root on +# sys.path so bare `pytest` collects without `python -m pytest`. +pythonpath = ["."] + +[dependency-groups] +dev = [ + "pytest", + "pytest-asyncio", + "httpx", + "ruff", + "pre-commit", +] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +# Core correctness rules only for now (same families as ruff defaults): +# E4/E7/E9 = syntax/indent errors and unused variables; F = pyflakes +# (unused imports, undefined names). Style families (pyupgrade, broad-except, +# import sorting) are deferred to keep the initial diff minimal. +select = ["E4", "E7", "E9", "F"] diff --git a/requirements.txt b/requirements.txt index 1e572e657..8a1227f2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aioboto3==15.5.0 aiohttp==3.14.2 aiofiles==25.1.0 fastapi==0.139.2 -starlette==1.3.1 +starlette==1.6.0 pydantic==2.12.5 uvicorn==0.51.0 tortoise-orm==0.25.3 diff --git a/tests/test_index_template_escaping.py b/tests/test_index_template_escaping.py new file mode 100644 index 000000000..b85670991 --- /dev/null +++ b/tests/test_index_template_escaping.py @@ -0,0 +1,68 @@ +import asyncio +import tempfile +import unittest +from pathlib import Path + +import main +from core.settings import settings + + +class IndexTemplateEscapingTests(unittest.TestCase): + """Regression: site config must be html.escape-d before template injection + (stored XSS prevention). + + Uses a temp file as a stand-in for themes/*/index.html so the test also + runs in a bare checkout without built frontends. + """ + + def setUp(self): + self._original_user_config = dict(settings.user_config) + self._original_resolve_theme_file = main.resolve_theme_file + tmp = tempfile.NamedTemporaryFile( + "w", suffix=".html", delete=False, encoding="utf-8" + ) + tmp.write( + "{{title}}" + '' + "
{{opacity}}
" + ) + tmp.close() + self._template_path = Path(tmp.name) + + def tearDown(self): + settings.user_config = self._original_user_config + main.resolve_theme_file = self._original_resolve_theme_file + self._template_path.unlink() + + def _patch_template(self): + main.resolve_theme_file = lambda *args, **kwargs: self._template_path + + def _render_index_html(self) -> str: + return asyncio.run(main.index()).body.decode("utf-8") + + def test_malicious_site_config_is_escaped(self): + self._patch_template() + settings.name = "" + settings.description = '">' + + html = self._render_index_html() + + self.assertNotIn("