From 7feec2b7d9f83a1f3e4cca4bc39b71c78e9a1f8c Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Tue, 12 May 2026 21:45:06 +0800 Subject: [PATCH 001/107] fix(driver): fix 189 & 189pc fastcopy form local storage (#2471) * fix(driver): fix 189 & 189pc fastcopy form local storage * fix(driver): fix 189 & 189pc fastcopy form local storage * fix(driver): fix 189 & 189pc fastcopy form local storage --- drivers/189/util.go | 143 +++++++++++++++++++++++++++++++++------- drivers/189pc/driver.go | 13 ++++ 2 files changed, 132 insertions(+), 24 deletions(-) diff --git a/drivers/189/util.go b/drivers/189/util.go index bb9a6adb4f..ca86937c97 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -311,48 +311,99 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } d.sessionKey = sessionKey const DEFAULT int64 = 10485760 - count := int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + fileSize := file.GetSize() + count := int64(math.Ceil(float64(fileSize) / float64(DEFAULT))) - res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + // 先计算文件完整MD5和分片MD5，用于秒传判断 + fileMd5Hex := file.GetHash().GetHash(utils.MD5) + sliceMd5Hex := "" + md5s := make([]string, 0) + + if len(fileMd5Hex) < utils.MD5.Width { + // 没有MD5，先缓存流并同时计算文件MD5和分片MD5 + fileMd5Hash := md5.New() + sliceMd5Hash := md5.New() + var finish int64 + cache, err := file.CacheFullAndWriter(nil, io.MultiWriter(fileMd5Hash, &sliceHashWriter{ + hash: sliceMd5Hash, + md5s: &md5s, + sliceSize: DEFAULT, + finish: &finish, + fileSize: fileSize, + up: up, + ctx: ctx, + })) + if err != nil { + return err + } + // 处理最后一个分片的MD5 + if finish%DEFAULT != 0 || finish == 0 { + md5s = append(md5s, strings.ToUpper(hex.EncodeToString(sliceMd5Hash.Sum(nil)))) + } + fileMd5Hex = hex.EncodeToString(fileMd5Hash.Sum(nil)) + + // seek回起始位置，供后续上传使用 + if _, err := cache.Seek(0, io.SeekStart); err != nil { + return err + } + } + + // 计算sliceMd5 + if fileSize > DEFAULT && len(md5s) > 0 { + sliceMd5Hex = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + } else { + sliceMd5Hex = fileMd5Hex + } + + // 带fileMd5调用initMultiUpload，支持秒传 + initParams := map[string]string{ "parentFolderId": dstDir.GetID(), "fileName": encode(file.GetName()), - "fileSize": strconv.FormatInt(file.GetSize(), 10), + "fileSize": strconv.FormatInt(fileSize, 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), - "lazyCheck": "1", - }, nil) + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + } + + res, err := d.uploadRequest("/person/initMultiUpload", initParams, nil) if err != nil { return err } uploadFileId := jsoniter.Get(res, "data", "uploadFileId").ToString() - //_, err = d.uploadRequest("/person/getUploadedPartsInfo", map[string]string{ - // "uploadFileId": uploadFileId, - //}, nil) + fileDataExists := jsoniter.Get(res, "data", "fileDataExists").ToInt() + + // 秒传成功，直接提交 + if fileDataExists == 1 { + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": "3", + }, nil) + return err + } + + // 非秒传，需要上传分片 var finish int64 = 0 var i int64 var byteSize int64 - md5s := make([]string, 0) - md5Sum := md5.New() for i = 1; i <= count; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize = file.GetSize() - finish + byteSize = fileSize - finish if DEFAULT < byteSize { byteSize = DEFAULT } - // log.Debugf("%d,%d", byteSize, finish) byteData := make([]byte, byteSize) n, err := io.ReadFull(file, byteData) - // log.Debug(err, n) if err != nil { return err } finish += int64(n) md5Bytes := getMd5(byteData) - md5Hex := hex.EncodeToString(md5Bytes) md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) - md5s = append(md5s, strings.ToUpper(md5Hex)) - md5Sum.Write(byteData) var resp UploadUrlsResp res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{ "partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64), @@ -379,17 +430,12 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } log.Debugf("%+v %+v", r, r.Request.Header) _ = r.Body.Close() - up(float64(i) * 100 / float64(count)) - } - fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) - sliceMd5 := fileMd5 - if file.GetSize() > DEFAULT { - sliceMd5 = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + up(50 + float64(i)*50/float64(count)) } res, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ "uploadFileId": uploadFileId, - "fileMd5": fileMd5, - "sliceMd5": sliceMd5, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "opertype": "3", }, nil) @@ -406,3 +452,52 @@ func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { } return &resp, nil } + +// sliceHashWriter 在写入过程中按分片大小自动切分并计算每个分片的MD5， +// 同时支持进度回调和取消检查。 +type sliceHashWriter struct { + hash io.Writer // 当前分片的MD5 hash + md5s *[]string // 收集每个分片的MD5十六进制字符串 + sliceSize int64 // 分片大小 + finish *int64 // 已写入的总字节数 + fileSize int64 // 文件总大小 + up driver.UpdateProgress + ctx context.Context +} + +func (w *sliceHashWriter) Write(p []byte) (int, error) { + if utils.IsCanceled(w.ctx) { + return 0, w.ctx.Err() + } + total := len(p) + written := 0 + for written < total { + // 当前分片还能写入的字节数 + sliceRemain := w.sliceSize - (*w.finish % w.sliceSize) + toWrite := int64(total - written) + if toWrite > sliceRemain { + toWrite = sliceRemain + } + n, err := w.hash.Write(p[written : written+int(toWrite)]) + if err != nil { + return written, err + } + written += n + *w.finish += int64(n) + + // 当前分片写满，记录MD5并重置 + if *w.finish%w.sliceSize == 0 { + if h, ok := w.hash.(interface{ Sum([]byte) []byte }); ok { + *w.md5s = append(*w.md5s, strings.ToUpper(hex.EncodeToString(h.Sum(nil)))) + } + if resetter, ok := w.hash.(interface{ Reset() }); ok { + resetter.Reset() + } + } + } + // 报告进度（缓存阶段占50%） + if w.fileSize > 0 && w.up != nil { + w.up(float64(*w.finish) / float64(w.fileSize) * 50) + } + return total, nil +} diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index af719401d3..901a434846 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -411,6 +411,19 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if stream.GetSize() == 0 { return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } + // 尝试秒传：如果已有MD5且启用了RapidUpload则用RapidUpload，否则走FastUpload（会计算MD5并尝试秒传） + if !stream.IsForceStreamUpload() { + fileMd5 := stream.GetHash().GetHash(utils.MD5) + if len(fileMd5) >= utils.MD5.Width && y.Addition.RapidUpload { + // 源文件已有MD5且启用了RapidUpload配置，尝试快速秒传 + if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { + return newObj, nil + } + } else if len(fileMd5) < utils.MD5.Width { + // 源文件无MD5（如从本地复制），走FastUpload计算MD5并尝试秒传 + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) + } + } fallthrough default: return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) From 0726d166321b06e363ce9e9892b978b2ce3cdcbb Mon Sep 17 00:00:00 2001 From: LuckyPhoenix <98106970+zypluckyphoenix@users.noreply.github.com> Date: Tue, 12 May 2026 23:08:02 +0800 Subject: [PATCH 002/107] feat(189pc): implement AccessToken login (#2245) Optimize the 189pc driver to implement AccessToken login --- drivers/189pc/driver.go | 7 ++++++- drivers/189pc/meta.go | 1 + drivers/189pc/utils.go | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 901a434846..b0a23e9d24 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -87,7 +87,12 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { } // 先尝试用Token刷新，之后尝试登陆 - if y.Addition.RefreshToken != "" { + if y.Addition.AccessToken != "" { + y.tokenInfo = &AppSessionResp{AccessToken: y.Addition.AccessToken, RefreshToken: y.Addition.RefreshToken} + if err = y.refreshSession(); err != nil { + return err + } + } else if y.Addition.RefreshToken != "" { y.tokenInfo = &AppSessionResp{RefreshToken: y.Addition.RefreshToken} if err = y.refreshToken(); err != nil { return err diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 670b991163..366f1bf818 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -10,6 +10,7 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` VCode string `json:"validate_code"` + AccessToken string `json:"access_token" required:"false"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 08ee658ca0..cc5e227c61 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -356,6 +356,7 @@ func (y *Cloud189PC) loginByPassword() (err error) { err = fmt.Errorf(tokenInfo.ResMessage) return err } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) @@ -414,6 +415,7 @@ func (y *Cloud189PC) loginByQRCode() error { if tokenInfo.ResCode != 0 { return fmt.Errorf(tokenInfo.ResMessage) } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) @@ -661,6 +663,7 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { return y.login() } + y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) From b6db83ed5ead0671140cd0af2a1871723feed231 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Thu, 14 May 2026 22:14:54 +0800 Subject: [PATCH 003/107] refactor: unify stream caching via HybridCache (#2460) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add LinearMemory * replace mmap with LinearMemory * remove unused code * add GuardedMemory; add `min_free_memoryMB` conf * add HybridCache and StreamBuffer * log * rename SizedReadWriterAt to Section * 重构 FileStream，改用 HybridCache * 重构 HybridCache，更新方法名并添加回滚功能；优化请求和流处理逻辑 * 重构 StreamSectionReader 接口，使用HybridCache * 在 NewGuardedMemory 函数中添加了对 LinearMemory 的最终化处理，以确保内存释放 * . * 优化检查逻辑 * 重命名 * 改进、重命名 * 修复 * 添加测试 * 移除HybridCacheReader并引入DynamicReadAtSeeker * 重构缓存读取逻辑，简化代码并引入ReadFromN方法 * 优化缓存配置注释并修复下载器部分大小限制逻辑 * 优化中断逻辑 * 优化下载器代码 * 修复bug * HybridCache添加多文件缓存模式 * 优化下载器并发，添加测试 * 修复bug * 重命名+注释 * . * fix(net): always cleanup downloader on interrupt * fix(net): guard chunk enqueue with context cancel * fix(net): update interrupt logic and add download interrupt test * fix(test): update concurrency limit in high concurrency test * refactor(buffer): simplify ReadAt logic * refactor(config): update memory configuration logic * . --------- Co-authored-by: Suyunmeng --- drivers/teldrive/types.go | 2 +- internal/bootstrap/config.go | 48 ++- internal/cache/file.go | 186 ++++++++++ internal/cache/file_test.go | 101 ++++++ internal/conf/config.go | 8 +- internal/conf/var.go | 13 +- internal/mem/cache.go | 205 +++++++++++ internal/mem/mem_other.go | 25 ++ internal/mem/mem_unix.go | 92 +++++ internal/mem/mem_windows.go | 94 ++++++ internal/mem/type.go | 9 + internal/mem/utils.go | 79 +++++ internal/model/file.go | 16 +- internal/net/request.go | 598 +++++++++++++-------------------- internal/net/request_test.go | 115 ++++++- internal/stream/stream.go | 163 ++++----- internal/stream/stream_test.go | 96 +++++- internal/stream/util.go | 192 ++++++----- pkg/buffer/bytes.go | 69 +++- pkg/buffer/bytes_test.go | 18 +- pkg/buffer/pipe.go | 157 +++++++++ pkg/buffer/type.go | 22 ++ pkg/buffer/utils.go | 97 ++++++ pkg/pool/pool.go | 13 +- 24 files changed, 1767 insertions(+), 651 deletions(-) create mode 100644 internal/cache/file.go create mode 100644 internal/cache/file_test.go create mode 100644 internal/mem/cache.go create mode 100644 internal/mem/mem_other.go create mode 100644 internal/mem/mem_unix.go create mode 100644 internal/mem/mem_windows.go create mode 100644 internal/mem/type.go create mode 100644 internal/mem/utils.go create mode 100644 pkg/buffer/pipe.go create mode 100644 pkg/buffer/type.go create mode 100644 pkg/buffer/utils.go diff --git a/drivers/teldrive/types.go b/drivers/teldrive/types.go index 084f967e6e..d75347b897 100644 --- a/drivers/teldrive/types.go +++ b/drivers/teldrive/types.go @@ -52,7 +52,7 @@ type chunkTask struct { fileName string chunkSize int64 reader io.ReadSeeker - ss stream.StreamSectionReaderIF + ss stream.StreamSectionReader } type CopyManager struct { diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 74e218f8f6..4e44d079bd 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "math" "net/url" "os" "path/filepath" @@ -96,27 +97,46 @@ func InitConfig() { confFromEnv() } - if conf.Conf.MaxConcurrency > 0 { - net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency} + if conf.Conf.MaxConcurrency > math.MaxInt32 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: math.MaxInt32} + } else if conf.Conf.MaxConcurrency > 0 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: uint32(conf.Conf.MaxConcurrency)} } - if conf.Conf.MaxBufferLimit < 0 { - m, _ := mem.VirtualMemory() - if m != nil { - conf.MaxBufferLimit = max(int(float64(m.Total)*0.05), 4*utils.MB) - conf.MaxBufferLimit -= conf.MaxBufferLimit % utils.MB + + memStat, _ := mem.VirtualMemory() + if memStat != nil { + log.Infof("total memory: %dMB, available: %dMB", memStat.Total>>20, memStat.Available>>20) + if conf.Conf.MinFreeMemory < 0 { + conf.MinFreeMemory = 0 + log.Info("disable memory cache") } else { - conf.MaxBufferLimit = 16 * utils.MB + if conf.Conf.MinFreeMemory < 16 { + t := (memStat.Total >> 20) / 10 + conf.MinFreeMemory = max(16, min(t, 1024)) << 20 + } else { + conf.MinFreeMemory = uint64(conf.Conf.MinFreeMemory) << 20 + } + log.Infof("min free memory: %dMB", conf.MinFreeMemory>>20) } + + if conf.Conf.MaxBlockLimit < 4 { + t := (memStat.Total >> 20) * 3 / 100 + conf.MaxBlockLimit = max(4, min(uint64(t), 64)) << 20 + } else { + conf.MaxBlockLimit = uint64(conf.Conf.MaxBlockLimit) << 20 + } + log.Infof("max block limit: %dMB", conf.MaxBlockLimit>>20) } else { - conf.MaxBufferLimit = conf.Conf.MaxBufferLimit * utils.MB + conf.MinFreeMemory = 0 + log.Warn("failed to get memory info, disable memory cache") } - log.Infof("max buffer limit: %dMB", conf.MaxBufferLimit/utils.MB) - if conf.Conf.MmapThreshold > 0 { - conf.MmapThreshold = conf.Conf.MmapThreshold * utils.MB + + if conf.Conf.CacheThreshold > 0 { + conf.CacheThreshold = uint64(conf.Conf.CacheThreshold) << 20 } else { - conf.MmapThreshold = 0 + conf.CacheThreshold = 0 } - log.Infof("mmap threshold: %dMB", conf.Conf.MmapThreshold) + log.Infof("cache threshold: %dMB", conf.CacheThreshold>>20) if len(conf.Conf.Log.Filter.Filters) == 0 { conf.Conf.Log.Filter.Enable = false diff --git a/internal/cache/file.go b/internal/cache/file.go new file mode 100644 index 0000000000..64e88fb844 --- /dev/null +++ b/internal/cache/file.go @@ -0,0 +1,186 @@ +package cache + +import ( + "errors" + "io" + "os" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" +) + +type FileCache interface { + buffer.Block + io.Closer + Truncate(size int64) error +} + +type singleFileCache struct { + *os.File + size int64 +} + +func (s *singleFileCache) Size() int64 { + return s.size +} + +func (s *singleFileCache) Truncate(size int64) error { + if size <= s.size { + return nil + } + err := s.File.Truncate(size) + if err == nil { + s.size = size + } + return err +} + +func (s *singleFileCache) Close() error { + err := s.File.Close() + _ = os.Remove(s.File.Name()) + return err +} + +type fileBlock struct { + file *os.File + size int64 + written int64 +} + +type MultiFileCache struct { + blocks []*fileBlock + size int64 +} + +func (s *MultiFileCache) Size() int64 { + return s.size +} + +func (m *MultiFileCache) Close() error { + var errs []error + for _, c := range m.blocks { + if err := c.file.Close(); err != nil { + errs = append(errs, err) + } + _ = os.Remove(c.file.Name()) + } + clear(m.blocks) + m.blocks = m.blocks[:0] + return errors.Join(errs...) +} + +func (m *MultiFileCache) Truncate(size int64) error { + if size <= m.size { + return nil + } + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + m.blocks = append(m.blocks, &fileBlock{file: f, size: size - m.size}) + m.size = size + return nil +} + +func (m *MultiFileCache) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + for _, c := range m.blocks { + if off >= c.size { + off -= c.size + continue + } + + canRead := min(len(p)-n, int(c.size-off)) + if canRead <= 0 { + break + } + + filled := 0 + + if off < c.written { + fileReadable := min(canRead, int(c.written-off)) + nn, fileErr := c.file.ReadAt(p[n:n+fileReadable], off) + n += nn + filled = nn + if fileErr != nil && !errors.Is(fileErr, io.EOF) { + return n, fileErr + } + } + + if n == len(p) { + return n, nil + } + + if zeroFill := canRead - filled; zeroFill > 0 { + clear(p[n : n+zeroFill]) + n += zeroFill + } + + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *MultiFileCache) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + for _, b := range m.blocks { + if off >= b.size { + off -= b.size + continue + } + + canWrite := min(len(p)-n, int(b.size-off)) + if canWrite <= 0 { + break + } + + nn, fileErr := b.file.WriteAt(p[n:n+canWrite], off) + if end := off + int64(nn); end > b.written { + b.written = end + } + n += nn + if fileErr != nil { + return n, fileErr + } + if nn < canWrite { + return n, io.ErrShortWrite + } + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func NewFileCache(blockSize int64) (FileCache, error) { + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + err = f.Truncate(blockSize) + if err == nil { + return &singleFileCache{File: f, size: blockSize}, nil + } + return &MultiFileCache{ + blocks: []*fileBlock{{file: f, size: blockSize}}, + size: blockSize, + }, nil +} diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go new file mode 100644 index 0000000000..5f30c01e82 --- /dev/null +++ b/internal/cache/file_test.go @@ -0,0 +1,101 @@ +package cache_test + +import ( + "bytes" + "errors" + "io" + "os" + "reflect" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestFile(t *testing.T) { + f, err := os.CreateTemp("", "writeat-*") + if err != nil { + t.Error(err) + return + } + defer os.Remove(f.Name()) + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + n, err := f.WriteAt([]byte("abc"), 20) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + stat, err := f.Stat() + if err != nil { + t.Errorf("stat err=%v", err) + return + } + if stat.Size() != 23 { + t.Fatalf("unexpected size: got %d want 23", stat.Size()) + } + + b := make([]byte, stat.Size()) + rn, rerr := f.ReadAt(b, 0) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := append(make([]byte, 20), []byte("abc")...) + if !reflect.DeepEqual(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} + +func TestMultiFileCache(t *testing.T) { + prevConf := conf.Conf + t.Cleanup(func() { + conf.Conf = prevConf + }) + conf.Conf = &conf.Config{} + f := cache.MultiFileCache{} + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + err := f.Truncate(15) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + n, err := f.WriteAt([]byte("abc"), 10) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + + err = f.Truncate(30) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + _, _ = f.WriteAt([]byte("123"), 15) + + b := append(make([]byte, 17), []byte("def")...) + b[0] = 'a' + rn, rerr := f.ReadAt(b, 8) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := []byte{0, 0, 'a', 'b', 'c', 0, 0, '1', '2', '3'} + want = append(want, make([]byte, 10)...) + if !bytes.Equal(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index f347380d81..a26ddc306f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -120,8 +120,9 @@ type Config struct { DistDir string `json:"dist_dir"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` - MaxBufferLimit int `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"` - MmapThreshold int `json:"mmap_thresholdMB" env:"MMAP_THRESHOLD_MB"` + MinFreeMemory int `json:"min_free_memory" env:"MIN_FREE_MEMORY"` + MaxBlockLimit int `json:"max_block_limit" env:"MAX_BLOCK_LIMIT"` + CacheThreshold int `json:"cache_threshold" env:"CACHE_THRESHOLD"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` @@ -178,8 +179,7 @@ func DefaultConfig(dataDir string) *Config { }, }, }, - MaxBufferLimit: -1, - MmapThreshold: 4, + CacheThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, TlsInsecureSkipVerify: false, diff --git a/internal/conf/var.go b/internal/conf/var.go index 972f69997a..0ca954dc34 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -25,10 +25,15 @@ var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( - // 单个Buffer最大限制 - MaxBufferLimit = 16 * 1024 * 1024 - // 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存 - MmapThreshold = 4 * 1024 * 1024 + // 限制单次内存的扩容大小，超过该阈值将分多次扩容。 + // CacheThreshold大于0时，也限制 Downloader 的PartSize + MaxBlockLimit uint64 = 16 * 1024 * 1024 + // 大于该阈值的数据流将使用HybridCache，可主动释放内存。 + // 否则使用Go的内存分配，直到GC回收。 + CacheThreshold uint64 = 4 * 1024 * 1024 + // 最小空闲内存，当内存不足时，HybridCache会回退到文件缓存。 + // 如果为0，HybridCache会使用文件缓存，不占用内存。 + MinFreeMemory uint64 = 16 * 1024 * 1024 ) var ( RawIndexHtml string diff --git a/internal/mem/cache.go b/internal/mem/cache.go new file mode 100644 index 0000000000..e2c008875c --- /dev/null +++ b/internal/mem/cache.go @@ -0,0 +1,205 @@ +package mem + +import ( + "errors" + "io" + "runtime" + + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// 优先使用内存，失败后才使用文件。 +// 线程不安全 +type HybridCache struct { + mem LinearMemory + memOffset uint64 + file cache.FileCache + fileOffset uint64 + blockSize uint64 +} + +func (hc *HybridCache) NextBlockWithSize(size uint64) (buffer.Block, error) { +retry: + if hc.file != nil { + if err := hc.file.Truncate(int64(hc.fileOffset + size)); err != nil { + return nil, err + } + base := hc.fileOffset + hc.fileOffset += size + fs := buffer.NewBlockAdapter( + io.NewOffsetWriter(hc.file, int64(base)), + io.NewSectionReader(hc.file, int64(base), int64(size)), + ) + return fs, nil + } + all, err := hc.mem.Reallocate(hc.memOffset + size) + if err == nil { + start := hc.memOffset + hc.memOffset += size + return buffer.NewByteBlock(all[start : start+size]), nil + } + if err := hc.initFileCache(); err != nil { + return nil, err + } + goto retry +} + +func (hc *HybridCache) NextBlock() (buffer.Block, error) { + return hc.NextBlockWithSize(hc.blockSize) +} + +func (hc *HybridCache) RollbackBlockWithSize(size uint64) { + if hc.fileOffset >= size { + hc.fileOffset -= size + return + } + size -= hc.fileOffset + hc.fileOffset = 0 + if hc.memOffset >= size { + hc.memOffset -= size + return + } + hc.memOffset = 0 +} + +func (hc *HybridCache) RollbackBlock() { + hc.RollbackBlockWithSize(hc.blockSize) +} + +func (hc *HybridCache) initFileCache() error { + file, err := cache.NewFileCache(int64(hc.blockSize)) + if err != nil { + return err + } + hc.file = file + return nil +} + +func (hc *HybridCache) Close() error { + var err error + if hc.mem != nil { + err = hc.mem.Free() + hc.mem = nil + } + if hc.file != nil { + err = errors.Join(err, hc.file.Close()) + hc.file = nil + } + return err +} + +func (hc *HybridCache) Size() int64 { + return int64(hc.memOffset + hc.fileOffset) +} + +func (hc *HybridCache) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.EOF + } + if off < int64(hc.memOffset) { + all, err := hc.mem.Reallocate(min(hc.memOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(p, all[off:]) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memOffset) + canRead := int64(hc.fileOffset) - off + if canRead <= 0 { + return n, io.EOF + } + nn, err := hc.file.ReadAt(p[:min(len(p), int(canRead))], off) + return n + nn, err +} + +func (hc *HybridCache) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.ErrShortWrite + } + + if off < int64(hc.memOffset) { + all, err := hc.mem.Reallocate(min(hc.memOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(all[off:], p) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memOffset) + canWrite := int64(hc.fileOffset) - off + if canWrite <= 0 { + return n, io.ErrShortWrite + } + nn, err := hc.file.WriteAt(p[:min(len(p), int(canWrite))], off) + return n + nn, err +} + +func (hc *HybridCache) CopyFromN(src io.Reader, n int64) (written int64, err error) { + limit := n + for limit > 0 { + blockSize := limit + if hc.file == nil && blockSize > int64(conf.MaxBlockLimit) { + blockSize = int64(conf.MaxBlockLimit) + } + b, err := hc.NextBlockWithSize(uint64(blockSize)) + if err != nil { + return written, err + } + nn, err := utils.CopyWithBufferN(buffer.WriteAtSeekerOf(b), src, blockSize) + written += nn + if nn != blockSize { + return written, err + } + limit -= nn + } + return written, nil +} + +// 优先使用内存，失败后才使用文件 +// 线程不安全 +func NewHybridCache(blockSize, maxMemorySize uint64) (*HybridCache, error) { + var err error + var hc *HybridCache + if conf.MinFreeMemory > 0 && maxMemorySize >= blockSize { + var m LinearMemory + m, err = NewGuardedMemory(blockSize, maxMemorySize) + if err == nil { + hc = &HybridCache{mem: m, blockSize: blockSize} + } + } + if hc == nil { + hc = &HybridCache{blockSize: blockSize} + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + } + runtime.SetFinalizer(hc, func(hc *HybridCache) { + if hc.file != nil { + _ = hc.file.Close() + hc.file = nil + } + }) + return hc, nil +} + +var _ buffer.Block = (*HybridCache)(nil) diff --git a/internal/mem/mem_other.go b/internal/mem/mem_other.go new file mode 100644 index 0000000000..2535c4570a --- /dev/null +++ b/internal/mem/mem_other.go @@ -0,0 +1,25 @@ +//go:build !unix && !windows + +package mem + +func NewMemory(cap, max uint64) (LinearMemory, error) { + return &sliceMemory{buf: make([]byte, 0, cap)}, nil +} + +type sliceMemory struct { + buf []byte +} + +func (b *sliceMemory) Free() error { + b.buf = nil + return nil +} + +func (b *sliceMemory) Reallocate(size uint64) ([]byte, error) { + if cap := uint64(cap(b.buf)); size > cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf, nil +} diff --git a/internal/mem/mem_unix.go b/internal/mem/mem_unix.go new file mode 100644 index 0000000000..bd1979298e --- /dev/null +++ b/internal/mem/mem_unix.go @@ -0,0 +1,92 @@ +//go:build unix + +package mem + +import ( + "math" + + "golang.org/x/sys/unix" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures int(res) overflows to a negative value, + // and unix.Mmap returns EINVAL. + res = math.MaxUint64 + } + + com := res + prot := unix.PROT_READ | unix.PROT_WRITE + if cap < max { // Commit memory only if cap=max. + com = 0 + prot = unix.PROT_NONE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(res), prot, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + return nil, err + } + return &mmappedMemory{buf: b[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type mmappedMemory struct { + buf []byte + growCheck GrowCheck +} + +func (m *mmappedMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *mmappedMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *mmappedMemory) Free() error { + if m.buf != nil { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + return err + } + m.buf = nil + } + return nil +} diff --git a/internal/mem/mem_windows.go b/internal/mem/mem_windows.go new file mode 100644 index 0000000000..e7a4bb27ef --- /dev/null +++ b/internal/mem/mem_windows.go @@ -0,0 +1,94 @@ +package mem + +import ( + "math" + "unsafe" + + "golang.org/x/sys/windows" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures uintptr(res) overflows to a large value, + // and windows.VirtualAlloc returns an error. + res = math.MaxUint64 + } + + com := res + kind := windows.MEM_COMMIT + if cap < max { // Commit memory only if cap=max. + com = 0 + kind = windows.MEM_RESERVE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + r, err := windows.VirtualAlloc(0, uintptr(res), uint32(kind), windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + buf := unsafe.Slice((*byte)(unsafe.Pointer(r)), int(res)) + return &virtualMemory{addr: r, buf: buf[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type virtualMemory struct { + buf []byte + addr uintptr + growCheck GrowCheck +} + +func (m *virtualMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *virtualMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *virtualMemory) Free() error { + if m.addr != 0 { + err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) + if err != nil { + return err + } + m.addr = 0 + m.buf = nil + } + return nil +} diff --git a/internal/mem/type.go b/internal/mem/type.go new file mode 100644 index 0000000000..3ba8e35ca0 --- /dev/null +++ b/internal/mem/type.go @@ -0,0 +1,9 @@ +package mem + +type LinearMemory interface { + // 线程不安全 + Reallocate(size uint64) (all []byte, err error) + Free() error +} + +type GrowCheck func(growSize uint64) error diff --git a/internal/mem/utils.go b/internal/mem/utils.go new file mode 100644 index 0000000000..21944066de --- /dev/null +++ b/internal/mem/utils.go @@ -0,0 +1,79 @@ +package mem + +import ( + "errors" + "fmt" + "runtime" + "sync/atomic" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/shirou/gopsutil/v4/mem" +) + +var ErrNotEnoughMemory = errors.New("not enough memory") + +func MemoryGrowCheck(growSize uint64) error { + if conf.MinFreeMemory == 0 { + return ErrNotEnoughMemory + } + m, err, _ := singleflight.AnyGroup.Do("MemoryGrowCheck", func() (any, error) { + m, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + if m.Available < conf.MinFreeMemory { + return nil, ErrNotEnoughMemory + } + return m, nil + }) + if err != nil { + return err + } + memStat := m.(*mem.VirtualMemoryStat) + for { + available := atomic.LoadUint64(&memStat.Available) + if available < growSize || available-growSize < conf.MinFreeMemory { + return ErrNotEnoughMemory + } + if atomic.CompareAndSwapUint64(&memStat.Available, available, available-growSize) { + return nil + } + } +} + +func NewGuardedMemory(cap, max uint64) (m LinearMemory, err error) { + if err := MemoryGrowCheck(cap); err != nil { + return nil, err + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + m, err = NewMemory(cap, max) + if err != nil { + return nil, err + } + if s, ok := m.(interface{ SetGrowCheck(GrowCheck) }); ok { + s.SetGrowCheck(MemoryGrowCheck) + } + gm := &guardedMemory{m} + runtime.SetFinalizer(gm, func(gm *guardedMemory) { + gm.Free() + }) + return gm, nil +} + +type guardedMemory struct { + LinearMemory +} + +func (s *guardedMemory) Reallocate(size uint64) (all []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + return s.LinearMemory.Reallocate(size) +} diff --git a/internal/model/file.go b/internal/model/file.go index 4ca7201e11..d6697cd0e8 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -7,24 +7,26 @@ import ( // File is basic file level accessing interface type File interface { - io.Reader io.ReaderAt - io.Seeker + io.ReadSeeker +} +type FileWriter interface { + io.WriterAt + io.WriteSeeker } type FileCloser struct { File io.Closer } -func (f *FileCloser) Close() error { - var errs []error +func (f *FileCloser) Close() (err error) { if clr, ok := f.File.(io.Closer); ok { - errs = append(errs, clr.Close()) + err = clr.Close() } if f.Closer != nil { - errs = append(errs, f.Closer.Close()) + return errors.Join(err, f.Closer.Close()) } - return errors.Join(errs...) + return } // FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead diff --git a/internal/net/request.go b/internal/net/request.go index e1f0451205..22ae1f44bc 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -5,17 +5,20 @@ import ( "errors" "fmt" "io" + "math/rand/v2" "net/http" stdpath "path" "strconv" "sync" + "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/mem" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/aws/aws-sdk-go/aws/awsutil" @@ -86,8 +89,8 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo if impl.cfg.PartSize == 0 { impl.cfg.PartSize = DefaultDownloadPartSize } - if conf.MaxBufferLimit > 0 && impl.cfg.PartSize > conf.MaxBufferLimit { - impl.cfg.PartSize = conf.MaxBufferLimit + if conf.MinFreeMemory > 0 && impl.cfg.PartSize > int(conf.MaxBlockLimit) { + impl.cfg.PartSize = int(conf.MaxBlockLimit) } if impl.cfg.HttpClient == nil { impl.cfg.HttpClient = DefaultHttpRequestFunc @@ -102,65 +105,57 @@ type downloader struct { cancel context.CancelCauseFunc cfg Downloader - params *HttpRequestParams //http request params - chunkChannel chan chunk //chunk chanel + params *HttpRequestParams //http request params + chunkCh chan chunk //chunk chanel //wg sync.WaitGroup - m sync.Mutex + mu sync.Mutex nextChunk int //next chunk id - bufs []*Buf + bufMap map[int]*buffer.PipeBuffer written int64 //total bytes of file downloaded from remote - err error concurrency int //剩余的并发数，递减。到0时停止并发 - maxPart int //有多少个分片 pos int64 maxPos int64 - m2 sync.Mutex - readingID int // 正在被读取的id + delayMu sync.Mutex + readingID int64 // 正在被读取的id + + hc *mem.HybridCache } type ConcurrencyLimit struct { - _m sync.Mutex - Limit int // 需要大于0 + mu sync.Mutex + + Limit uint32 } var ErrExceedMaxConcurrency = HttpStatusCodeError(http.StatusTooManyRequests) -func (l *ConcurrencyLimit) sub() error { - l._m.Lock() - defer l._m.Unlock() - if l.Limit-1 < 0 { +func (l *ConcurrencyLimit) Acquire() error { + if l == nil { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + if l.Limit == 0 { return ErrExceedMaxConcurrency } l.Limit-- - // log.Debugf("ConcurrencyLimit.sub: %d", l.Limit) - return nil -} -func (l *ConcurrencyLimit) add() { - l._m.Lock() - defer l._m.Unlock() - l.Limit++ - // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) -} - -// 检测是否超过限制 -func (d *downloader) concurrencyCheck() error { - if d.cfg.ConcurrencyLimit != nil { - return d.cfg.ConcurrencyLimit.sub() - } return nil } -func (d *downloader) concurrencyFinish() { - if d.cfg.ConcurrencyLimit != nil { - d.cfg.ConcurrencyLimit.add() +func (l *ConcurrencyLimit) Release() { + if l == nil { + return } + l.mu.Lock() + l.Limit++ + l.mu.Unlock() } // download performs the implementation of the object download across ranged GETs. func (d *downloader) download() (io.ReadCloser, error) { - if err := d.concurrencyCheck(); err != nil { + if err := d.cfg.ConcurrencyLimit.Acquire(); err != nil { return nil, err } @@ -176,16 +171,16 @@ func (d *downloader) download() (io.ReadCloser, error) { if maxPart == 1 { resp, err := d.cfg.HttpClient(d.ctx, d.params) if err != nil { - d.concurrencyFinish() + d.cfg.ConcurrencyLimit.Release() return nil, err } closeFunc := resp.Body.Close resp.Body = utils.NewReadCloser(resp.Body, func() error { - d.m.Lock() - defer d.m.Unlock() + d.mu.Lock() + defer d.mu.Unlock() var err error if closeFunc != nil { - d.concurrencyFinish() + d.cfg.ConcurrencyLimit.Release() err = closeFunc() closeFunc = nil } @@ -196,103 +191,142 @@ func (d *downloader) download() (io.ReadCloser, error) { d.ctx, d.cancel = context.WithCancelCause(d.ctx) // workers - d.chunkChannel = make(chan chunk, d.cfg.Concurrency) + d.chunkCh = make(chan chunk, d.cfg.Concurrency) - d.maxPart = maxPart d.pos = d.params.Range.Start d.maxPos = d.params.Range.Start + d.params.Range.Length d.concurrency = d.cfg.Concurrency - _ = d.sendChunkTask(true) - var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) + var err error + if d.params.Range.Length > int64(conf.CacheThreshold) { + d.hc, err = mem.NewHybridCache(uint64(d.cfg.PartSize), uint64(d.params.Range.Length)) + } + if err == nil { + d.bufMap = make(map[int]*buffer.PipeBuffer, d.cfg.Concurrency) + err = d.sendChunkTask(true) + } + if err != nil { + d.cancel(err) + d.cfg.ConcurrencyLimit.Release() + return nil, d.interrupt() + } - // Return error - return rc, d.err + d.mu.Lock() + defer d.mu.Unlock() + return &multiReadCloser{d: d, curBuf: d.popBuf(0), maxPos: maxPart}, nil } -func (d *downloader) sendChunkTask(newConcurrency bool) error { - d.m.Lock() - defer d.m.Unlock() - isNewBuf := d.concurrency > 0 +func (d *downloader) sendChunkTask(newConcurrency bool) (err error) { + d.mu.Lock() + defer d.mu.Unlock() + if d.pos >= d.maxPos { + return nil + } if newConcurrency { if d.concurrency <= 0 { return nil } if d.nextChunk > 0 { // 第一个不检查，因为已经检查过了 - if err := d.concurrencyCheck(); err != nil { + if err := d.cfg.ConcurrencyLimit.Acquire(); err != nil { return err } + defer func() { + if err != nil { + d.cfg.ConcurrencyLimit.Release() + } + }() } - d.concurrency-- - go d.downloadPart() } - var buf *Buf - if isNewBuf { - buf = NewBuf(d.ctx, d.cfg.PartSize) - d.bufs = append(d.bufs, buf) - } else { - buf = d.getBuf(d.nextChunk) - } - - if d.pos < d.maxPos { - finalSize := int64(d.cfg.PartSize) - switch d.nextChunk { - case 0: - // 最小分片在前面有助视频播放？ - firstSize := d.params.Range.Length % finalSize - if firstSize > 0 { - minSize := finalSize / 2 - if firstSize < minSize { // 最小分片太小就调整到一半 - finalSize = minSize - } else { - finalSize = firstSize - } + br := d.bufMap[d.nextChunk] + if br == nil { + var b buffer.Block + if d.hc != nil { + b, err = d.hc.NextBlock() + if err != nil { + return err } - case 1: - firstSize := d.params.Range.Length % finalSize + } else { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic in creating new byte block: %v", r) + } + }() + b = buffer.NewByteBlock(make([]byte, d.cfg.PartSize)) + } + br = buffer.NewPipeBuffer(d.ctx, b) + d.bufMap[d.nextChunk] = br + } + + finalSize := int64(d.cfg.PartSize) + switch d.nextChunk { + case 0: + // 最小分片在前面有助视频播放？ + firstSize := d.params.Range.Length % finalSize + if firstSize > 0 { minSize := finalSize / 2 - if firstSize > 0 && firstSize < minSize { - finalSize += firstSize - minSize - } + // 最小分片太小就调整到一半 + finalSize = max(firstSize, minSize) } - err := buf.Reset(int(finalSize)) - if err != nil { - return err + case 1: + firstSize := d.params.Range.Length % finalSize + minSize := finalSize / 2 + if firstSize > 0 && firstSize < minSize { + finalSize += firstSize - minSize } - ch := chunk{ - start: d.pos, - size: finalSize, - id: d.nextChunk, - buf: buf, + } + err = br.Reset(int(finalSize)) + if err != nil { + return err // 分片算法错误或者下载中断 + } + if newConcurrency { + go d.downloadPart() + d.concurrency-- + } + ch := chunk{ + start: d.pos, + size: finalSize, + id: d.nextChunk, + buf: br, - newConcurrency: newConcurrency, - } - d.pos += finalSize - d.nextChunk++ - d.chunkChannel <- ch + newConcurrency: newConcurrency, + } + d.pos += finalSize + d.nextChunk++ + select { + case <-d.ctx.Done(): + return context.Cause(d.ctx) + case d.chunkCh <- ch: return nil } - return nil } // when the final reader Close, we interrupt func (d *downloader) interrupt() error { - d.m.Lock() - defer d.m.Unlock() - err := d.err - if err == nil && d.written != d.params.Range.Length { - log.Debugf("Downloader interrupt before finish") - err := fmt.Errorf("interrupted") - d.err = err - } - close(d.chunkChannel) - if d.bufs != nil { - d.cancel(err) - for _, buf := range d.bufs { - buf.Close() - } - d.bufs = nil + err := context.Cause(d.ctx) + if err == nil { + if atomic.LoadInt64(&d.written) != d.params.Range.Length { + err = fmt.Errorf("interrupted") + } + } else if errors.Is(err, context.Canceled) { + err = nil + } + d.cancel(err) + d.mu.Lock() + defer d.mu.Unlock() + if d.bufMap != nil { + for _, buf := range d.bufMap { + _ = buf.Close() + } + d.bufMap = nil + } + if d.hc != nil { + _ = d.hc.Close() + d.hc = nil + } + if d.maxPos != 0 { + d.maxPos = 0 + close(d.chunkCh) if d.concurrency > 0 { d.concurrency = -d.concurrency } @@ -300,49 +334,49 @@ func (d *downloader) interrupt() error { } return err } -func (d *downloader) getBuf(id int) (b *Buf) { - return d.bufs[id%len(d.bufs)] +func (d *downloader) popBuf(id int) *buffer.PipeBuffer { + br := d.bufMap[id] + if br != nil { + delete(d.bufMap, id) + d.bufMap[-1] = br // -1 保存最后一次取出的buf用来关闭 + } + return br } -func (d *downloader) finishBuf(id int) (isLast bool, nextBuf *Buf) { - id++ - if id >= d.maxPart { - return true, nil + +func (d *downloader) finishBuf(nextId int, prev *buffer.PipeBuffer) (next *buffer.PipeBuffer) { + atomic.StoreInt64(&d.readingID, int64(nextId)) + + d.mu.Lock() + shouldSendTask := d.bufMap[d.nextChunk] == nil + if shouldSendTask { + d.bufMap[d.nextChunk] = prev } + d.mu.Unlock() - _ = d.sendChunkTask(false) + if shouldSendTask { + _ = d.sendChunkTask(false) + } else { + _ = prev.Close() + } - d.readingID = id - return false, d.getBuf(id) + d.mu.Lock() + defer d.mu.Unlock() + return d.popBuf(nextId) } // downloadPart is an individual goroutine worker reading from the ch channel // and performing Http request on the data with a given byte range. func (d *downloader) downloadPart() { - defer d.concurrencyFinish() + defer d.cfg.ConcurrencyLimit.Release() for { select { case <-d.ctx.Done(): return - case c, ok := <-d.chunkChannel: + case c, ok := <-d.chunkCh: if !ok { return } - if d.getErr() != nil { - // Drain the channel if there is an error, to prevent deadlocking - // of download producer. - return - } - if err := d.downloadChunk(&c); err != nil { - if err == errCancelConcurrency { - return - } - if err == context.Canceled { - if e := context.Cause(d.ctx); e != nil { - err = e - } - } - d.setErr(err) - d.cancel(err) + if !d.downloadChunk(&c) { return } } @@ -350,26 +384,20 @@ func (d *downloader) downloadPart() { } // downloadChunk downloads the chunk -func (d *downloader) downloadChunk(ch *chunk) error { +func (d *downloader) downloadChunk(ch *chunk) bool { log.Debugf("start chunk_%d, %+v", ch.id, ch) params := d.getParamsFromChunk(ch) - var n int64 var err error for retry := 0; retry <= d.cfg.PartBodyMaxRetries; retry++ { - if d.getErr() != nil { - return nil - } + var n int64 n, err = d.tryDownloadChunk(params, ch) if err == nil { d.incrWritten(n) log.Debugf("chunk_%d downloaded", ch.id) - break - } - if d.getErr() != nil { - return nil + return true } - if utils.IsCanceled(d.ctx) { - return d.ctx.Err() + if d.ctx.Err() != nil { + return false } // Check if the returned error is an errNeedRetry. // If this occurs we unwrap the err to set the underlying error @@ -389,17 +417,31 @@ func (d *downloader) downloadChunk(ch *chunk) error { ch.id, params.URL, retry, err) } else if err == errInfiniteRetry { retry-- - continue + } else if err == errCancelConcurrency { + return false // 取消一个的并发 } else { break } } + if err != nil { + d.cancel(err) // 取消所有的并发 + } + return false +} - return err +func (d *downloader) delay(ti time.Duration) bool { + t := time.NewTimer(ti) + select { + case <-d.ctx.Done(): + t.Stop() + return false + case <-t.C: + return true + } } -var errCancelConcurrency = errors.New("cancel concurrency") -var errInfiniteRetry = errors.New("infinite retry") +var errCancelConcurrency = errors.New("") +var errInfiniteRetry = errors.New("") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) @@ -420,15 +462,17 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int case http.StatusServiceUnavailable: case http.StatusGatewayTimeout: } - <-time.After(time.Millisecond * 200) - return 0, &errNeedRetry{err: err} + if !d.delay(time.Millisecond * time.Duration(rand.Uint32N(300)+200)) { + return 0, errCancelConcurrency + } + return 0, &errNeedRetry{err} } // 来到这 说明第1个分片下载 连接成功了 // 后续分片下载出错都当超载处理 log.Debugf("err chunk_%d, try downloading:%v", ch.id, err) - d.m.Lock() + d.mu.Lock() isCancelConcurrency := ch.newConcurrency if d.concurrency > 0 { // 取消剩余的并发任务 // 用于计算实际的并发数 @@ -437,18 +481,25 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int } if isCancelConcurrency { d.concurrency-- - d.chunkChannel <- *ch - d.m.Unlock() - return 0, errCancelConcurrency + d.mu.Unlock() + select { + case <-d.ctx.Done(): + return 0, errCancelConcurrency + case d.chunkCh <- *ch: + return 0, errCancelConcurrency + } } - d.m.Unlock() - if ch.id != d.readingID { //正在被读取的优先重试 - d.m2.Lock() - defer d.m2.Unlock() - <-time.After(time.Millisecond * 200) + d.mu.Unlock() + if int64(ch.id) != atomic.LoadInt64(&d.readingID) { //正在被读取的优先重试 + d.delayMu.Lock() + defer d.delayMu.Unlock() + if !d.delay(time.Millisecond * time.Duration(rand.Uint32N(300)+200)) { + return 0, errCancelConcurrency + } } return 0, errInfiniteRetry } + defer resp.Body.Close() //only check file size on the first task if ch.id == 0 { @@ -461,11 +512,11 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { - return n, &errNeedRetry{err: err} + return n, &errNeedRetry{err} } if n != ch.size { err = fmt.Errorf("chunk download size incorrect, expected=%d, got=%d", ch.size, n) - return n, &errNeedRetry{err: err} + return n, &errNeedRetry{err} } return n, nil @@ -497,7 +548,8 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { totalStr := stdpath.Base(contentRange) if totalStr != "*" { - if total, err := strconv.ParseInt(totalStr, 10, 64); err != nil { + var total int64 + if total, err = strconv.ParseInt(totalStr, 10, 64); err != nil { err = fmt.Errorf("failed extracting file size: %s", totalStr) } else { totalBytes = total @@ -510,35 +562,12 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { if totalBytes != d.params.Size && err == nil { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } - if err != nil { - d.setErr(err) - d.cancel(err) - } return err } func (d *downloader) incrWritten(n int64) { - d.m.Lock() - defer d.m.Unlock() - - d.written += n -} - -// getErr is a thread-safe getter for the error object -func (d *downloader) getErr() error { - d.m.Lock() - defer d.m.Unlock() - - return d.err -} - -// setErr is a thread-safe setter for the error object -func (d *downloader) setErr(e error) { - d.m.Lock() - defer d.m.Unlock() - - d.err = e + atomic.AddInt64(&d.written, n) } // Chunk represents a single chunk of data to write by the worker routine. @@ -548,7 +577,7 @@ func (d *downloader) setErr(e error) { type chunk struct { start int64 size int64 - buf *Buf + buf *buffer.PipeBuffer id int newConcurrency bool @@ -587,196 +616,39 @@ type HttpRequestParams struct { Size int64 } type errNeedRetry struct { - err error -} - -func (e *errNeedRetry) Error() string { - return e.err.Error() + error } func (e *errNeedRetry) Unwrap() error { - return e.err + return e.error } -type MultiReadCloser struct { - cfg *cfg - closer closerFunc - finish finishBufFUnc +type multiReadCloser struct { + pos int //current reader position, start from 0 + maxPos int + curBuf *buffer.PipeBuffer + d *downloader } -type cfg struct { - rPos int //current reader position, start from 0 - curBuf *Buf -} - -type closerFunc func() error -type finishBufFUnc func(id int) (isLast bool, buf *Buf) - -// NewMultiReadCloser to save memory, we re-use limited Buf, and feed data to Read() -func NewMultiReadCloser(buf *Buf, c closerFunc, fb finishBufFUnc) *MultiReadCloser { - return &MultiReadCloser{closer: c, finish: fb, cfg: &cfg{curBuf: buf}} -} - -func (mr MultiReadCloser) Read(p []byte) (n int, err error) { - if mr.cfg.curBuf == nil { +func (mr *multiReadCloser) Read(p []byte) (n int, err error) { + if mr.curBuf == nil { return 0, io.EOF } - n, err = mr.cfg.curBuf.Read(p) - //log.Debugf("read_%d read current buffer, n=%d ,err=%+v", mr.cfg.rPos, n, err) + n, err = mr.curBuf.Read(p) + // log.Debugf("read_%d read current buffer, n=%d ,err=%+v", mr.rPos, n, err) if err == io.EOF { - log.Debugf("read_%d finished current buffer", mr.cfg.rPos) + log.Debugf("read_%d finished current buffer", mr.pos) - isLast, next := mr.finish(mr.cfg.rPos) - if isLast { + mr.pos++ + if mr.pos >= mr.maxPos { return n, io.EOF } - mr.cfg.curBuf = next - mr.cfg.rPos++ + mr.curBuf = mr.d.finishBuf(mr.pos, mr.curBuf) return n, nil } - if err == context.Canceled { - if e := context.Cause(mr.cfg.curBuf.ctx); e != nil { - err = e - } - } return n, err } -func (mr MultiReadCloser) Close() error { - return mr.closer() -} -type Buf struct { - size int //expected size - ctx context.Context - offR int - offW int - rw sync.Mutex - buf []byte - mmap bool - - readSignal chan struct{} - readPending bool -} - -// NewBuf is a buffer that can have 1 read & 1 write at the same time. -// when read is faster write, immediately feed data to read after written -func NewBuf(ctx context.Context, maxSize int) *Buf { - br := &Buf{ - ctx: ctx, - size: maxSize, - readSignal: make(chan struct{}, 1), - } - if conf.MmapThreshold > 0 && maxSize >= conf.MmapThreshold { - m, err := mmap.Alloc(maxSize) - if err == nil { - br.buf = m - br.mmap = true - return br - } - } - br.buf = make([]byte, maxSize) - return br -} - -func (br *Buf) Reset(size int) error { - br.rw.Lock() - defer br.rw.Unlock() - if br.buf == nil { - return io.ErrClosedPipe - } - if size > cap(br.buf) { - return fmt.Errorf("reset size %d exceeds max size %d", size, cap(br.buf)) - } - br.size = size - br.offR = 0 - br.offW = 0 - return nil -} - -func (br *Buf) Read(p []byte) (int, error) { - if err := br.ctx.Err(); err != nil { - return 0, err - } - if len(p) == 0 { - return 0, nil - } - if br.offR >= br.size { - return 0, io.EOF - } - for { - br.rw.Lock() - if br.buf == nil { - br.rw.Unlock() - return 0, io.ErrClosedPipe - } - - if br.offW < br.offR { - br.rw.Unlock() - return 0, io.ErrUnexpectedEOF - } - if br.offW == br.offR { - br.readPending = true - br.rw.Unlock() - select { - case <-br.ctx.Done(): - return 0, br.ctx.Err() - case _, ok := <-br.readSignal: - if !ok { - return 0, io.ErrClosedPipe - } - continue - } - } - - n := copy(p, br.buf[br.offR:br.offW]) - br.offR += n - br.rw.Unlock() - if n < len(p) && br.offR >= br.size { - return n, io.EOF - } - return n, nil - } -} - -func (br *Buf) Write(p []byte) (int, error) { - if err := br.ctx.Err(); err != nil { - return 0, err - } - if len(p) == 0 { - return 0, nil - } - br.rw.Lock() - defer br.rw.Unlock() - if br.buf == nil { - return 0, io.ErrClosedPipe - } - if br.offW >= br.size { - return 0, io.ErrShortWrite - } - n := copy(br.buf[br.offW:], p[:min(br.size-br.offW, len(p))]) - br.offW += n - if br.readPending { - br.readPending = false - select { - case br.readSignal <- struct{}{}: - default: - } - } - if n < len(p) { - return n, io.ErrShortWrite - } - return n, nil -} - -func (br *Buf) Close() error { - br.rw.Lock() - defer br.rw.Unlock() - var err error - if br.mmap { - err = mmap.Free(br.buf) - br.mmap = false - } - br.buf = nil - close(br.readSignal) - return err +func (mr *multiReadCloser) Close() error { + return mr.d.interrupt() } diff --git a/internal/net/request_test.go b/internal/net/request_test.go index da16a31658..0fdc56eb33 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -11,13 +11,12 @@ import ( "net/http" "sync" "testing" + "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/sirupsen/logrus" ) -var buf22MB = make([]byte, 1024*1024*22) - func containsString(slice []string, val string) bool { for _, item := range slice { if item == val { @@ -27,18 +26,6 @@ func containsString(slice []string, val string) bool { return false } -func dummyHttpRequest(data []byte, p http_range.Range) io.ReadCloser { - - end := p.Start + p.Length - 1 - - if end >= int64(len(data)) { - end = int64(len(data)) - } - - bodyBytes := data[p.Start:end] - return io.NopCloser(bytes.NewReader(bodyBytes)) -} - func TestDownloadOrder(t *testing.T) { buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} downloader, invocations, ranges := newDownloadRangeClient(buff) @@ -67,8 +54,8 @@ func TestDownloadOrder(t *testing.T) { if err != nil { t.Fatalf("expect no error, got %v", err) } - if exp, a := int(length), len(resultBuf); exp != a { - t.Errorf("expect buffer length=%d, got %d", exp, a) + if exp, a := buff[start:start+length2], resultBuf; !bytes.Equal(exp, a) { + t.Errorf("expect buffer %v, got %v", exp, a) } chunkSize := int(length+int64(partSize)-1) / partSize if e, a := chunkSize, *invocations; e != a { @@ -84,7 +71,100 @@ func TestDownloadOrder(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } + +func TestDownloadInterrupt(t *testing.T) { + buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + buff = append(buff, buff...) + downloader, _, _ := newDownloadRangeClient(buff) + con, partSize := 6, 3 + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: 5, + } + }) + + var start, length int64 = 0, int64(len(buff)) + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + ctx, cancel := context.WithCancel(context.Background()) + readCloser, err := d.Download(ctx, req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + _, err = io.CopyN(io.Discard, readCloser, 8) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + cancel() + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } +} + +func TestHighConcurrency(t *testing.T) { + buff := make([]byte, 8<<10) + for i := range len(buff) { + buff[i] = byte(i % 256) + } + downloader, invocations, _ := newDownloadRangeClient(buff) + con, partSize := 64, 100 + concurrencyLimit := uint32(32) + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: concurrencyLimit, + } + }) + + var start, length int64 = 2, 7 << 10 + length2 := length + if length2 == -1 { + length2 = int64(len(buff)) - start + } + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + readCloser, err := d.Download(context.Background(), req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + resultBuf, err := io.ReadAll(readCloser) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + if !bytes.Equal(buff[start:start+length2], resultBuf) { + t.Error("expect buffer content matches, but got mismatch") + } + chunkSize := int(length+int64(partSize)-1) / partSize + if e, a := chunkSize, *invocations; e != a { + t.Errorf("expect %v API calls, got %v", e, a) + } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } + for range 100 { + time.Sleep(10 * time.Millisecond) + if d.ConcurrencyLimit.Limit == concurrencyLimit { + return + } + } + t.Errorf("expect concurrency limit to be %v, got %v", concurrencyLimit, d.ConcurrencyLimit.Limit) +} + func init() { Formatter := new(logrus.TextFormatter) Formatter.TimestampFormat = "2006-01-02T15:04:05.999999999" @@ -136,6 +216,9 @@ func TestDownloadSingle(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } type downloadCaptureClient struct { diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100e..e88e033913 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -1,6 +1,7 @@ package stream import ( + "bytes" "context" "errors" "fmt" @@ -10,11 +11,11 @@ import ( "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/mem" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" "go4.org/readerutil" ) @@ -28,8 +29,9 @@ type FileStream struct { Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers size int64 - peekBuff *buffer.Reader oriReader io.Reader // the original reader, used for caching + hc *mem.HybridCache + peek buffer.SizedReadAtSeeker } func (f *FileStream) GetSize() int64 { @@ -51,15 +53,6 @@ func (f *FileStream) IsForceStreamUpload() bool { return f.ForceStreamUpload } -func (f *FileStream) Close() error { - if f.peekBuff != nil { - f.peekBuff.Reset() - f.oriReader = nil - f.peekBuff = nil - } - return f.Closers.Close() -} - func (f *FileStream) GetExist() model.Obj { return f.Exist } @@ -101,79 +94,57 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } reader := f.Reader - if f.peekBuff != nil { - f.peekBuff.Seek(0, io.SeekStart) + if f.peek != nil { + f.peek.Seek(0, io.SeekStart) if writer != nil { - _, err := utils.CopyWithBuffer(writer, f.peekBuff) + _, err := utils.CopyWithBuffer(writer, f.peek) if err != nil { return nil, err } - f.peekBuff.Seek(0, io.SeekStart) + f.peek.Seek(0, io.SeekStart) } reader = f.oriReader } if writer != nil { reader = io.TeeReader(reader, writer) } + + // 如果文件大小未知，直接缓存到磁盘 if f.GetSize() < 0 { - if f.peekBuff == nil { - f.peekBuff = &buffer.Reader{} - } // 检查是否有数据 buf := []byte{0} n, err := io.ReadFull(reader, buf) - if n > 0 { - f.peekBuff.Append(buf[:n]) - } - if err == io.ErrUnexpectedEOF { - f.size = f.peekBuff.Size() - f.Reader = f.peekBuff - return f.peekBuff, nil + br := bytes.NewReader(buf[:n]) + if err == io.ErrUnexpectedEOF || err == io.EOF { + f.size = br.Size() + f.Reader = br + return br, nil } else if err != nil { return nil, err } - if conf.MaxBufferLimit-n > conf.MmapThreshold && conf.MmapThreshold > 0 { - m, err := mmap.Alloc(conf.MaxBufferLimit - n) - if err == nil { - f.Add(utils.CloseFunc(func() error { - return mmap.Free(m) - })) - n, err = io.ReadFull(reader, m) - if n > 0 { - f.peekBuff.Append(m[:n]) - } - if err == io.ErrUnexpectedEOF { - f.size = f.peekBuff.Size() - f.Reader = f.peekBuff - return f.peekBuff, nil - } else if err != nil { - return nil, err - } - } - } - tmpF, err := utils.CreateTempFile(reader, 0) + tmpF, err := utils.CreateTempFile(io.MultiReader(br, reader), 0) if err != nil { return nil, err } f.Add(utils.CloseFunc(func() error { return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) })) - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) + stat, err := tmpF.Stat() if err != nil { return nil, err } - f.size = peekF.Size() - f.Reader = peekF - return peekF, nil + f.size = stat.Size() + f.Reader = tmpF + return tmpF, nil } if up != nil { cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) *up = model.UpdateProgressWithRange(*up, 50, 100) size := f.GetSize() - if f.peekBuff != nil { - peekSize := f.peekBuff.Size() - cacheProgress(float64(peekSize) / float64(size) * 100) + if f.peek != nil { + peekSize := f.peek.Size() + // cacheProgress(float64(peekSize) / float64(size) * 100) size -= peekSize } reader = &ReaderUpdatingProgress{ @@ -185,7 +156,7 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } } - if f.peekBuff != nil { + if f.oriReader != nil { f.oriReader = reader } else { f.Reader = reader @@ -225,63 +196,49 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 确保指定大小的数据被缓存 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { - if maxCacheSize > int64(conf.MaxBufferLimit) { - size := f.GetSize() - reader := f.Reader - if f.peekBuff != nil { - size -= f.peekBuff.Size() - reader = f.oriReader - } - tmpF, err := utils.CreateTempFile(reader, size) - if err != nil { - return nil, err - } - f.Add(utils.CloseFunc(func() error { - return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) - })) - if f.peekBuff != nil { - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) + if f.peek == nil { + f.oriReader = f.Reader + if f.GetSize() > int64(conf.CacheThreshold) { + blockSize := min(f.GetSize(), int64(conf.MaxBlockLimit)) + hc, err := mem.NewHybridCache(uint64(blockSize), uint64(f.GetSize())) if err != nil { return nil, err } - f.Reader = peekF - return peekF, nil + f.hc = hc + f.peek = buffer.NewDynamicReadAtSeeker(hc) + f.Reader = io.MultiReader(f.peek, f.oriReader) + f.Add(hc) + } else { + br := &buffer.Reader{} + f.peek = br + f.Reader = io.MultiReader(br, f.oriReader) + f.Add(br) + } + } + cacheSize := maxCacheSize - f.peek.Size() + if cacheSize <= 0 { + return f.peek, nil + } + if f.hc != nil { + written, err := f.hc.CopyFromN(f.oriReader, cacheSize) + if written != cacheSize { + f.hc.RollbackBlockWithSize(uint64(cacheSize - written)) + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", cacheSize, written, err) } - f.Reader = tmpF - return tmpF, nil - } - - if f.peekBuff == nil { - f.peekBuff = &buffer.Reader{} - f.oriReader = f.Reader - f.Reader = io.MultiReader(f.peekBuff, f.oriReader) - } - bufSize := maxCacheSize - f.peekBuff.Size() - if bufSize <= 0 { - return f.peekBuff, nil - } - var buf []byte - if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) { - m, err := mmap.Alloc(int(bufSize)) - if err == nil { - f.Add(utils.CloseFunc(func() error { - return mmap.Free(m) - })) - buf = m + } else { + buf := make([]byte, cacheSize) + n, err := io.ReadFull(f.oriReader, buf) + if n > 0 { + f.peek.(*buffer.Reader).Append(buf[:n]) + } + if n != len(buf) { + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", len(buf), n, err) } } - if buf == nil { - buf = make([]byte, bufSize) - } - n, err := io.ReadFull(f.oriReader, buf) - if bufSize != int64(n) { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err) - } - f.peekBuff.Append(buf) - if f.peekBuff.Size() >= f.GetSize() { - f.Reader = f.peekBuff + if f.peek.Size() >= f.GetSize() { + f.Reader = f.peek } - return f.peekBuff, nil + return f.peek, nil } var _ model.FileStreamer = (*SeekableStream)(nil) diff --git a/internal/stream/stream_test.go b/internal/stream/stream_test.go index 9a81e7d413..5c2fa03ee3 100644 --- a/internal/stream/stream_test.go +++ b/internal/stream/stream_test.go @@ -1,4 +1,4 @@ -package stream +package stream_test import ( "bytes" @@ -7,27 +7,37 @@ import ( "io" "testing" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) -func TestFileStream_RangeRead(t *testing.T) { +func TestRangeRead(t *testing.T) { type args struct { httpRange http_range.Range } buf := []byte("github.com/OpenListTeam/OpenList") - f := &FileStream{ + f := &stream.FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } + prevCacheThreshold := conf.CacheThreshold + prevMaxBlockLimit := conf.MaxBlockLimit + t.Cleanup(func() { + conf.CacheThreshold = prevCacheThreshold + conf.MaxBlockLimit = prevMaxBlockLimit + }) + conf.CacheThreshold = 10 + conf.MaxBlockLimit = 15 tests := []struct { name string - f *FileStream + f *stream.FileStream args args - want func(f *FileStream, got io.Reader, err error) error + want func(f *stream.FileStream, got io.Reader, err error) error }{ { name: "range 11-12", @@ -35,7 +45,7 @@ func TestFileStream_RangeRead(t *testing.T) { args: args{ httpRange: http_range.Range{Start: 11, Length: 12}, }, - want: func(f *FileStream, got io.Reader, err error) error { + want: func(f *stream.FileStream, got io.Reader, err error) error { if f.GetFile() != nil { return errors.New("cached") } @@ -52,7 +62,7 @@ func TestFileStream_RangeRead(t *testing.T) { args: args{ httpRange: http_range.Range{Start: 11, Length: 21}, }, - want: func(f *FileStream, got io.Reader, err error) error { + want: func(f *stream.FileStream, got io.Reader, err error) error { if f.GetFile() == nil { return errors.New("not cached") } @@ -84,14 +94,22 @@ func TestFileStream_RangeRead(t *testing.T) { } } -func TestFileStream_With_PreHash(t *testing.T) { +func TestPreHash(t *testing.T) { buf := []byte("github.com/OpenListTeam/OpenList") - f := &FileStream{ + f := &stream.FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } + prevCacheThreshold := conf.CacheThreshold + prevMaxBlockLimit := conf.MaxBlockLimit + t.Cleanup(func() { + conf.CacheThreshold = prevCacheThreshold + conf.MaxBlockLimit = prevMaxBlockLimit + }) + conf.CacheThreshold = 10 + conf.MaxBlockLimit = 15 const hashSize int64 = 20 reader, _ := f.RangeRead(http_range.Range{Start: 0, Length: hashSize}) @@ -99,7 +117,7 @@ func TestFileStream_With_PreHash(t *testing.T) { if preHash == "" { t.Error("preHash is empty") } - tmpF, fullHash, _ := CacheFullAndHash(f, nil, utils.SHA1) + tmpF, fullHash, _ := stream.CacheFullAndHash(f, nil, utils.SHA1) fmt.Println(fullHash) fileFullHash, _ := utils.HashFile(utils.SHA1, tmpF) fmt.Println(fileFullHash) @@ -107,3 +125,61 @@ func TestFileStream_With_PreHash(t *testing.T) { t.Errorf("fullHash and fileFullHash should match: fullHash=%s fileFullHash=%s", fullHash, fileFullHash) } } + +func TestStreamSectionReader(t *testing.T) { + buf := make([]byte, 8<<10) + for i := range len(buf) { + buf[i] = byte(i % 256) + } + f := &stream.FileStream{ + Obj: &model.Object{ + Size: int64(len(buf)), + }, + Reader: io.NopCloser(bytes.NewReader(buf)), + } + prevCacheThreshold := conf.CacheThreshold + prevMaxBlockLimit := conf.MaxBlockLimit + prevConf := conf.Conf + t.Cleanup(func() { + conf.CacheThreshold = prevCacheThreshold + conf.MaxBlockLimit = prevMaxBlockLimit + conf.Conf = prevConf + }) + conf.CacheThreshold = 1 + conf.MaxBlockLimit = 2 << 10 + partSize := 3 << 10 + conf.Conf = &conf.Config{} + ss, err := stream.NewStreamSectionReader(f, partSize, nil) + if err != nil { + t.Errorf("NewStreamSectionReader() error = %v", err) + } + for i := 0; i < len(buf); i += partSize { + length := partSize + if i+length > len(buf) { + length = len(buf) - i + } + rs, err := ss.GetSectionReader(int64(i), int64(length)) + if err != nil { + t.Errorf("StreamSectionReader.GetSectionReader() error = %v", err) + } + b1, err := io.ReadAll(rs) + if err != nil { + t.Errorf("StreamSectionReader.Read() error = %v", err) + } + rs.Seek(1, io.SeekStart) + b2, _ := io.ReadAll(rs) + if !bytes.Equal(b1[1:], b2) { + t.Errorf("StreamSectionReader.Read() = %s, want %s", b1[1:], b2) + } + if !bytes.Equal(buf[i:i+length], b1) { + t.Errorf("StreamSectionReader.Read() = %s, want %s", b1, buf[i:i+length]) + } + if i == 0 { + prevMinFreeMemory := conf.MinFreeMemory + conf.MinFreeMemory = 0 // 强制使用文件缓存 + t.Cleanup(func() { + conf.MinFreeMemory = prevMinFreeMemory + }) + } + } +} diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d6..8db4b7ae6d 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -8,16 +8,17 @@ import ( "fmt" "io" "net/http" - "os" + "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/mem" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/pool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/rclone/rclone/lib/mmap" log "github.com/sirupsen/logrus" ) @@ -174,79 +175,39 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } -type StreamSectionReaderIF interface { +type StreamSectionReader interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) + // 线程安全 FreeSectionReader(sr io.ReadSeeker) // 线程不安全 DiscardSection(off int64, length int64) error } -func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (StreamSectionReaderIF, error) { +func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (StreamSectionReader, error) { if file.GetFile() != nil { return &cachedSectionReader{file.GetFile()}, nil } maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if maxBufferSize > conf.MaxBufferLimit { - f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - - if f.Truncate(file.GetSize()) != nil { - // fallback to full cache - _, _ = f.Close(), os.Remove(f.Name()) - cache, err := file.CacheFullAndWriter(up, nil) - if err != nil { - return nil, err - } - return &cachedSectionReader{cache}, nil - } - - ss := &fileSectionReader{file: file, temp: f} - ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ - New: func() *offsetWriterWithBase { - base := ss.tempOffset - ss.tempOffset += int64(maxBufferSize) - return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} - }, - } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) - })) - return ss, nil - } - - ss := &directSectionReader{file: file} - if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { - ss.bufPool = &pool.Pool[[]byte]{ - New: func() []byte { - buf, err := mmap.Alloc(maxBufferSize) - if err == nil { - file.Add(utils.CloseFunc(func() error { - return mmap.Free(buf) - })) - } else { - buf = make([]byte, maxBufferSize) - } - return buf - }, - } - } else { - ss.bufPool = &pool.Pool[[]byte]{ + if file.GetSize() <= int64(conf.CacheThreshold) { + bufPool := &pool.Pool[[]byte]{ New: func() []byte { return make([]byte, maxBufferSize) }, } + file.Add(bufPool) + ss := &byteSectionReader{file: file, bufPool: bufPool} + return ss, nil } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return nil - })) - return ss, nil + blockSize := min(uint64(maxBufferSize), conf.MaxBlockLimit) + hc, err := mem.NewHybridCache(blockSize, uint64(file.GetSize())) + if err != nil { + return nil, err + } + file.Add(hc) + return &hybridSectionReader{file: file, hc: hc}, nil } type cachedSectionReader struct { @@ -261,21 +222,14 @@ func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker } func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} -type fileSectionReader struct { +type byteSectionReader struct { file model.FileStreamer fileOffset int64 - temp *os.File - tempOffset int64 - bufPool *pool.Pool[*offsetWriterWithBase] -} - -type offsetWriterWithBase struct { - *io.OffsetWriter - base int64 + bufPool *pool.Pool[[]byte] } // 线程不安全 -func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { +func (ss *byteSectionReader) DiscardSection(off int64, length int64) error { if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -287,42 +241,43 @@ func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { return nil } -type fileBufferSectionReader struct { +type bytesRefReadSeeker struct { io.ReadSeeker - fileBuf *offsetWriterWithBase + buf []byte } // 线程不安全 -func (ss *fileSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { +func (ss *byteSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - fileBuf := ss.bufPool.Get() - _, _ = fileBuf.Seek(0, io.SeekStart) - n, err := utils.CopyWithBufferN(fileBuf, ss.file, length) - ss.fileOffset += n - if err != nil { + tempBuf := ss.bufPool.Get() + buf := tempBuf[:length] + n, err := io.ReadFull(ss.file, buf) + ss.fileOffset += int64(n) + if int64(n) != length { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } - return &fileBufferSectionReader{io.NewSectionReader(ss.temp, fileBuf.base, length), fileBuf}, nil + return &bytesRefReadSeeker{bytes.NewReader(buf), buf}, nil } - -func (ss *fileSectionReader) FreeSectionReader(rs io.ReadSeeker) { - if sr, ok := rs.(*fileBufferSectionReader); ok { - ss.bufPool.Put(sr.fileBuf) - sr.fileBuf = nil +func (ss *byteSectionReader) FreeSectionReader(rs io.ReadSeeker) { + if sr, ok := rs.(*bytesRefReadSeeker); ok { + ss.bufPool.Put(sr.buf[0:cap(sr.buf)]) + sr.buf = nil sr.ReadSeeker = nil } } -type directSectionReader struct { +type hybridSectionReader struct { file model.FileStreamer fileOffset int64 - bufPool *pool.Pool[[]byte] + hc *mem.HybridCache + mu sync.Mutex + cache []buffer.Block } // 线程不安全 -func (ss *directSectionReader) DiscardSection(off int64, length int64) error { +func (ss *hybridSectionReader) DiscardSection(off int64, length int64) error { if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -334,29 +289,70 @@ func (ss *directSectionReader) DiscardSection(off int64, length int64) error { return nil } -type bufferSectionReader struct { +type blockRefReadSeeker struct { io.ReadSeeker - buf []byte + b buffer.Block } // 线程不安全 -func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { +func (ss *hybridSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - tempBuf := ss.bufPool.Get() - buf := tempBuf[:length] - n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + b := ss.get() + if b == nil { + offset := int64(ss.hc.Size()) + written, err := ss.hc.CopyFromN(ss.file, length) + ss.fileOffset += written + if written != length { + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, written, err) + } + b = buffer.NewBlockAdapter( + io.NewOffsetWriter(ss.hc, offset), + io.NewSectionReader(ss.hc, offset, length), + ) + } else { + ws := buffer.WriteAtSeekerOf(b) + if _, err := ws.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("failed to reset cached block writer: %w", err) + } + written, err := utils.CopyWithBufferN(ws, ss.file, length) + ss.fileOffset += written + if written != length { + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, written, err) + } } - return &bufferSectionReader{bytes.NewReader(buf), buf}, nil + + if length == b.Size() { + rs := buffer.ReadAtSeekerOf(b) + if _, err := rs.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("failed to reset cached block reader: %w", err) + } + return &blockRefReadSeeker{rs, b}, nil + } + return &blockRefReadSeeker{io.NewSectionReader(b, 0, length), b}, nil } -func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { - if sr, ok := rs.(*bufferSectionReader); ok { - ss.bufPool.Put(sr.buf[0:cap(sr.buf)]) - sr.buf = nil + +func (ss *hybridSectionReader) get() buffer.Block { + ss.mu.Lock() + defer ss.mu.Unlock() + if len(ss.cache) > 0 { + b := ss.cache[len(ss.cache)-1] + ss.cache = ss.cache[:len(ss.cache)-1] + return b + } + return nil +} +func (ss *hybridSectionReader) put(b buffer.Block) { + ss.mu.Lock() + defer ss.mu.Unlock() + ss.cache = append(ss.cache, b) +} + +func (ss *hybridSectionReader) FreeSectionReader(rs io.ReadSeeker) { + if sr, ok := rs.(*blockRefReadSeeker); ok { + ss.put(sr.b) + sr.b = nil sr.ReadSeeker = nil } } diff --git a/pkg/buffer/bytes.go b/pkg/buffer/bytes.go index 3e6cb54054..ca71e8f24b 100644 --- a/pkg/buffer/bytes.go +++ b/pkg/buffer/bytes.go @@ -30,29 +30,25 @@ func (r *Reader) Read(p []byte) (int, error) { } func (r *Reader) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } if off < 0 || off >= r.size { return 0, io.EOF } n := 0 - readFrom := false for _, buf := range r.bufs { - if readFrom { - nn := copy(p[n:], buf) - n += nn - if n == len(p) { - return n, nil - } - } else if newOff := off - int64(len(buf)); newOff >= 0 { - off = newOff - } else { - nn := copy(p, buf[off:]) - if nn == len(p) { - return nn, nil - } - n += nn - readFrom = true + if off >= int64(len(buf)) { + off -= int64(len(buf)) + continue + } + nn := copy(p[n:], buf[off:]) + n += nn + if n == len(p) { + return n, nil } + off = 0 } return n, io.EOF @@ -84,6 +80,11 @@ func (r *Reader) Reset() { r.offset = 0 } +func (r *Reader) Close() error { + r.Reset() + return nil +} + func NewReader(buf ...[]byte) *Reader { b := &Reader{ bufs: make([][]byte, 0, len(buf)), @@ -93,3 +94,39 @@ func NewReader(buf ...[]byte) *Reader { } return b } + +type byteBlock struct { + buf []byte +} + +func NewByteBlock(buf []byte) Block { + return &byteBlock{buf: buf} +} + +func (b *byteBlock) Size() int64 { + return int64(len(b.buf)) +} + +func (b *byteBlock) ReadAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.EOF + } + n = copy(p, b.buf[off:]) + if n < len(p) { + err = io.EOF + } + return +} + +func (b *byteBlock) WriteAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.ErrShortWrite + } + n = copy(b.buf[off:], p) + if n < len(p) { + err = io.ErrShortWrite + } + return +} + +var _ Block = (*byteBlock)(nil) diff --git a/pkg/buffer/bytes_test.go b/pkg/buffer/bytes_test.go index 3f4d855639..73b546f0c4 100644 --- a/pkg/buffer/bytes_test.go +++ b/pkg/buffer/bytes_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func TestReader_ReadAt(t *testing.T) { +func TestReadAt(t *testing.T) { type args struct { p []byte off int64 @@ -17,10 +17,10 @@ func TestReader_ReadAt(t *testing.T) { bs.Append([]byte("Team/")) bs.Append([]byte("OpenList")) tests := []struct { - name string - b *Reader - args args - want func(a args, n int, err error) error + name string + b *Reader + args args + check func(a args, n int, err error) error }{ { name: "readAt len 10 offset 0", @@ -29,7 +29,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 10), off: 0, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } @@ -49,7 +49,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 12), off: 11, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } @@ -69,7 +69,7 @@ func TestReader_ReadAt(t *testing.T) { p: make([]byte, 50), off: 24, }, - want: func(a args, n int, err error) error { + check: func(a args, n int, err error) error { if n != int(bs.Size()-a.off) { return errors.New("read length not match") } @@ -86,7 +86,7 @@ func TestReader_ReadAt(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := tt.b.ReadAt(tt.args.p, tt.args.off) - if err := tt.want(tt.args, got, err); err != nil { + if err := tt.check(tt.args, got, err); err != nil { t.Errorf("Bytes.ReadAt() error = %v", err) } }) diff --git a/pkg/buffer/pipe.go b/pkg/buffer/pipe.go new file mode 100644 index 0000000000..194fd58cf4 --- /dev/null +++ b/pkg/buffer/pipe.go @@ -0,0 +1,157 @@ +package buffer + +import ( + "context" + "fmt" + "io" + "sync" +) + +type PipeBuffer struct { + limit int //expected size + ctx context.Context + offR int + offW int + rw sync.Mutex + block Block + + readSignal chan struct{} + readPending bool +} + +// NewPipeBuffer is a buffer that can have 1 read & 1 write at the same time. +// when read is faster write, immediately feed data to read after written +func NewPipeBuffer(ctx context.Context, block Block) *PipeBuffer { + br := &PipeBuffer{ + ctx: ctx, + limit: int(block.Size()), + readSignal: make(chan struct{}, 1), + block: block, + } + return br +} + +func (br *PipeBuffer) Read(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + if br.offR >= br.limit { + return 0, io.EOF + } + + for { + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + if br.offW == br.offR { + br.readPending = true + br.rw.Unlock() + select { + case <-br.ctx.Done(): + return 0, br.ctx.Err() + case _, ok := <-br.readSignal: + if !ok { + return 0, io.ErrClosedPipe + } + continue + } + } + break + } + + canRead := br.offW - br.offR + if canRead < 0 { + br.rw.Unlock() + return 0, io.ErrUnexpectedEOF + } + + off := br.offR + block := br.block + br.rw.Unlock() + + n, err := block.ReadAt(p[:min(len(p), canRead)], int64(off)) + + br.rw.Lock() + br.offR += n + br.rw.Unlock() + + if n < len(p) && br.offR >= br.limit { + return n, io.EOF + } + return n, err +} + +func (br *PipeBuffer) Write(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + canWrite := br.limit - br.offW + if canWrite <= 0 { + br.rw.Unlock() + return 0, io.ErrShortWrite + } + + off := br.offW + block := br.block + br.rw.Unlock() + + n, err := block.WriteAt(p[:min(canWrite, len(p))], int64(off)) + + br.rw.Lock() + br.offW += n + if br.readPending { + br.readPending = false + select { + case br.readSignal <- struct{}{}: + default: + } + } + br.rw.Unlock() + + if n < len(p) && err == nil { + return n, io.ErrShortWrite + } + return n, err +} + +func (br *PipeBuffer) Reset(limit int) error { + br.rw.Lock() + defer br.rw.Unlock() + if br.block == nil { + return io.ErrClosedPipe + } + if int64(limit) > br.block.Size() { + return fmt.Errorf("reset limit %d exceeds max size %d", limit, br.block.Size()) + } + br.limit = limit + br.offR = 0 + br.offW = 0 + return nil +} + +func (br *PipeBuffer) Close() error { + br.rw.Lock() + defer br.rw.Unlock() + if br.block != nil { + br.block = nil + br.readPending = false + close(br.readSignal) + } + return nil +} diff --git a/pkg/buffer/type.go b/pkg/buffer/type.go new file mode 100644 index 0000000000..e4d7132168 --- /dev/null +++ b/pkg/buffer/type.go @@ -0,0 +1,22 @@ +package buffer + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type Block interface { + io.ReaderAt + io.WriterAt + Size() int64 +} + +type WriteAtSeeker = model.FileWriter + +type ReadAtSeeker = model.File + +type SizedReadAtSeeker interface { + ReadAtSeeker + Size() int64 +} diff --git a/pkg/buffer/utils.go b/pkg/buffer/utils.go new file mode 100644 index 0000000000..a1bde567a6 --- /dev/null +++ b/pkg/buffer/utils.go @@ -0,0 +1,97 @@ +package buffer + +import ( + "errors" + "io" +) + +type WriteAtSeekerProvider interface{ GetWriteAtSeeker() WriteAtSeeker } + +func WriteAtSeekerOf(b Block) WriteAtSeeker { + if p, ok := b.(WriteAtSeekerProvider); ok { + return p.GetWriteAtSeeker() + } + return io.NewOffsetWriter(b, 0) +} + +type ReadAtSeekerProvider interface{ GetReadAtSeeker() ReadAtSeeker } + +// 将一个Block包装为ReadAtSeeker。 +// 固定大小：当前Block的Size()。 +func ReadAtSeekerOf(b Block) ReadAtSeeker { + if p, ok := b.(ReadAtSeekerProvider); ok { + return p.GetReadAtSeeker() + } + return io.NewSectionReader(b, 0, b.Size()) +} + +type blockAdapter struct { + WriteAtSeeker + SizedReadAtSeeker +} + +func (b *blockAdapter) GetWriteAtSeeker() WriteAtSeeker { + return b.WriteAtSeeker +} + +func (b *blockAdapter) GetReadAtSeeker() ReadAtSeeker { + return b.SizedReadAtSeeker +} +func NewBlockAdapter(w WriteAtSeeker, r SizedReadAtSeeker) Block { + return &blockAdapter{ + WriteAtSeeker: w, + SizedReadAtSeeker: r, + } +} + +var _ Block = (*blockAdapter)(nil) + +// 将一个Block包装为ReadAtSeeker。 +// 动态大小：Size() 是动态跟随底层 Block。 +type DynamicReadAtSeeker struct { + block Block + offset int64 +} + +func (r *DynamicReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { + return r.block.ReadAt(p, off) +} + +func (r *DynamicReadAtSeeker) Read(p []byte) (n int, err error) { + n, err = r.block.ReadAt(p, r.offset) + if n > 0 { + r.offset += int64(n) + } + return n, err +} + +func (r *DynamicReadAtSeeker) Size() int64 { + return r.block.Size() +} + +func (r *DynamicReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + if offset == 0 { + return r.offset, nil + } + offset = r.offset + offset + case io.SeekEnd: + offset = r.block.Size() + offset + default: + return 0, errors.New("Seek: invalid whence") + } + + if offset < 0 || offset > r.block.Size() { + return 0, errors.New("Seek: invalid offset") + } + r.offset = offset + return offset, nil +} + +func NewDynamicReadAtSeeker(block Block) *DynamicReadAtSeeker { + return &DynamicReadAtSeeker{ + block: block, + } +} diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index ce92cd1fc3..01cd736d32 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -3,9 +3,7 @@ package pool import "sync" type Pool[T any] struct { - New func() T - MaxCap int - + New func() T cache []T mu sync.Mutex } @@ -24,9 +22,7 @@ func (p *Pool[T]) Get() T { func (p *Pool[T]) Put(item T) { p.mu.Lock() defer p.mu.Unlock() - if p.MaxCap == 0 || len(p.cache) < int(p.MaxCap) { - p.cache = append(p.cache, item) - } + p.cache = append(p.cache, item) } func (p *Pool[T]) Reset() { @@ -35,3 +31,8 @@ func (p *Pool[T]) Reset() { clear(p.cache) p.cache = nil } + +func (p *Pool[T]) Close() error { + p.Reset() + return nil +} From ea19bed247e767c998cd7e62d35032394e065579 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Thu, 14 May 2026 23:23:44 +0800 Subject: [PATCH 004/107] fix(setting): handle delete of setting item with empty key (#2131) --- internal/bootstrap/data/setting.go | 7 +++++++ internal/db/settingitem.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index d7fd8ea478..4526aa5234 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -34,6 +34,13 @@ func initSettings() { } settingMap := map[string]*model.SettingItem{} for _, v := range settings { + if v.Key == "" { + err := db.DeleteSettingItemByKey(v.Key) + if err != nil { + utils.Log.Errorf("failed delete setting with empty key: %+v", err) + } + continue + } if !isActive(v.Key) && v.Flag != model.DEPRECATED { v.Flag = model.DEPRECATED err = op.SaveSettingItem(&v) diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index f20e507f07..0d42aca6d5 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -65,5 +65,5 @@ func SaveSettingItem(item *model.SettingItem) error { } func DeleteSettingItemByKey(key string) error { - return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error) + return errors.WithStack(db.Where(fmt.Sprintf("%s = ?", columnName("key")), key).Delete(model.SettingItem{}).Error) } From 219564b56ca662b06df807f927758039be261ccb Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Fri, 15 May 2026 17:59:53 +0800 Subject: [PATCH 005/107] refactor(func): replace GinWithValue with GinAppendValues (#2475) --- server/common/common.go | 17 ++++++++++++++--- server/handles/archive.go | 4 ++-- server/handles/fsbatch.go | 6 +++--- server/handles/fsmanage.go | 2 +- server/handles/fsread.go | 8 ++++---- server/middlewares/auth.go | 12 ++++++------ server/middlewares/check.go | 2 +- server/middlewares/down.go | 4 ++-- server/middlewares/sharing.go | 4 ++-- server/webdav.go | 16 ++++++++-------- 10 files changed, 43 insertions(+), 32 deletions(-) diff --git a/server/common/common.go b/server/common/common.go index d780512684..a1a9a6319e 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -128,13 +128,24 @@ func Pluralize(count int, singular, plural string) string { return plural } -func GinWithValue(c *gin.Context, keyAndValue ...any) { +type requestContext struct { + context.Context +} + +// GinAppendValues 向当前请求上下文追加键值，提供类似 gin.Context Set/Get 的可变语义。 +// 同一请求内，已持有的上下文引用会同步看到后续更新。 +func GinAppendValues(c *gin.Context, keyAndValue ...any) { + ctx := c.Request.Context() + if r, ok := ctx.(*requestContext); ok { + r.Context = ContentWithValues(r.Context, keyAndValue...) + return + } c.Request = c.Request.WithContext( - ContentWithValue(c.Request.Context(), keyAndValue...), + &requestContext{ContentWithValues(ctx, keyAndValue...)}, ) } -func ContentWithValue(ctx context.Context, keyAndValue ...any) context.Context { +func ContentWithValues(ctx context.Context, keyAndValue ...any) context.Context { if len(keyAndValue) < 1 || len(keyAndValue)%2 != 0 { panic("keyAndValue must be an even number of arguments (key, value, ...)") } diff --git a/server/handles/archive.go b/server/handles/archive.go index d46f83c868..364e93edc5 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -105,7 +105,7 @@ func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -188,7 +188,7 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 28588d6683..e672165cd8 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -49,7 +49,7 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, srcMeta) + common.GinAppendValues(c, conf.MetaKey, srcMeta) dstDir, err := user.JoinPath(req.DstDir) if err != nil { @@ -183,7 +183,7 @@ func FsBatchRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { if renameObject.SrcName == "" || renameObject.NewName == "" { continue @@ -236,7 +236,7 @@ func FsRegexRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) srcRegexp, err := regexp.Compile(req.SrcNameRegex) if err != nil { diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 94e7c3fec5..52bdb20bf1 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -425,7 +425,7 @@ func FsRemoveEmptyDirectory(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index a90fc1082b..856f46e543 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -88,7 +88,7 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -152,7 +152,7 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -291,7 +291,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -415,7 +415,7 @@ func FsOther(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 0fc243617f..ca67e4a6dd 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -24,7 +24,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -41,7 +41,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -69,7 +69,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } @@ -84,7 +84,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -96,7 +96,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -124,7 +124,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } diff --git a/server/middlewares/check.go b/server/middlewares/check.go index c7203a4907..c5e07874cd 100644 --- a/server/middlewares/check.go +++ b/server/middlewares/check.go @@ -29,7 +29,7 @@ func StoragesLoaded(c *gin.Context) { return } } - common.GinWithValue(c, + common.GinAppendValues(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), ) c.Next() diff --git a/server/middlewares/down.go b/server/middlewares/down.go index c1f81b54b3..d71be00b36 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -17,7 +17,7 @@ import ( func PathParse(c *gin.Context) { rawPath := parsePath(c.Param("path")) - common.GinWithValue(c, conf.PathKey, rawPath) + common.GinAppendValues(c, conf.PathKey, rawPath) c.Next() } @@ -29,7 +29,7 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { common.ErrorPage(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) // verify sign if needSign(meta, rawPath) { s := c.Query("sign") diff --git a/server/middlewares/sharing.go b/server/middlewares/sharing.go index d7549202f3..aa0cab0c63 100644 --- a/server/middlewares/sharing.go +++ b/server/middlewares/sharing.go @@ -8,11 +8,11 @@ import ( func SharingIdParse(c *gin.Context) { sid := c.Param("sid") - common.GinWithValue(c, conf.SharingIDKey, sid) + common.GinAppendValues(c, conf.SharingIDKey, sid) c.Next() } func EmptyPathParse(c *gin.Context) { - common.GinWithValue(c, conf.PathKey, "/") + common.GinAppendValues(c, conf.PathKey, "/") c.Next() } diff --git a/server/webdav.go b/server/webdav.go index a949068f01..74523b0b30 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -54,7 +54,7 @@ func WebDAVAuth(c *gin.Context) { count, cok := model.LoginCache.Get(ip) if cok && count >= model.DefaultMaxAuthRetries { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -78,13 +78,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) c.Next() return } } if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -96,7 +96,7 @@ func WebDAVAuth(c *gin.Context) { user, ok := tryLogin(username, password) if !ok { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -109,7 +109,7 @@ func WebDAVAuth(c *gin.Context) { model.LoginCache.Del(ip) if user.Disabled || !user.CanWebdavRead() { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -142,11 +142,11 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) if user.IsGuest() { - common.GinWithValue(c, conf.MetaPassKey, password) + common.GinAppendValues(c, conf.MetaPassKey, password) } else { - common.GinWithValue(c, conf.MetaPassKey, "") + common.GinAppendValues(c, conf.MetaPassKey, "") } c.Next() } From daad21ef917ca4383cf865138df476490d157e5f Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 17 May 2026 22:31:41 +0800 Subject: [PATCH 006/107] fix(driver/139): remove RFC-incompatible request header (#2478) fix(139): remove RFC8441-incompatible Connection header Agent-Logs-Url: https://github.com/OpenListTeam/OpenList/sessions/1c6b226d-02c4-4b43-80ad-5ab2861d30c3 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> --- drivers/139/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/139/util.go b/drivers/139/util.go index 85c798cc7c..c026de52a2 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1180,7 +1180,6 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", "Content-Type": "text/plain;charset=UTF-8", "Host": "user-njs.yun.139.com", - "Connection": "Keep-Alive", "Accept-Encoding": "gzip", "User-Agent": "okhttp/3.12.2", } From 9eae62581a7b1cc8f90d8231e4dc7d1a21a00b71 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 18 May 2026 15:11:39 +0800 Subject: [PATCH 007/107] refactor(cache)!: extract hybrid_cache package (#2477) * refactor(HybridCache)!: extract hybrid_cache package and rename cache_threshold to auto_memory_limit * split HybridCache and stores into internal/hybrid_cache package * introduce BackingStore abstraction with BufferStore and FileStore * rename CacheThreshold to AutoMemoryLimit in config/env/bootstrap * update stream/request integration and related tests * rename singleFileCache and MultiFileCache to singleFileStore and MultiFileStore * refactor(HybridCache): improve memory management strategies in NewHybridCache * rename * alias hybrid_cache to hcache * omments --- internal/bootstrap/config.go | 8 +- internal/conf/config.go | 4 +- internal/conf/var.go | 12 +- internal/hybrid_cache/buffer.go | 96 +++++++ .../hybrid_cache/buffer_test.go | 20 +- internal/{cache => hybrid_cache}/file.go | 35 +-- internal/{cache => hybrid_cache}/file_test.go | 10 +- internal/hybrid_cache/hybrid_cache.go | 240 ++++++++++++++++++ internal/hybrid_cache/type.go | 13 + internal/mem/cache.go | 205 --------------- internal/net/request.go | 23 +- internal/stream/stream.go | 58 ++--- internal/stream/stream_test.go | 18 +- internal/stream/util.go | 70 +---- pkg/buffer/buffer.go | 41 +++ pkg/buffer/bytes.go | 132 ---------- pkg/buffer/file.go | 88 ------- pkg/buffer/type.go | 2 + pkg/buffer/utils.go | 4 - 19 files changed, 476 insertions(+), 603 deletions(-) create mode 100644 internal/hybrid_cache/buffer.go rename pkg/buffer/bytes_test.go => internal/hybrid_cache/buffer_test.go (81%) rename internal/{cache => hybrid_cache}/file.go (76%) rename internal/{cache => hybrid_cache}/file_test.go (92%) create mode 100644 internal/hybrid_cache/hybrid_cache.go create mode 100644 internal/hybrid_cache/type.go delete mode 100644 internal/mem/cache.go create mode 100644 pkg/buffer/buffer.go delete mode 100644 pkg/buffer/bytes.go delete mode 100644 pkg/buffer/file.go diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 4e44d079bd..8304468080 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -131,12 +131,12 @@ func InitConfig() { log.Warn("failed to get memory info, disable memory cache") } - if conf.Conf.CacheThreshold > 0 { - conf.CacheThreshold = uint64(conf.Conf.CacheThreshold) << 20 + if conf.Conf.AutoMemoryLimit > 0 { + conf.AutoMemoryLimit = uint64(conf.Conf.AutoMemoryLimit) << 20 } else { - conf.CacheThreshold = 0 + conf.AutoMemoryLimit = 0 } - log.Infof("cache threshold: %dMB", conf.CacheThreshold>>20) + log.Infof("auto memory limit: %dMB", conf.AutoMemoryLimit>>20) if len(conf.Conf.Log.Filter.Filters) == 0 { conf.Conf.Log.Filter.Enable = false diff --git a/internal/conf/config.go b/internal/conf/config.go index a26ddc306f..4a11728158 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -120,9 +120,9 @@ type Config struct { DistDir string `json:"dist_dir"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` + AutoMemoryLimit int `json:"auto_memory_limit" env:"AUTO_MEMORY_LIMIT"` MinFreeMemory int `json:"min_free_memory" env:"MIN_FREE_MEMORY"` MaxBlockLimit int `json:"max_block_limit" env:"MAX_BLOCK_LIMIT"` - CacheThreshold int `json:"cache_threshold" env:"CACHE_THRESHOLD"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` @@ -179,7 +179,7 @@ func DefaultConfig(dataDir string) *Config { }, }, }, - CacheThreshold: 4, + AutoMemoryLimit: 4, MaxConnections: 0, MaxConcurrency: 64, TlsInsecureSkipVerify: false, diff --git a/internal/conf/var.go b/internal/conf/var.go index 0ca954dc34..6b25bcfb5b 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -25,16 +25,16 @@ var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( - // 限制单次内存的扩容大小，超过该阈值将分多次扩容。 - // CacheThreshold大于0时，也限制 Downloader 的PartSize - MaxBlockLimit uint64 = 16 * 1024 * 1024 - // 大于该阈值的数据流将使用HybridCache，可主动释放内存。 - // 否则使用Go的内存分配，直到GC回收。 - CacheThreshold uint64 = 4 * 1024 * 1024 + // 在HybridCache中使用[]byte缓存数据流的限制，内存为Go自动管理，直到GC + AutoMemoryLimit uint64 = 4 * 1024 * 1024 // 最小空闲内存，当内存不足时，HybridCache会回退到文件缓存。 // 如果为0，HybridCache会使用文件缓存，不占用内存。 MinFreeMemory uint64 = 16 * 1024 * 1024 + // 限制HybridCache手动管理内存单次的扩容大小，超过该阈值将分多次扩容。 + // MinFreeMemory大于0时，也限制 Downloader 的PartSize + MaxBlockLimit uint64 = 16 * 1024 * 1024 ) + var ( RawIndexHtml string ManageHtml string diff --git a/internal/hybrid_cache/buffer.go b/internal/hybrid_cache/buffer.go new file mode 100644 index 0000000000..0023996e63 --- /dev/null +++ b/internal/hybrid_cache/buffer.go @@ -0,0 +1,96 @@ +package hybrid_cache + +import ( + "fmt" + "io" +) + +type BufferStore struct { + blocks [][]byte + size int64 +} + +func (m *BufferStore) Size() int64 { + return m.size +} + +// 用于存储不复用的[]byte +func (m *BufferStore) Append(buf []byte) { + m.size += int64(len(buf)) + m.blocks = append(m.blocks, buf) +} + +func (m *BufferStore) Close() error { + if len(m.blocks) > 0 { + clear(m.blocks) + m.blocks = m.blocks[:0] + m.size = 0 + } + return nil +} + +func (m *BufferStore) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + var n int + for _, buf := range m.blocks { + if off >= int64(len(buf)) { + off -= int64(len(buf)) + continue + } + nn := copy(p[n:], buf[off:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *BufferStore) WriteAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + var n int + for _, b := range m.blocks { + if off >= int64(len(b)) { + off -= int64(len(b)) + continue + } + nn := copy(b[off:], p[n:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func (m *BufferStore) GrowTo(size int64) (err error) { + if size <= m.size { + return nil + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("recovered in %v", r) + } + }() + m.blocks = append(m.blocks, make([]byte, size-m.size)) + m.size = size + return nil +} + +var _ BackingStore = (*BufferStore)(nil) diff --git a/pkg/buffer/bytes_test.go b/internal/hybrid_cache/buffer_test.go similarity index 81% rename from pkg/buffer/bytes_test.go rename to internal/hybrid_cache/buffer_test.go index 73b546f0c4..439d7cb78e 100644 --- a/pkg/buffer/bytes_test.go +++ b/internal/hybrid_cache/buffer_test.go @@ -1,24 +1,30 @@ -package buffer +package hybrid_cache_test import ( "errors" "io" "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" ) -func TestReadAt(t *testing.T) { +func TestBufferStore(t *testing.T) { type args struct { p []byte off int64 } - bs := &Reader{} + bs := &hybrid_cache.BufferStore{} bs.Append([]byte("github.com")) bs.Append([]byte("/OpenList")) - bs.Append([]byte("Team/")) - bs.Append([]byte("OpenList")) + bs.Append([]byte("Team/?")) + b := []byte("OpenList") + off := bs.Size() - 1 + _ = bs.GrowTo(off + int64(len(b))) + _, _ = bs.WriteAt(b, off) + tests := []struct { name string - b *Reader + b *hybrid_cache.BufferStore args args check func(a args, n int, err error) error }{ @@ -87,7 +93,7 @@ func TestReadAt(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := tt.b.ReadAt(tt.args.p, tt.args.off) if err := tt.check(tt.args, got, err); err != nil { - t.Errorf("Bytes.ReadAt() error = %v", err) + t.Errorf("BufferStore.ReadAt() error = %v", err) } }) } diff --git a/internal/cache/file.go b/internal/hybrid_cache/file.go similarity index 76% rename from internal/cache/file.go rename to internal/hybrid_cache/file.go index 64e88fb844..bca197dadb 100644 --- a/internal/cache/file.go +++ b/internal/hybrid_cache/file.go @@ -1,4 +1,4 @@ -package cache +package hybrid_cache import ( "errors" @@ -6,25 +6,18 @@ import ( "os" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/pkg/buffer" ) -type FileCache interface { - buffer.Block - io.Closer - Truncate(size int64) error -} - -type singleFileCache struct { +type singleFileStore struct { *os.File size int64 } -func (s *singleFileCache) Size() int64 { +func (s *singleFileStore) Size() int64 { return s.size } -func (s *singleFileCache) Truncate(size int64) error { +func (s *singleFileStore) GrowTo(size int64) error { if size <= s.size { return nil } @@ -35,7 +28,7 @@ func (s *singleFileCache) Truncate(size int64) error { return err } -func (s *singleFileCache) Close() error { +func (s *singleFileStore) Close() error { err := s.File.Close() _ = os.Remove(s.File.Name()) return err @@ -47,16 +40,16 @@ type fileBlock struct { written int64 } -type MultiFileCache struct { +type MultiFileStore struct { blocks []*fileBlock size int64 } -func (s *MultiFileCache) Size() int64 { +func (s *MultiFileStore) Size() int64 { return s.size } -func (m *MultiFileCache) Close() error { +func (m *MultiFileStore) Close() error { var errs []error for _, c := range m.blocks { if err := c.file.Close(); err != nil { @@ -69,7 +62,7 @@ func (m *MultiFileCache) Close() error { return errors.Join(errs...) } -func (m *MultiFileCache) Truncate(size int64) error { +func (m *MultiFileStore) GrowTo(size int64) error { if size <= m.size { return nil } @@ -82,7 +75,7 @@ func (m *MultiFileCache) Truncate(size int64) error { return nil } -func (m *MultiFileCache) ReadAt(p []byte, off int64) (n int, err error) { +func (m *MultiFileStore) ReadAt(p []byte, off int64) (n int, err error) { if len(p) == 0 { return 0, nil } @@ -131,7 +124,7 @@ func (m *MultiFileCache) ReadAt(p []byte, off int64) (n int, err error) { return n, io.EOF } -func (m *MultiFileCache) WriteAt(p []byte, off int64) (n int, err error) { +func (m *MultiFileStore) WriteAt(p []byte, off int64) (n int, err error) { if len(p) == 0 { return 0, nil } @@ -170,16 +163,16 @@ func (m *MultiFileCache) WriteAt(p []byte, off int64) (n int, err error) { return n, io.ErrShortWrite } -func NewFileCache(blockSize int64) (FileCache, error) { +func NewFileStore(blockSize int64) (BackingStore, error) { f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } err = f.Truncate(blockSize) if err == nil { - return &singleFileCache{File: f, size: blockSize}, nil + return &singleFileStore{File: f, size: blockSize}, nil } - return &MultiFileCache{ + return &MultiFileStore{ blocks: []*fileBlock{{file: f, size: blockSize}}, size: blockSize, }, nil diff --git a/internal/cache/file_test.go b/internal/hybrid_cache/file_test.go similarity index 92% rename from internal/cache/file_test.go rename to internal/hybrid_cache/file_test.go index 5f30c01e82..7f83625db8 100644 --- a/internal/cache/file_test.go +++ b/internal/hybrid_cache/file_test.go @@ -1,4 +1,4 @@ -package cache_test +package hybrid_cache_test import ( "bytes" @@ -8,8 +8,8 @@ import ( "reflect" "testing" - "github.com/OpenListTeam/OpenList/v4/internal/cache" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" ) func TestFile(t *testing.T) { @@ -59,7 +59,7 @@ func TestMultiFileCache(t *testing.T) { conf.Conf = prevConf }) conf.Conf = &conf.Config{} - f := cache.MultiFileCache{} + f := hybrid_cache.MultiFileStore{} defer f.Close() t.Run("ReadAt", func(t *testing.T) { _, err := f.ReadAt(make([]byte, 1), 20) @@ -68,7 +68,7 @@ func TestMultiFileCache(t *testing.T) { } }) t.Run("WriteAt", func(t *testing.T) { - err := f.Truncate(15) + err := f.GrowTo(15) if err != nil { t.Errorf("truncate err=%v", err) return @@ -79,7 +79,7 @@ func TestMultiFileCache(t *testing.T) { return } - err = f.Truncate(30) + err = f.GrowTo(30) if err != nil { t.Errorf("truncate err=%v", err) return diff --git a/internal/hybrid_cache/hybrid_cache.go b/internal/hybrid_cache/hybrid_cache.go new file mode 100644 index 0000000000..d80f768f18 --- /dev/null +++ b/internal/hybrid_cache/hybrid_cache.go @@ -0,0 +1,240 @@ +package hybrid_cache + +import ( + "errors" + "io" + "runtime" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/mem" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// 线程不安全，单线程使用，或者外部加锁保护 +type HybridCache struct { + blockSize uint64 + memoryStore mem.LinearMemory + memoryOffset uint64 + backingStore BackingStore + backingOffset uint64 +} + +// HybridCache本身是一个大的Block，支持分块成多个小的Block + +// 分配一个新的Block，支持读写，大小为size +func (hc *HybridCache) AllocBlock(size uint64) (buffer.Block, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + fs := buffer.NewBlockAdapter( + io.NewOffsetWriter(hc.backingStore, int64(base)), + io.NewSectionReader(hc.backingStore, int64(base), int64(size)), + ) + return fs, nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return buffer.NewByteBlock(all[start : start+size]), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) allocWriteAtSeeker(size uint64) (buffer.WriteAtSeeker, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + return io.NewOffsetWriter(hc.backingStore, int64(base)), nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return io.NewOffsetWriter(buffer.NewByteBlock(all[start:start+size]), 0), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) NextBlock() (buffer.Block, error) { + return hc.AllocBlock(hc.blockSize) +} + +func (hc *HybridCache) RewindBySize(size uint64) { + if hc.backingOffset >= size { + hc.backingOffset -= size + return + } + size -= hc.backingOffset + hc.backingOffset = 0 + if hc.memoryOffset >= size { + hc.memoryOffset -= size + return + } + size -= hc.memoryOffset + hc.memoryOffset = 0 +} + +func (hc *HybridCache) RewindOneBlock() { + hc.RewindBySize(hc.blockSize) +} + +func (hc *HybridCache) initFileCache() error { + file, err := NewFileStore(int64(hc.blockSize)) + if err != nil { + return err + } + hc.backingStore = file + return nil +} + +func (hc *HybridCache) Close() error { + var err error + if hc.memoryStore != nil { + err = hc.memoryStore.Free() + hc.memoryStore = nil + hc.memoryOffset = 0 + } + if hc.backingStore != nil { + err = errors.Join(err, hc.backingStore.Close()) + hc.backingStore = nil + hc.backingOffset = 0 + } + return err +} + +func (hc *HybridCache) Size() int64 { + return int64(hc.memoryOffset + hc.backingOffset) +} + +func (hc *HybridCache) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.EOF + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(p, all[off:]) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canRead := int64(hc.backingOffset) - off + if canRead <= 0 { + return n, io.EOF + } + nn, err := hc.backingStore.ReadAt(p[:min(len(p), int(canRead))], off) + return n + nn, err +} + +func (hc *HybridCache) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.ErrShortWrite + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(all[off:], p) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canWrite := int64(hc.backingOffset) - off + if canWrite <= 0 { + return n, io.ErrShortWrite + } + nn, err := hc.backingStore.WriteAt(p[:min(len(p), int(canWrite))], off) + return n + nn, err +} + +func (hc *HybridCache) CopyFromN(src io.Reader, n int64) (written int64, err error) { + limit := n + for limit > 0 { + blockSize := limit + if hc.backingStore == nil && blockSize > int64(conf.MaxBlockLimit) { + blockSize = int64(conf.MaxBlockLimit) + } + b, err := hc.allocWriteAtSeeker(uint64(blockSize)) + if err != nil { + return written, err + } + nn, err := utils.CopyWithBufferN(b, src, blockSize) + written += nn + if nn != blockSize { + return written, err + } + limit -= nn + } + return written, nil +} + +// HybridCache 线程不安全，单线程使用，或者外部加锁保护 +func NewHybridCache(blockSize, maxMemorySize uint64) (hc *HybridCache, err error) { + if conf.MinFreeMemory > 0 { + // 策略1: Go自动内存管理 + if maxMemorySize <= conf.AutoMemoryLimit { + return &HybridCache{backingStore: &BufferStore{}, blockSize: blockSize}, nil + } + + // 策略2: 手动内存管理 + if maxMemorySize >= blockSize { + var m mem.LinearMemory + // 手动管理内存，Uinx Mmap 或者 Windows VirtualAlloc + if m, err = mem.NewGuardedMemory(blockSize, maxMemorySize); err == nil { + hc = &HybridCache{memoryStore: m, blockSize: blockSize} + } + } + } + // 策略3: 文件后备 + if hc == nil { + hc = &HybridCache{blockSize: blockSize} + // 文件 + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + } + runtime.SetFinalizer(hc, func(hc *HybridCache) { + if hc.backingStore != nil { + _ = hc.backingStore.Close() + hc.backingStore = nil + } + }) + return hc, nil +} + +var _ buffer.Block = (*HybridCache)(nil) diff --git a/internal/hybrid_cache/type.go b/internal/hybrid_cache/type.go new file mode 100644 index 0000000000..edc6d3deb9 --- /dev/null +++ b/internal/hybrid_cache/type.go @@ -0,0 +1,13 @@ +package hybrid_cache + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" +) + +type BackingStore interface { + buffer.Block + io.Closer + GrowTo(size int64) error +} diff --git a/internal/mem/cache.go b/internal/mem/cache.go deleted file mode 100644 index e2c008875c..0000000000 --- a/internal/mem/cache.go +++ /dev/null @@ -1,205 +0,0 @@ -package mem - -import ( - "errors" - "io" - "runtime" - - "github.com/OpenListTeam/OpenList/v4/internal/cache" - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/pkg/buffer" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" -) - -// 优先使用内存，失败后才使用文件。 -// 线程不安全 -type HybridCache struct { - mem LinearMemory - memOffset uint64 - file cache.FileCache - fileOffset uint64 - blockSize uint64 -} - -func (hc *HybridCache) NextBlockWithSize(size uint64) (buffer.Block, error) { -retry: - if hc.file != nil { - if err := hc.file.Truncate(int64(hc.fileOffset + size)); err != nil { - return nil, err - } - base := hc.fileOffset - hc.fileOffset += size - fs := buffer.NewBlockAdapter( - io.NewOffsetWriter(hc.file, int64(base)), - io.NewSectionReader(hc.file, int64(base), int64(size)), - ) - return fs, nil - } - all, err := hc.mem.Reallocate(hc.memOffset + size) - if err == nil { - start := hc.memOffset - hc.memOffset += size - return buffer.NewByteBlock(all[start : start+size]), nil - } - if err := hc.initFileCache(); err != nil { - return nil, err - } - goto retry -} - -func (hc *HybridCache) NextBlock() (buffer.Block, error) { - return hc.NextBlockWithSize(hc.blockSize) -} - -func (hc *HybridCache) RollbackBlockWithSize(size uint64) { - if hc.fileOffset >= size { - hc.fileOffset -= size - return - } - size -= hc.fileOffset - hc.fileOffset = 0 - if hc.memOffset >= size { - hc.memOffset -= size - return - } - hc.memOffset = 0 -} - -func (hc *HybridCache) RollbackBlock() { - hc.RollbackBlockWithSize(hc.blockSize) -} - -func (hc *HybridCache) initFileCache() error { - file, err := cache.NewFileCache(int64(hc.blockSize)) - if err != nil { - return err - } - hc.file = file - return nil -} - -func (hc *HybridCache) Close() error { - var err error - if hc.mem != nil { - err = hc.mem.Free() - hc.mem = nil - } - if hc.file != nil { - err = errors.Join(err, hc.file.Close()) - hc.file = nil - } - return err -} - -func (hc *HybridCache) Size() int64 { - return int64(hc.memOffset + hc.fileOffset) -} - -func (hc *HybridCache) ReadAt(p []byte, off int64) (n int, err error) { - if len(p) == 0 { - return 0, nil - } - if off < 0 || off >= hc.Size() { - return 0, io.EOF - } - if off < int64(hc.memOffset) { - all, err := hc.mem.Reallocate(min(hc.memOffset, uint64(off)+uint64(len(p)))) - if err != nil { - // 不可能失败 - panic(err) - } - n = copy(p, all[off:]) - if n == len(p) { - return n, nil - } - p = p[n:] - } - - off += int64(n) - int64(hc.memOffset) - canRead := int64(hc.fileOffset) - off - if canRead <= 0 { - return n, io.EOF - } - nn, err := hc.file.ReadAt(p[:min(len(p), int(canRead))], off) - return n + nn, err -} - -func (hc *HybridCache) WriteAt(p []byte, off int64) (n int, err error) { - if len(p) == 0 { - return 0, nil - } - if off < 0 || off >= hc.Size() { - return 0, io.ErrShortWrite - } - - if off < int64(hc.memOffset) { - all, err := hc.mem.Reallocate(min(hc.memOffset, uint64(off)+uint64(len(p)))) - if err != nil { - // 不可能失败 - panic(err) - } - n = copy(all[off:], p) - if n == len(p) { - return n, nil - } - p = p[n:] - } - - off += int64(n) - int64(hc.memOffset) - canWrite := int64(hc.fileOffset) - off - if canWrite <= 0 { - return n, io.ErrShortWrite - } - nn, err := hc.file.WriteAt(p[:min(len(p), int(canWrite))], off) - return n + nn, err -} - -func (hc *HybridCache) CopyFromN(src io.Reader, n int64) (written int64, err error) { - limit := n - for limit > 0 { - blockSize := limit - if hc.file == nil && blockSize > int64(conf.MaxBlockLimit) { - blockSize = int64(conf.MaxBlockLimit) - } - b, err := hc.NextBlockWithSize(uint64(blockSize)) - if err != nil { - return written, err - } - nn, err := utils.CopyWithBufferN(buffer.WriteAtSeekerOf(b), src, blockSize) - written += nn - if nn != blockSize { - return written, err - } - limit -= nn - } - return written, nil -} - -// 优先使用内存，失败后才使用文件 -// 线程不安全 -func NewHybridCache(blockSize, maxMemorySize uint64) (*HybridCache, error) { - var err error - var hc *HybridCache - if conf.MinFreeMemory > 0 && maxMemorySize >= blockSize { - var m LinearMemory - m, err = NewGuardedMemory(blockSize, maxMemorySize) - if err == nil { - hc = &HybridCache{mem: m, blockSize: blockSize} - } - } - if hc == nil { - hc = &HybridCache{blockSize: blockSize} - if err2 := hc.initFileCache(); err2 != nil { - return nil, errors.Join(err, err2) - } - } - runtime.SetFinalizer(hc, func(hc *HybridCache) { - if hc.file != nil { - _ = hc.file.Close() - hc.file = nil - } - }) - return hc, nil -} - -var _ buffer.Block = (*HybridCache)(nil) diff --git a/internal/net/request.go b/internal/net/request.go index 22ae1f44bc..4fcdad0fd4 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -15,7 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" - "github.com/OpenListTeam/OpenList/v4/internal/mem" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -121,7 +121,7 @@ type downloader struct { delayMu sync.Mutex readingID int64 // 正在被读取的id - hc *mem.HybridCache + hc *hcache.HybridCache } type ConcurrencyLimit struct { @@ -198,9 +198,7 @@ func (d *downloader) download() (io.ReadCloser, error) { d.concurrency = d.cfg.Concurrency var err error - if d.params.Range.Length > int64(conf.CacheThreshold) { - d.hc, err = mem.NewHybridCache(uint64(d.cfg.PartSize), uint64(d.params.Range.Length)) - } + d.hc, err = hcache.NewHybridCache(uint64(d.cfg.PartSize), uint64(d.params.Range.Length)) if err == nil { d.bufMap = make(map[int]*buffer.PipeBuffer, d.cfg.Concurrency) err = d.sendChunkTask(true) @@ -241,18 +239,9 @@ func (d *downloader) sendChunkTask(newConcurrency bool) (err error) { br := d.bufMap[d.nextChunk] if br == nil { var b buffer.Block - if d.hc != nil { - b, err = d.hc.NextBlock() - if err != nil { - return err - } - } else { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic in creating new byte block: %v", r) - } - }() - b = buffer.NewByteBlock(make([]byte, d.cfg.PartSize)) + b, err = d.hc.NextBlock() + if err != nil { + return err } br = buffer.NewPipeBuffer(d.ctx, b) d.bufMap[d.nextChunk] = br diff --git a/internal/stream/stream.go b/internal/stream/stream.go index e88e033913..9ae8313f0c 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -11,7 +11,7 @@ import ( "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/mem" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -30,7 +30,7 @@ type FileStream struct { utils.Closers size int64 oriReader io.Reader // the original reader, used for caching - hc *mem.HybridCache + hc *hcache.HybridCache peek buffer.SizedReadAtSeeker } @@ -161,7 +161,7 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } else { f.Reader = reader } - return f.cache(f.GetSize()) + return f.ensureCache(f.GetSize()) } func (f *FileStream) GetFile() model.File { @@ -182,7 +182,7 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - cache, err := f.cache(httpRange.Start + httpRange.Length) + cache, err := f.ensureCache(httpRange.Start + httpRange.Length) if err != nil { return nil, err } @@ -195,45 +195,27 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 -func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { +func (f *FileStream) ensureCache(size int64) (model.File, error) { if f.peek == nil { - f.oriReader = f.Reader - if f.GetSize() > int64(conf.CacheThreshold) { - blockSize := min(f.GetSize(), int64(conf.MaxBlockLimit)) - hc, err := mem.NewHybridCache(uint64(blockSize), uint64(f.GetSize())) - if err != nil { - return nil, err - } - f.hc = hc - f.peek = buffer.NewDynamicReadAtSeeker(hc) - f.Reader = io.MultiReader(f.peek, f.oriReader) - f.Add(hc) - } else { - br := &buffer.Reader{} - f.peek = br - f.Reader = io.MultiReader(br, f.oriReader) - f.Add(br) + blockSize := min(size, f.GetSize(), int64(conf.MaxBlockLimit)) + var err error + f.hc, err = hcache.NewHybridCache(uint64(blockSize), uint64(f.GetSize())) + if err != nil { + return nil, err } + f.peek = buffer.NewDynamicReadAtSeeker(f.hc) + f.oriReader = f.Reader + f.Reader = io.MultiReader(f.peek, f.oriReader) + f.Add(f.hc) } - cacheSize := maxCacheSize - f.peek.Size() - if cacheSize <= 0 { + size = size - f.peek.Size() + if size <= 0 { return f.peek, nil } - if f.hc != nil { - written, err := f.hc.CopyFromN(f.oriReader, cacheSize) - if written != cacheSize { - f.hc.RollbackBlockWithSize(uint64(cacheSize - written)) - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", cacheSize, written, err) - } - } else { - buf := make([]byte, cacheSize) - n, err := io.ReadFull(f.oriReader, buf) - if n > 0 { - f.peek.(*buffer.Reader).Append(buf[:n]) - } - if n != len(buf) { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", len(buf), n, err) - } + written, err := f.hc.CopyFromN(f.oriReader, size) + if written != size { + f.hc.RewindBySize(uint64(size - written)) + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, written, err) } if f.peek.Size() >= f.GetSize() { f.Reader = f.peek diff --git a/internal/stream/stream_test.go b/internal/stream/stream_test.go index 5c2fa03ee3..1d8d002e2d 100644 --- a/internal/stream/stream_test.go +++ b/internal/stream/stream_test.go @@ -25,13 +25,13 @@ func TestRangeRead(t *testing.T) { }, Reader: io.NopCloser(bytes.NewReader(buf)), } - prevCacheThreshold := conf.CacheThreshold + prevAutoMemoryLimit := conf.AutoMemoryLimit prevMaxBlockLimit := conf.MaxBlockLimit t.Cleanup(func() { - conf.CacheThreshold = prevCacheThreshold + conf.AutoMemoryLimit = prevAutoMemoryLimit conf.MaxBlockLimit = prevMaxBlockLimit }) - conf.CacheThreshold = 10 + conf.AutoMemoryLimit = 0 conf.MaxBlockLimit = 15 tests := []struct { name string @@ -102,13 +102,13 @@ func TestPreHash(t *testing.T) { }, Reader: io.NopCloser(bytes.NewReader(buf)), } - prevCacheThreshold := conf.CacheThreshold + prevAutoMemoryLimit := conf.AutoMemoryLimit prevMaxBlockLimit := conf.MaxBlockLimit t.Cleanup(func() { - conf.CacheThreshold = prevCacheThreshold + conf.AutoMemoryLimit = prevAutoMemoryLimit conf.MaxBlockLimit = prevMaxBlockLimit }) - conf.CacheThreshold = 10 + conf.AutoMemoryLimit = 0 conf.MaxBlockLimit = 15 const hashSize int64 = 20 @@ -137,15 +137,15 @@ func TestStreamSectionReader(t *testing.T) { }, Reader: io.NopCloser(bytes.NewReader(buf)), } - prevCacheThreshold := conf.CacheThreshold + prevAutoMemoryLimit := conf.AutoMemoryLimit prevMaxBlockLimit := conf.MaxBlockLimit prevConf := conf.Conf t.Cleanup(func() { - conf.CacheThreshold = prevCacheThreshold + conf.AutoMemoryLimit = prevAutoMemoryLimit conf.MaxBlockLimit = prevMaxBlockLimit conf.Conf = prevConf }) - conf.CacheThreshold = 1 + conf.AutoMemoryLimit = 0 conf.MaxBlockLimit = 2 << 10 partSize := 3 << 10 conf.Conf = &conf.Config{} diff --git a/internal/stream/util.go b/internal/stream/util.go index 8db4b7ae6d..2947fcbc2b 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -1,7 +1,6 @@ package stream import ( - "bytes" "context" "encoding/hex" "errors" @@ -12,12 +11,11 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" - "github.com/OpenListTeam/OpenList/v4/internal/mem" + hcache "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" - "github.com/OpenListTeam/OpenList/v4/pkg/pool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -184,25 +182,13 @@ type StreamSectionReader interface { DiscardSection(off int64, length int64) error } -func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (StreamSectionReader, error) { +func NewStreamSectionReader(file model.FileStreamer, sectionSize int, up *model.UpdateProgress) (StreamSectionReader, error) { if file.GetFile() != nil { return &cachedSectionReader{file.GetFile()}, nil } - maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if file.GetSize() <= int64(conf.CacheThreshold) { - bufPool := &pool.Pool[[]byte]{ - New: func() []byte { - return make([]byte, maxBufferSize) - }, - } - file.Add(bufPool) - ss := &byteSectionReader{file: file, bufPool: bufPool} - return ss, nil - } - - blockSize := min(uint64(maxBufferSize), conf.MaxBlockLimit) - hc, err := mem.NewHybridCache(blockSize, uint64(file.GetSize())) + blockSize := min(uint64(sectionSize), uint64(file.GetSize()), conf.MaxBlockLimit) + hc, err := hcache.NewHybridCache(blockSize, uint64(file.GetSize())) if err != nil { return nil, err } @@ -222,56 +208,10 @@ func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker } func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} -type byteSectionReader struct { - file model.FileStreamer - fileOffset int64 - bufPool *pool.Pool[[]byte] -} - -// 线程不安全 -func (ss *byteSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.fileOffset { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) - } - n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) - ss.fileOffset += n - if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) - } - return nil -} - -type bytesRefReadSeeker struct { - io.ReadSeeker - buf []byte -} - -// 线程不安全 -func (ss *byteSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.fileOffset { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) - } - tempBuf := ss.bufPool.Get() - buf := tempBuf[:length] - n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) - } - return &bytesRefReadSeeker{bytes.NewReader(buf), buf}, nil -} -func (ss *byteSectionReader) FreeSectionReader(rs io.ReadSeeker) { - if sr, ok := rs.(*bytesRefReadSeeker); ok { - ss.bufPool.Put(sr.buf[0:cap(sr.buf)]) - sr.buf = nil - sr.ReadSeeker = nil - } -} - type hybridSectionReader struct { file model.FileStreamer fileOffset int64 - hc *mem.HybridCache + hc *hcache.HybridCache mu sync.Mutex cache []buffer.Block } diff --git a/pkg/buffer/buffer.go b/pkg/buffer/buffer.go new file mode 100644 index 0000000000..54d13c56d3 --- /dev/null +++ b/pkg/buffer/buffer.go @@ -0,0 +1,41 @@ +package buffer + +import ( + "io" +) + +type byteBlock struct { + buf []byte +} + +func NewByteBlock(buf []byte) Block { + return &byteBlock{buf: buf} +} + +func (b *byteBlock) Size() int64 { + return int64(len(b.buf)) +} + +func (b *byteBlock) ReadAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.EOF + } + n = copy(p, b.buf[off:]) + if n < len(p) { + err = io.EOF + } + return +} + +func (b *byteBlock) WriteAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.ErrShortWrite + } + n = copy(b.buf[off:], p) + if n < len(p) { + err = io.ErrShortWrite + } + return +} + +var _ Block = (*byteBlock)(nil) diff --git a/pkg/buffer/bytes.go b/pkg/buffer/bytes.go deleted file mode 100644 index ca71e8f24b..0000000000 --- a/pkg/buffer/bytes.go +++ /dev/null @@ -1,132 +0,0 @@ -package buffer - -import ( - "errors" - "io" -) - -// 用于存储不复用的[]byte -type Reader struct { - bufs [][]byte - size int64 - offset int64 -} - -func (r *Reader) Size() int64 { - return r.size -} - -func (r *Reader) Append(buf []byte) { - r.size += int64(len(buf)) - r.bufs = append(r.bufs, buf) -} - -func (r *Reader) Read(p []byte) (int, error) { - n, err := r.ReadAt(p, r.offset) - if n > 0 { - r.offset += int64(n) - } - return n, err -} - -func (r *Reader) ReadAt(p []byte, off int64) (int, error) { - if len(p) == 0 { - return 0, nil - } - if off < 0 || off >= r.size { - return 0, io.EOF - } - - n := 0 - for _, buf := range r.bufs { - if off >= int64(len(buf)) { - off -= int64(len(buf)) - continue - } - nn := copy(p[n:], buf[off:]) - n += nn - if n == len(p) { - return n, nil - } - off = 0 - } - - return n, io.EOF -} - -func (r *Reader) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - case io.SeekCurrent: - offset = r.offset + offset - case io.SeekEnd: - offset = r.size + offset - default: - return 0, errors.New("Seek: invalid whence") - } - - if offset < 0 || offset > r.size { - return 0, errors.New("Seek: invalid offset") - } - - r.offset = offset - return offset, nil -} - -func (r *Reader) Reset() { - clear(r.bufs) - r.bufs = nil - r.size = 0 - r.offset = 0 -} - -func (r *Reader) Close() error { - r.Reset() - return nil -} - -func NewReader(buf ...[]byte) *Reader { - b := &Reader{ - bufs: make([][]byte, 0, len(buf)), - } - for _, b1 := range buf { - b.Append(b1) - } - return b -} - -type byteBlock struct { - buf []byte -} - -func NewByteBlock(buf []byte) Block { - return &byteBlock{buf: buf} -} - -func (b *byteBlock) Size() int64 { - return int64(len(b.buf)) -} - -func (b *byteBlock) ReadAt(p []byte, off int64) (n int, err error) { - if len(b.buf) == 0 || off < 0 || off >= b.Size() { - return 0, io.EOF - } - n = copy(p, b.buf[off:]) - if n < len(p) { - err = io.EOF - } - return -} - -func (b *byteBlock) WriteAt(p []byte, off int64) (n int, err error) { - if len(b.buf) == 0 || off < 0 || off >= b.Size() { - return 0, io.ErrShortWrite - } - n = copy(b.buf[off:], p) - if n < len(p) { - err = io.ErrShortWrite - } - return -} - -var _ Block = (*byteBlock)(nil) diff --git a/pkg/buffer/file.go b/pkg/buffer/file.go deleted file mode 100644 index 48edf5a4cb..0000000000 --- a/pkg/buffer/file.go +++ /dev/null @@ -1,88 +0,0 @@ -package buffer - -import ( - "errors" - "io" - "os" -) - -type PeekFile struct { - peek *Reader - file *os.File - offset int64 - size int64 -} - -func (p *PeekFile) Read(b []byte) (n int, err error) { - n, err = p.ReadAt(b, p.offset) - if n > 0 { - p.offset += int64(n) - } - return n, err -} - -func (p *PeekFile) ReadAt(b []byte, off int64) (n int, err error) { - if off < p.peek.Size() { - n, err = p.peek.ReadAt(b, off) - if err == nil || n == len(b) { - return n, nil - } - // EOF - } - var nn int - nn, err = p.file.ReadAt(b[n:], off+int64(n)-p.peek.Size()) - return n + nn, err -} - -func (p *PeekFile) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - case io.SeekCurrent: - if offset == 0 { - return p.offset, nil - } - offset = p.offset + offset - case io.SeekEnd: - offset = p.size + offset - default: - return 0, errors.New("Seek: invalid whence") - } - - if offset < 0 || offset > p.size { - return 0, errors.New("Seek: invalid offset") - } - if offset <= p.peek.Size() { - _, err := p.peek.Seek(offset, io.SeekStart) - if err != nil { - return 0, err - } - _, err = p.file.Seek(0, io.SeekStart) - if err != nil { - return 0, err - } - } else { - _, err := p.peek.Seek(p.peek.Size(), io.SeekStart) - if err != nil { - return 0, err - } - _, err = p.file.Seek(offset-p.peek.Size(), io.SeekStart) - if err != nil { - return 0, err - } - } - - p.offset = offset - return offset, nil -} - -func (p *PeekFile) Size() int64 { - return p.size -} - -func NewPeekFile(peek *Reader, file *os.File) (*PeekFile, error) { - stat, err := file.Stat() - if err == nil { - return &PeekFile{peek: peek, file: file, size: stat.Size() + peek.Size()}, nil - } - return nil, err -} diff --git a/pkg/buffer/type.go b/pkg/buffer/type.go index e4d7132168..ce0d78b2e9 100644 --- a/pkg/buffer/type.go +++ b/pkg/buffer/type.go @@ -13,8 +13,10 @@ type Block interface { } type WriteAtSeeker = model.FileWriter +type WriteAtSeekerProvider interface{ GetWriteAtSeeker() WriteAtSeeker } type ReadAtSeeker = model.File +type ReadAtSeekerProvider interface{ GetReadAtSeeker() ReadAtSeeker } type SizedReadAtSeeker interface { ReadAtSeeker diff --git a/pkg/buffer/utils.go b/pkg/buffer/utils.go index a1bde567a6..4783f4b704 100644 --- a/pkg/buffer/utils.go +++ b/pkg/buffer/utils.go @@ -5,8 +5,6 @@ import ( "io" ) -type WriteAtSeekerProvider interface{ GetWriteAtSeeker() WriteAtSeeker } - func WriteAtSeekerOf(b Block) WriteAtSeeker { if p, ok := b.(WriteAtSeekerProvider); ok { return p.GetWriteAtSeeker() @@ -14,8 +12,6 @@ func WriteAtSeekerOf(b Block) WriteAtSeeker { return io.NewOffsetWriter(b, 0) } -type ReadAtSeekerProvider interface{ GetReadAtSeeker() ReadAtSeeker } - // 将一个Block包装为ReadAtSeeker。 // 固定大小：当前Block的Size()。 func ReadAtSeekerOf(b Block) ReadAtSeeker { From 31b41f99f8ac0c11f7f5b0aef6dce251eb69ee62 Mon Sep 17 00:00:00 2001 From: Chisiung Date: Mon, 18 May 2026 15:12:15 +0800 Subject: [PATCH 008/107] fix(offline_download): fix login failure caused by qBittorrent 5.2.0 returning HTTP 204 No Content on successful authentication (#2476) --- pkg/qbittorrent/client.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/qbittorrent/client.go b/pkg/qbittorrent/client.go index cc8be87071..e4c12db6fc 100644 --- a/pkg/qbittorrent/client.go +++ b/pkg/qbittorrent/client.go @@ -99,12 +99,14 @@ func (c *client) login() error { defer resp.Body.Close() // avoid long waiting time if being upgraded to websocket connections (e.g. 101 responses) - // as per API documentation, qBittorrent returns only 200 on successful login - // so we safely treat any non-200 response as a failure - if resp.StatusCode != http.StatusOK { + // as per API documentation, qBittorrent returns only 200 on successful login (qBittorrent < 5.2.0) + // qBittorrent 5.2.0 /api/v2/auth/login returns HTTP 204 on success + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return errors.New("failed to login into qBittorrent webui with status code: " + resp.Status) } - + if resp.StatusCode == http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) _, err = resp.Body.Read(body) @@ -180,6 +182,10 @@ func (c *client) AddFromLink(link string, savePath string, id string) error { return err } defer resp.Body.Close() + // qBittorrent 5.2.0 returns 204 on success. + if resp.StatusCode != http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) From 7cfb25558f368add8ccad9d2962e9bd568f05559 Mon Sep 17 00:00:00 2001 From: Yinan Qin Date: Thu, 21 May 2026 15:52:11 +0800 Subject: [PATCH 009/107] fix(drivers/local): capture ENOTTY in reflink to allow fallback (#2492) --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index cd86a81474..1df78f4b22 100644 --- a/go.mod +++ b/go.mod @@ -313,3 +313,5 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go + +replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 7587412499..4ec25fe0b3 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= -github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -35,6 +33,8 @@ github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+ github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 h1:qLqJPr/FAsZTJiqy65JKKuFJP0V9pRVtSaIE0kqaQ8w= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= From c3df8da577ea41f95424839d94b5c1bb4f11db8b Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Sun, 24 May 2026 09:49:07 +0800 Subject: [PATCH 010/107] chore(README): fix-sites (#2501) * Squashed commit of the following: commit f029df88e4ebc36e0886662193dc99f8276959ce Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 14:00:36 2025 +0800 Add Terms of Use and Privacy Policy links to READMEs Added links to the Terms of Use and Privacy Policy in the English, Chinese, Japanese, and Dutch README files to provide users with easy access to legal information. commit 00f825d9e26afcd8aa6c0d627d5351d1811df610 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:53:29 2025 +0800 Update documentation links in README files Replaced plain documentation URLs with labeled links and icons in English, Chinese, Japanese, and Dutch README files for improved clarity and user experience. commit 732dcfa5b1dbc60abf55eec062371cbe3466c1f2 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:40:52 2025 +0800 fix format commit e2ad8eabb8eb0f98d5838f74380b498637261f9e Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:38:28 2025 +0800 Revert "test large name" This reverts commit affedc845b6106726de13db5d883b19fd64e7d2a. commit affedc845b6106726de13db5d883b19fd64e7d2a Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:37:43 2025 +0800 test large name commit 382cd6425fd41a8ddbb47385219544d09ab6a6be Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:34:42 2025 +0800 Add Dutch README and update language links Added a new Dutch translation (README_nl.md) and updated language navigation links in the English, Chinese, and Japanese README files to include Dutch. commit e880acb71d893b911af333f7d06e30c18894a2f5 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:29:51 2025 +0800 Add AGPL-3.0 license links to README files Updated the English, Chinese, and Japanese README files to include direct links to the AGPL-3.0 license text and the LICENSE file for clarity and easier access. commit a0d1eadf3eda55e55d9e3a0ceca1550e5f709273 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:25:52 2025 +0800 Move language and links sections below logo in READMEs Repositioned the language selection and related links sections to appear after the logo and separator in README.md, README_cn.md, and README_ja.md for improved layout consistency. commit 70a0a32b7bb4748fbc409f97e437cb1e0a381c21 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 13:23:36 2025 +0800 Revise and unify README files across languages Updated README.md, README_cn.md, and README_ja.md to improve structure, add navigation links, clarify project purpose, and unify feature lists. Enhanced formatting, added acknowledgments to original authors, and improved legal/disclaimer sections for consistency across English, Chinese, and Japanese documentation. commit 2f32120908d8b7d4f8da7da618a08644dbff9169 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:56:26 2025 +0800 Update Go Report Card badge URL in README Changed the Go Report Card badge to reference v3 instead of v4. This ensures the badge displays the correct status for the intended version. commit 0fdfa2b365d1448dc0452704d67d9a6029e40128 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:53:43 2025 +0800 Update README.md commit 82713611c01371c413412265651e32216777d4e5 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:53:22 2025 +0800 Update Go Report Card badge URL in README Changed the Go Report Card badge link to remove the '/v3' suffix, ensuring it points to the correct repository path. commit 41acb3e8654a3a5be2e30c8cd0238e6b36b5b92b Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:52:46 2025 +0800 Update project description in README Revised the introductory paragraph to emphasize OpenList's resilience and community-driven nature as a fork of AList, highlighting its commitment to defending open source against trust-based attacks. commit 77aca6609a28d11169653e8c35f445b8748ae783 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:50:45 2025 +0800 Update README.md commit 63a597f8027408206d58e952f17486016a666532 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:49:57 2025 +0800 Improve README badge formatting and alignment Reformatted the badge section in the README for better readability and visual alignment. Updated the div to use 'align="center"' and placed each badge on its own line with proper indentation. commit fcf7530dd84d557f649e0c9c69d2776a6fded225 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:46:38 2025 +0800 Update logo size and remove migration note in README Set explicit width and height for the logo image and removed the note about migration progress, reflecting project updates. commit 5f0645ded842643890ad62ba425740f5dea7cd4d Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:45:04 2025 +0800 Revert README header to HTML Replaces markdown-based center alignment and badge/image syntax with HTML tags for better visual formatting and consistency in the README header. commit 0f7ba9599d7e7a398da11a15403dae4d83ba0584 Author: jyxjjj <773933146@qq.com> Date: Tue Jul 1 12:42:59 2025 +0800 Revise README formatting and update project info Refactored the README to use markdown badge/link syntax, improved formatting, and clarified the disclaimer section. Updated Docker Deploy status, added a Contact Us section, and reordered the Contributors section for better project transparency and communication. * chore(readme): fix sites * chore(readme): fix sites --- README.md | 7 ++++--- README_cn.md | 13 +++++++------ README_ja.md | 13 +++++++------ README_nl.md | 7 ++++--- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1ec96df4d9..4a94dbfcf6 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,10 @@ Thank you for your support and understanding of the OpenList project. ## Document -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Docs](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Terms of Use](https://doc.oplist.org/terms) +- 🔒 [Privacy Policy](https://doc.oplist.org/privacy) ## Demo diff --git a/README_cn.md b/README_cn.md index 1bcfdbed9d..55fe221060 100644 --- a/README_cn.md +++ b/README_cn.md @@ -116,14 +116,15 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 ## 文档 -- 🌏 [国内站点](https://doc.oplist.org.cn) -- 📘 [海外站点](https://doc.oplist.org) -- 📚 [备用站点](https://doc.openlist.team) +- 📘 [文档](https://doc.oplist.org) +- 🌏 [中国镜像](https://doc.oplist.org.cn) +- ⚖️ [使用条款](https://doc.oplist.org/terms) +- 🔒 [隐私政策](https://doc.oplist.org/privacy) -## 演示 +## Demo -- 🇨🇳 [国内演示站](https://demo.oplist.org.cn) -- 🌎 [海外演示站](https://demo.oplist.org) +- 🌎 [全球 Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## 讨论 diff --git a/README_ja.md b/README_ja.md index 3a5d5d19f7..261223de3a 100644 --- a/README_ja.md +++ b/README_ja.md @@ -116,14 +116,15 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい ## ドキュメント -- 📘 [グローバルサイト](https://doc.oplist.org) -- 📚 [バックアップサイト](https://doc.openlist.team) -- 🌏 [CNサイト](https://doc.oplist.org.cn) +- 📘 [ドキュメント](https://doc.oplist.org) +- 🌏 [中国ミラー](https://doc.oplist.org.cn) +- ⚖️ [利用規約](https://doc.oplist.org/terms) +- 🔒 [プライバシーポリシー](https://doc.oplist.org/privacy) -## デモ +## Demo -- 🌎 [グローバルデモ](https://demo.oplist.org) -- 🇨🇳 [CNデモ](https://demo.oplist.org.cn) +- 🌎 [グローバル Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## ディスカッション diff --git a/README_nl.md b/README_nl.md index 86e90e740c..d3be2703f9 100644 --- a/README_nl.md +++ b/README_nl.md @@ -116,9 +116,10 @@ Dank u voor uw ondersteuning en begrip ## Documentatie -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Documentatie](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Gebruiksvoorwaarden](https://doc.oplist.org/terms) +- 🔒 [Privacybeleid](https://doc.oplist.org/privacy) ## Demo From 948e87159d731cefa9ec644dec339e106cd43d27 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Mon, 25 May 2026 12:13:22 +0800 Subject: [PATCH 011/107] docs(community): update contribution guidelines and templates (#2507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化 PR 模板结构和检查项 增加 PR 中 AI 辅助披露说明和工具范围 在 Issue 模板中增加 AI 辅助内容字段 补充贡献指南中的社区政策和 AI 披露要求 将 Go 版本要求改为以 go.mod 声明为准 修正文档链接和行为准则格式细节 --- .github/ISSUE_TEMPLATE/00-bug_report_zh.yml | 6 + .github/ISSUE_TEMPLATE/01-bug_report_en.yml | 6 + .../ISSUE_TEMPLATE/02-feature_request_zh.yml | 6 + .../ISSUE_TEMPLATE/03-feature_request_en.yml | 6 + .github/PULL_REQUEST_TEMPLATE.md | 138 ++++++++++++------ CODE_OF_CONDUCT.md | 8 +- CONTRIBUTING.md | 37 ++++- 7 files changed, 155 insertions(+), 52 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index c6aa3fc623..ec18a650b4 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -82,3 +82,9 @@ body: label: 复现链接（可选） description: | 请提供能复现此问题的链接。 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此问题是由AI辅助您发现的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index d99968d907..0085d4de53 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -82,3 +82,9 @@ body: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this issue was identified with the assistance of AI, please provide the complete chat log, including information about the model used. diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 8339d947f9..9072402611 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -48,3 +48,9 @@ body: label: 附加信息 description: | 相关的任何其他上下文或截图，或者你觉得有帮助的信息 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此请求是由AI辅助您提交的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 41c9990cfe..393118592f 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -48,3 +48,9 @@ body: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this request was submitted with the assistance of an AI, please provide the complete chat log, including information about the model used. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f1687eabfe..58e122837b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,58 +1,114 @@ + +## Summary / 摘要 + -## Description / 描述 + - +- 列出用户可感知的行为变化。 +- 列出重要实现变化。 +- 如涉及配置、存储、API 或兼容性变化，请明确说明。 +--> -## Motivation and Context / 背景 +- [ ] This PR has breaking changes. + / 此 PR 包含破坏性变更。 +- [ ] This PR changes public API, config, storage format, or migration behavior. + / 此 PR 修改了公开 API、配置、存储格式或迁移行为。 +- [ ] This PR requires corresponding changes in related repositories. + / 此 PR 需要关联仓库同步修改。 - - +Related repository PRs / 关联仓库 PR: - - +- OpenList-Frontend: +- OpenList-Docs: -Closes #XXXX +## Related Issues / 关联 Issue - - + -Relates to #XXXX +## Testing / 测试 -## How Has This Been Tested? / 测试 + - - +- [ ] `go test ./...` +- [ ] Manual test / 手动测试: ## Checklist / 检查清单 - - - - - - -- [ ] I have read the [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) document. - 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) 文档。 -- [ ] I have formatted my code with `go fmt` or [prettier](https://prettier.io/). - 我已使用 `go fmt` 或 [prettier](https://prettier.io/) 格式化提交的代码。 -- [ ] I have added appropriate labels to this PR (or mentioned needed labels in the description if lacking permissions). - 我已为此 PR 添加了适当的标签（如无权限或需要的标签不存在，请在描述中说明，管理员将后续处理）。 -- [ ] I have requested review from relevant code authors using the "Request review" feature when applicable. - 我已在适当情况下使用"Request review"功能请求相关代码作者进行审查。 -- [ ] I have updated the repository accordingly (If it’s needed). - 我已相应更新了相关仓库（若适用）。 - - [ ] [OpenList-Frontend](https://github.com/OpenListTeam/OpenList-Frontend) #XXXX - - [ ] [OpenList-Docs](https://github.com/OpenListTeam/OpenList-Docs) #XXXX +- [ ] I have read [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md). + / 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md)。 +- [ ] I confirm this contribution follows the repository license, contribution policy, and code of conduct. + / 我确认此贡献符合仓库许可证、贡献规范和行为准则。 +- [ ] I have formatted the changed code with `gofmt`, `go fmt`, or `prettier` where applicable. + / 我已按适用情况使用 `gofmt`、`go fmt` 或 `prettier` 格式化变更代码。 +- [ ] I have requested review from relevant maintainers or code owners where applicable. + / 我已在适用情况下请求相关维护者或代码所有者审查。 + +## AI Disclosure / AI 使用声明 + + + +- [ ] This PR includes AI-assisted content. + / 此 PR 包含 AI 辅助内容。 + +Tools used / 使用工具: + +- [ ] ChatGPT +- [ ] Codex +- [ ] GitHub Copilot +- [ ] Claude +- [ ] Gemini +- [ ] Other (please specify) / 其他（请注明）: + +Usage scope / 使用范围: + +- [ ] Code generation / 代码生成 +- [ ] Refactoring / 重构 +- [ ] Documentation / 文档 +- [ ] Tests / 测试 +- [ ] Translation / 翻译 +- [ ] Review assistance / 审查辅助 + +- [ ] I have reviewed and validated all AI-assisted content included in this PR. + / 我已审核并验证此 PR 中的所有 AI 辅助内容。 +- [ ] I have ensured that all AI-assisted commits include `Co-Authored-By` attribution. + / 我已确保所有 AI 辅助提交都包含 `Co-Authored-By` 归属信息。 +- [ ] I can reproduce all AI-assisted content included in this PR without any AI tools. + / 我可以在没有任何 AI 工具的情况下重现此 PR 中包含的所有 AI 辅助内容。 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c8344eb6c7..6d4a6ef273 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -106,7 +106,7 @@ Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an +standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within @@ -116,7 +116,7 @@ the community. This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). @@ -124,5 +124,5 @@ enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6064b03cb5..57317c6290 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Prerequisites: - [git](https://git-scm.com) -- [Go 1.24+](https://golang.org/doc/install) +- [Go](https://golang.org/doc/install) version declared in [`go.mod`](./go.mod) - [gcc](https://gcc.gnu.org/) - [nodejs](https://nodejs.org/) @@ -16,8 +16,8 @@ Prerequisites: Fork and clone `OpenList` and `OpenList-Frontend` anywhere: ```shell -$ git clone https://github.com//OpenList.git -$ git clone --recurse-submodules https://github.com//OpenList-Frontend.git +git clone https://github.com//OpenList.git +git clone --recurse-submodules https://github.com//OpenList-Frontend.git ``` ## Creating a branch @@ -25,7 +25,7 @@ $ git clone --recurse-submodules https://github.com//OpenList-Fro Create a new branch from the `main` branch, with an appropriate name. ```shell -$ git checkout -b +git checkout -b ``` ## Preview your change @@ -33,26 +33,36 @@ $ git checkout -b ### backend ```shell -$ go run main.go +go run main.go ``` ### frontend ```shell -$ pnpm dev +pnpm dev ``` ## Add a new driver Copy `drivers/template` folder and rename it, and follow the comments in it. +## Community and policies + +By contributing, you agree to follow the repository's code of conduct and license terms. + +- Code of conduct: [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) +- License: [LICENSE](./LICENSE) +- Security issues: please report privately according to [SECURITY.md](./SECURITY.md) + +If your contribution includes substantial AI-assisted content, disclose the tools used and the scope of assistance in the pull request. + ## Create a commit Commit messages should be well formatted, and to make that "standardized". Submit your pull request. For PR titles, follow [Conventional Commits](https://www.conventionalcommits.org). -https://github.com/OpenListTeam/OpenList/issues/376 + It's suggested to sign your commits. See: [How to sign commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) @@ -72,6 +82,19 @@ At least 1 approving review is required by reviewers with write access. You can (Optional) After your pull request is merged, you can delete your branch. +## AI Disclosure + +If your pull request includes substantial AI-assisted content, disclose it in the PR description. + +Please include: + +- Tools used, such as ChatGPT, GitHub Copilot, Claude, Cursor, or other AI tools. +- Usage scope, such as code generation, refactoring, documentation, tests, translation, or review assistance. +- Confirmation that you have reviewed and validated all AI-assisted content before submission. +- Confirmation that the submitted content complies with this repository's license and contribution policies. + +Minor AI assistance, such as typo fixes, autocomplete, formatting suggestions, or wording polish, does not need to be disclosed. + --- Thank you for your contribution! Let's make OpenList better together! From 1d071c0fd875780e16a086e69f6fc8fbb01091df Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Mon, 25 May 2026 14:34:21 +0800 Subject: [PATCH 012/107] feat(func): support ed2k & magnet & torrent offline download (#2452) Add end-to-end offline download support for torrent files, magnet links, and ed2k links, including torrent parse and generate APIs, CAS-based rapid upload for Cloud189, and protocol-aware tool routing with transfer flow improvements. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Co-authored-by: j2rong4cn Co-authored-by: Suyunjing --- drivers/189/torrent.go | 149 +++++++ drivers/189/util.go | 55 ++- drivers/189pc/driver.go | 23 +- drivers/189pc/meta.go | 19 +- drivers/189pc/torrent.go | 296 ++++++++++++++ drivers/189pc/utils.go | 62 ++- internal/offline_download/aria2/aria2.go | 5 + internal/offline_download/tool/add.go | 59 +++ internal/offline_download/tool/transfer.go | 73 +++- pkg/torrent/bencode.go | 261 ++++++++++++ pkg/torrent/generate.go | 123 ++++++ pkg/torrent/hash_writer.go | 229 +++++++++++ pkg/torrent/torrent.go | 439 +++++++++++++++++++++ server/handles/torrent.go | 433 ++++++++++++++++++++ server/router.go | 5 + 15 files changed, 2202 insertions(+), 29 deletions(-) create mode 100644 drivers/189/torrent.go create mode 100644 drivers/189pc/torrent.go create mode 100644 pkg/torrent/bencode.go create mode 100644 pkg/torrent/generate.go create mode 100644 pkg/torrent/hash_writer.go create mode 100644 pkg/torrent/torrent.go create mode 100644 server/handles/torrent.go diff --git a/drivers/189/torrent.go b/drivers/189/torrent.go new file mode 100644 index 0000000000..4e41bf4905 --- /dev/null +++ b/drivers/189/torrent.go @@ -0,0 +1,149 @@ +package _189 + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +func (d *Cloud189) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte) error { + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 获取 sessionKey + sessionKey, err := d.getSessionKey() + if err != nil { + return err + } + d.sessionKey = sessionKey + + // 初始化上传 + res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + "parentFolderId": dstDir.GetID(), + "fileName": encode(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(cas.SliceSize), + "lazyCheck": "1", + }, nil) + if err != nil { + return fmt.Errorf("初始化上传失败: %w", err) + } + + uploadFileId := utils.Json.Get(res, "data", "uploadFileId").ToString() + + // 提交上传（使用 CAS 信息秒传） + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": cas.FileMD5, + "sliceMd5": cas.SliceMD5, + "lazyCheck": "1", + "opertype": "3", + }, nil) + if err != nil { + return fmt.Errorf("秒传提交失败: %w", err) + } + + return nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} diff --git a/drivers/189/util.go b/drivers/189/util.go index ca86937c97..a98b0d0161 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/md5" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "errors" @@ -18,6 +19,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" @@ -388,6 +391,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F var finish int64 = 0 var i int64 var byteSize int64 + + // 额外计算 SHA-1 piece hash 用于生成 torrent + pieceSHA1Hashes := make([]byte, 0, int(count)*20) + for i = 1; i <= count; i++ { if utils.IsCanceled(ctx) { return ctx.Err() @@ -404,6 +411,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F finish += int64(n) md5Bytes := getMd5(byteData) md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) + + // 计算 SHA-1 piece hash + sha1Hash := sha1Pkg.Sum(byteData) + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Hash[:]...) var resp UploadUrlsResp res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{ "partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64), @@ -439,7 +450,49 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F "lazyCheck": "1", "opertype": "3", }, nil) - return err + if err != nil { + return err + } + + // 生成 torrent 文件（异步，不影响上传结果） + capturedDstDir := dstDir + capturedFileName := file.GetName() + capturedFileSize := fileSize + capturedFileMd5Hex := fileMd5Hex + capturedMd5s := md5s + go func() { + fileMD5Upper := strings.ToUpper(capturedFileMd5Hex) + torrentData, err := GenerateTorrent(capturedFileName, capturedFileSize, fileMD5Upper, capturedMd5s, DEFAULT, pieceSHA1Hashes) + if err != nil { + log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录 + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + uploadErr := d.oldUpload(capturedDstDir, torrentFileStream) + if uploadErr != nil { + log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(d, capturedDstDir.GetPath()) + } + }() + + return nil } func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index b0a23e9d24..104832f74d 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -343,6 +343,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // 响应时间长,按需启用 if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { + // 尝试妙传 if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } @@ -351,10 +352,11 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil uploadMethod := y.UploadMethod if stream.IsForceStreamUpload() { uploadMethod = "stream" - } - - // 旧版上传家庭云也有限制 - if uploadMethod == "old" { + } else if y.Addition.RapidUpload && stream.GetFile() != nil { + // 文件流支持随机读取，走FastUpload计算MD5并尝试秒传 + uploadMethod = "rapid" + } else if uploadMethod == "old" { + // 旧版上传家庭云也有限制 return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) } @@ -416,19 +418,6 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if stream.GetSize() == 0 { return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } - // 尝试秒传：如果已有MD5且启用了RapidUpload则用RapidUpload，否则走FastUpload（会计算MD5并尝试秒传） - if !stream.IsForceStreamUpload() { - fileMd5 := stream.GetHash().GetHash(utils.MD5) - if len(fileMd5) >= utils.MD5.Width && y.Addition.RapidUpload { - // 源文件已有MD5且启用了RapidUpload配置，尝试快速秒传 - if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { - return newObj, nil - } - } else if len(fileMd5) < utils.MD5.Width { - // 源文件无MD5（如从本地复制），走FastUpload计算MD5并尝试秒传 - return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) - } - } fallthrough default: return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 366f1bf818..3a5e02979d 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -13,15 +13,16 @@ type Addition struct { AccessToken string `json:"access_token" required:"false"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - Type string `json:"type" type:"select" options:"personal,family" default:"personal"` - FamilyID string `json:"family_id"` - UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` - UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` - FamilyTransfer bool `json:"family_transfer"` - RapidUpload bool `json:"rapid_upload"` - NoUseOcr bool `json:"no_use_ocr"` + OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + Type string `json:"type" type:"select" options:"personal,family" default:"personal"` + FamilyID string `json:"family_id"` + UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + FamilyTransfer bool `json:"family_transfer"` + RapidUpload bool `json:"rapid_upload"` + NoUseOcr bool `json:"no_use_ocr"` + GenerateTorrent bool `json:"generate_torrent" help:"Generate torrent file with CAS extension after upload"` } var config = driver.Config{ diff --git a/drivers/189pc/torrent.go b/drivers/189pc/torrent.go new file mode 100644 index 0000000000..3068a0f7b9 --- /dev/null +++ b/drivers/189pc/torrent.go @@ -0,0 +1,296 @@ +package _189pc + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/url" + "strings" + + "github.com/go-resty/resty/v2" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +// fileMD5: 整文件 MD5（大写十六进制） +// sliceMD5s: 每个分片的 MD5 列表（大写十六进制） +// sliceSize: 分片大小 +// pieceHashes: SHA-1 piece hashes 拼接（每 20 字节一个） +// fileName: 文件名 +// fileSize: 文件大小 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +// 返回值：上传成功的文件对象、错误 +func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte, overwrite bool) (model.Obj, error) { + isFamily := y.isFamily() + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return nil, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 统一 MD5 为大写（与正常上传保持一致，天翼云盘要求大写） + fileMD5Upper := strings.ToUpper(cas.FileMD5) + + // 优先使用 torrent 中嵌入的分片大小，与生成时保持一致 + sliceSize := cas.SliceSize + if sliceSize <= 0 { + sliceSize = partSize(fileSize) + } + + // 计算 sliceMd5（与上传时一致的算法） + // 优先使用 torrent 中已有的 SliceMD5；仅当有多分片列表时才重新计算 + sliceMd5Hex := strings.ToUpper(cas.SliceMD5) + if sliceMd5Hex == "" { + sliceMd5Hex = fileMD5Upper + } + if len(cas.SliceMD5s) > 1 { + // 分片 MD5 也需要统一大写后再拼接计算 + upperSliceMD5s := make([]string, len(cas.SliceMD5s)) + for i, s := range cas.SliceMD5s { + upperSliceMD5s[i] = strings.ToUpper(s) + } + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n"))) + } + + + // 使用与 Web 端一致的三步秒传流程 + fullUrl := "https://upload.cloud.189.cn" + if isFamily { + fullUrl += "/family" + } else { + fullUrl += "/person" + } + + // Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck） + initParams := Params{ + "parentFolderId": dstDir.GetID(), + "fileName": url.QueryEscape(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(sliceSize), + "lazyCheck": "1", + } + if isFamily { + initParams.Set("familyId", y.FamilyID) + } + + + var uploadInfo InitMultiUploadResp + _, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, initParams, &uploadInfo, isFamily) + if err != nil { + return nil, fmt.Errorf("initMultiUpload 失败: %w", err) + } + + + uploadFileId := uploadInfo.Data.UploadFileID + + // Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传） + checkParams := Params{ + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "uploadFileId": uploadFileId, + } + + + var checkResp struct { + Data struct { + FileDataExists int `json:"fileDataExists"` + } `json:"data"` + } + _, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, checkParams, &checkResp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("秒传检查失败: %w", err) + } + + + if checkResp.Data.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize) + } + + // Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5） + + var resp CommitMultiUploadFileResp + commitParams := Params{ + "uploadFileId": uploadFileId, + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": IF(overwrite, "3", "1"), + } + + _, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, commitParams, &resp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return resp.toFile(), nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +// 适用于：已有文件需要生成 torrent 的场景（如下载完成后生成） +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +// 返回：CAS 信息、文件名、文件大小、错误 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// InjectCASIntoTorrent 向已有的 torrent 文件注入 CAS 扩展信息 +// 用于：下载完成后，计算了 MD5 信息，写回到 torrent 中 +func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string, sliceSize int64) ([]byte, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + // 注入 CAS 信息 + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + // 同时更新 info 中的 md5sum 字段 + if t.Info.MD5Sum == "" { + t.Info.MD5Sum = fileMD5 + } + + return t.Encode() +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} + +// ComputeSliceMD5sFromReader 从 reader 中计算每个 10MB 分片的 MD5 +// 返回：整文件 MD5、分片 MD5 列表 +func ComputeSliceMD5sFromReader(reader io.Reader, sliceSize int64) (string, []string, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + fileMD5Hash := utils.MD5.NewFunc() + sliceMD5s := make([]string, 0) + + buf := make([]byte, sliceSize) + for { + n, err := io.ReadFull(reader, buf) + if n > 0 { + chunk := buf[:n] + fileMD5Hash.Write(chunk) + // 计算该分片的 MD5 + sliceMD5 := strings.ToUpper(utils.HashData(utils.MD5, chunk)) + sliceMD5s = append(sliceMD5s, sliceMD5) + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return "", nil, err + } + } + + fileMD5Hex := strings.ToUpper(hex.EncodeToString(fileMD5Hash.Sum(nil))) + return fileMD5Hex, sliceMD5s, nil +} diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index cc5e227c61..e863060afe 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -3,6 +3,7 @@ package _189pc import ( "bytes" "context" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "encoding/xml" @@ -742,6 +743,10 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo silceMd5 := utils.MD5.NewFunc() var writers io.Writer = silceMd5 + // 如果启用了 torrent 生成，额外计算 SHA-1 piece hash + generateTorrent := y.Addition.GenerateTorrent + pieceSHA1Hashes := make([]byte, 0, count*20) + fileMd5Hex := file.GetHash().GetHash(utils.MD5) var fileMd5 hash.Hash if len(fileMd5Hex) != utils.MD5.Width { @@ -766,7 +771,18 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo return err } silceMd5.Reset() - w, err := utils.CopyWithBuffer(writers, reader) + + // 如果需要生成 torrent，同时计算 SHA-1 + var sha1Writer hash.Hash + var multiWriter io.Writer + if generateTorrent { + sha1Writer = sha1Pkg.New() + multiWriter = io.MultiWriter(writers, sha1Writer) + } else { + multiWriter = writers + } + + w, err := utils.CopyWithBuffer(multiWriter, reader) if w != partSize { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) } @@ -774,6 +790,11 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo md5Bytes := silceMd5.Sum(nil) silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) + + // 收集 SHA-1 piece hash + if generateTorrent && sha1Writer != nil { + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Writer.Sum(nil)...) + } return nil }, Do: func(ctx context.Context) (err error) { @@ -827,6 +848,45 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return nil, err } + + // 生成 torrent 文件（异步，不影响上传结果） + if generateTorrent && len(pieceSHA1Hashes) > 0 { + // 捕获必要的变量 + capturedDstDir := dstDir + capturedIsFamily := isFamily + capturedFileName := file.GetName() + go func() { + torrentData, err := GenerateTorrent(capturedFileName, fileSize, fileMd5Hex, silceMd5Hexs, sliceSize, pieceSHA1Hashes) + if err != nil { + utils.Log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + utils.Log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录（使用 FastUpload，因为 torrent 文件很小） + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + _, uploadErr := y.FastUpload(context.Background(), capturedDstDir, torrentFileStream, func(p float64) {}, capturedIsFamily, false) + if uploadErr != nil { + utils.Log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + utils.Log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(y, capturedDstDir.GetPath()) + } + }() + } + return resp.toFile(), nil } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index b04435aca1..5c037ff4cb 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -61,6 +62,10 @@ func (a *Aria2) IsReady() bool { } func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { + // aria2 不支持 ed2k 协议，提前检测并返回明确错误 + if strings.HasPrefix(strings.ToLower(args.Url), "ed2k://") { + return "", fmt.Errorf("aria2 does not support ed2k protocol. Please use Thunder/ThunderX/ThunderBrowser tool for ed2k links") + } options := map[string]interface{}{ "dir": args.TempDir, } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 0f574571ec..e42b08c31c 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,9 +2,11 @@ package tool import ( "context" + "fmt" "net/url" stdpath "path" "path/filepath" + "strings" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" @@ -71,6 +73,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro if err == nil || !errors.Is(err, errs.NotImplement) { return nil, err } + // SimpleHttp 不支持非 HTTP/HTTPS 协议（如 magnet、ed2k 等） + // tryPutUrl 返回 NotImplement 说明 URL 不是 HTTP/HTTPS + return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") + } + + // ed2k 链接自动路由：如果当前工具不支持 ed2k，自动尝试使用迅雷系工具 + if isEd2kURL(args.URL) { + if !isEd2kCapableTool(args.Tool) { + // 尝试找到一个可用的支持 ed2k 的工具 + fallbackTool, fallbackName := findEd2kCapableTool() + if fallbackTool != nil { + // 使用找到的迅雷工具替代 + args.Tool = fallbackName + } else { + return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + } + } } // get tool @@ -165,9 +184,49 @@ func tryPutUrl(ctx context.Context, path, urlStr string) error { var dstName string u, err := url.Parse(urlStr) if err == nil { + // 只支持 HTTP/HTTPS 协议，其他协议（magnet、ed2k 等）返回 NotImplement + if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" { + return errors.WithStack(errs.NotImplement) + } dstName = stdpath.Base(u.Path) } else { dstName = "UnnamedURL" } return fs.PutURL(ctx, path, dstName, urlStr) } + +// isEd2kURL 检测 URL 是否为 ed2k 协议 +func isEd2kURL(urlStr string) bool { + return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") +} + +// ed2kCapableTools 支持 ed2k 协议的工具列表（迅雷系） +var ed2kCapableTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} + +// isEd2kCapableTool 检查工具是否支持 ed2k 协议 +func isEd2kCapableTool(toolName string) bool { + for _, t := range ed2kCapableTools { + if t == toolName { + return true + } + } + return false +} + +// findEd2kCapableTool 查找一个可用的支持 ed2k 的工具 +func findEd2kCapableTool() (Tool, string) { + for _, name := range ed2kCapableTools { + t, err := Tools.Get(name) + if err != nil { + continue + } + if t.IsReady() { + return t, name + } + // 尝试初始化 + if _, err := t.Init(); err == nil && t.IsReady() { + return t, name + } + } + return nil, "" +} diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index fd6b8f4648..7109669ee7 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" @@ -201,8 +203,28 @@ func transferStdFile(t *TransferTask) error { } info, err := rc.Stat() if err != nil { + rc.Close() return errors.Wrapf(err, "failed to get file %s", t.SrcActualPath) } + + // 尝试对天翼云进行秒传（计算 MD5 + sliceMD5） + if rapidObj, rapidErr := tryRapidUpload189(t, rc, info.Size()); rapidErr == nil && rapidObj != nil { + rc.Close() + log.Infof("秒传成功: %s -> %s", t.SrcActualPath, t.DstStorageMp) + return nil + } + + // 秒传失败或不支持，回退到普通上传 + // 重新 seek 到文件开头 + if _, err := rc.Seek(0, 0); err != nil { + rc.Close() + // 重新打开文件 + rc, err = os.Open(t.SrcActualPath) + if err != nil { + return errors.Wrapf(err, "failed to reopen file %s", t.SrcActualPath) + } + } + mimetype := utils.GetMimeType(t.SrcActualPath) s := &stream.FileStream{ Ctx: t.Ctx(), @@ -217,7 +239,12 @@ func transferStdFile(t *TransferTask) error { Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) - return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + if err != nil { + return err + } + + return nil } func removeStdTemp(t *TransferTask) { @@ -341,3 +368,47 @@ func removeObjTemp(t *TransferTask) { log.Errorf("failed to delete temp obj %s, error: %s", t.SrcActualPath, err.Error()) } } + +// tryRapidUpload189 尝试对天翼云进行秒传 +// 通过计算文件的 MD5 来尝试秒传（使用旧版接口） +// 返回上传成功的对象和错误，如果不支持秒传则返回 nil, error +func tryRapidUpload189(t *TransferTask, file *os.File, fileSize int64) (model.Obj, error) { + // 检查目标存储是否是天翼云 PC 驱动 + cloud189PC, ok := t.DstStorage.(*_189pc.Cloud189PC) + if !ok { + return nil, fmt.Errorf("not 189pc storage") + } + + // 计算整文件 MD5（旧接口只需要 fileMD5） + fileMD5, _, err := _189pc.ComputeSliceMD5sFromReader(file, torrent.DefaultPieceSize) + if err != nil { + return nil, fmt.Errorf("计算 MD5 失败: %w", err) + } + + // 获取目标目录 + dstDir, err := op.Get(t.Ctx(), t.DstStorage, t.DstActualPath) + if err != nil { + return nil, fmt.Errorf("获取目标目录失败: %w", err) + } + + // 构造文件名 + fileName := filepath.Base(t.SrcActualPath) + + // 尝试秒传（使用旧接口） + uploadInfo, err := cloud189PC.OldUploadCreate(t.Ctx(), dstDir.GetID(), fileMD5, fileName, fmt.Sprint(fileSize), false) + if err != nil { + return nil, fmt.Errorf("创建上传任务失败: %w", err) + } + + if uploadInfo.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件") + } + + // 秒传成功，提交 + obj, err := cloud189PC.OldUploadCommit(t.Ctx(), uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, false, true) + if err != nil { + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return obj, nil +} diff --git a/pkg/torrent/bencode.go b/pkg/torrent/bencode.go new file mode 100644 index 0000000000..2d4fd782c6 --- /dev/null +++ b/pkg/torrent/bencode.go @@ -0,0 +1,261 @@ +package torrent + +import ( + "bytes" + "fmt" + "io" + "sort" + "strconv" +) + +// bencode 编码 + +// BencodeEncode 将值编码为 bencode 格式 +func BencodeEncode(v interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := bencodeEncodeValue(&buf, v); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func bencodeEncodeValue(w io.Writer, v interface{}) error { + switch val := v.(type) { + case int: + return bencodeEncodeInt(w, int64(val)) + case int64: + return bencodeEncodeInt(w, val) + case string: + return bencodeEncodeString(w, val) + case []byte: + return bencodeEncodeBytes(w, val) + case []interface{}: + return bencodeEncodeList(w, val) + case map[string]interface{}: + return bencodeEncodeDict(w, val) + case OrderedDict: + return bencodeEncodeOrderedDict(w, val) + default: + return fmt.Errorf("bencode: unsupported type %T", v) + } +} + +func bencodeEncodeInt(w io.Writer, v int64) error { + _, err := fmt.Fprintf(w, "i%de", v) + return err +} + +func bencodeEncodeString(w io.Writer, v string) error { + _, err := fmt.Fprintf(w, "%d:%s", len(v), v) + return err +} + +func bencodeEncodeBytes(w io.Writer, v []byte) error { + _, err := fmt.Fprintf(w, "%d:", len(v)) + if err != nil { + return err + } + _, err = w.Write(v) + return err +} + +func bencodeEncodeList(w io.Writer, v []interface{}) error { + if _, err := w.Write([]byte("l")); err != nil { + return err + } + for _, item := range v { + if err := bencodeEncodeValue(w, item); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +func bencodeEncodeDict(w io.Writer, v map[string]interface{}) error { + // bencode 字典要求 key 按字典序排列 + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, v[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// OrderedDict 有序字典，保持插入顺序 +type OrderedDict struct { + Keys []string + Values map[string]interface{} +} + +func NewOrderedDict() OrderedDict { + return OrderedDict{ + Keys: make([]string, 0), + Values: make(map[string]interface{}), + } +} + +func (d *OrderedDict) Set(key string, value interface{}) { + if _, exists := d.Values[key]; !exists { + d.Keys = append(d.Keys, key) + } + d.Values[key] = value +} + +func (d *OrderedDict) Get(key string) (interface{}, bool) { + v, ok := d.Values[key] + return v, ok +} + +func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error { + // 按字典序排列 key（bencode 规范要求） + keys := make([]string, len(d.Keys)) + copy(keys, d.Keys) + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, d.Values[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// bencode 解码 + +// BencodeDecode 从字节数组解码 bencode 数据 +func BencodeDecode(data []byte) (interface{}, error) { + reader := bytes.NewReader(data) + val, err := bencodeDecodeValue(reader) + if err != nil { + return nil, err + } + return val, nil +} + +func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + + switch { + case b == 'i': + return bencodeDecodeInt(r) + case b == 'l': + return bencodeDecodeList(r) + case b == 'd': + return bencodeDecodeDict(r) + case b >= '0' && b <= '9': + r.UnreadByte() + return bencodeDecodeString(r) + default: + return nil, fmt.Errorf("bencode: unexpected byte '%c' at position %d", b, int64(r.Len())) + } +} + +func bencodeDecodeInt(r *bytes.Reader) (int64, error) { + var buf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return 0, err + } + if b == 'e' { + break + } + buf.WriteByte(b) + } + return strconv.ParseInt(buf.String(), 10, 64) +} + +func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { + // 读取长度 + var lenBuf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == ':' { + break + } + lenBuf.WriteByte(b) + } + length, err := strconv.ParseInt(lenBuf.String(), 10, 64) + if err != nil { + return nil, fmt.Errorf("bencode: invalid string length: %v", err) + } + if length < 0 || length > 100*1024*1024 { + return nil, fmt.Errorf("bencode: string length out of bounds: %d", length) + } + // Safe to convert to int: bounds check above ensures length <= 100MB which fits in int32 + data := make([]byte, int(length)) + _, err = io.ReadFull(r, data) + if err != nil { + return nil, err + } + return data, nil +} + +func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { + var list []interface{} + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return list, nil + } + r.UnreadByte() + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + list = append(list, val) + } +} + +func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { + dict := make(map[string]interface{}) + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return dict, nil + } + r.UnreadByte() + keyBytes, err := bencodeDecodeString(r) + if err != nil { + return nil, err + } + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + dict[string(keyBytes)] = val + } +} diff --git a/pkg/torrent/generate.go b/pkg/torrent/generate.go new file mode 100644 index 0000000000..566cad86f7 --- /dev/null +++ b/pkg/torrent/generate.go @@ -0,0 +1,123 @@ +package torrent + +import ( + "io" + "os" + "strings" +) + +// GenerateFromFile 从文件路径生成通用的 torrent 文件（不含 CAS 扩展） +// 这是一个通用函数，适用于所有驱动 +func GenerateFromFile(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReader(f, info.Name(), info.Size(), DefaultPieceSize) +} + +// GenerateFromReader 从 io.Reader 生成通用的 torrent 文件（不含 CAS 扩展） +// 返回 torrent 字节数据 +func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + pieceHashes := hw.GetPieceHashes() + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + + return t.Encode() +} + +// GenerateFromReaderWithCAS 从 io.Reader 生成包含 CAS 扩展的 torrent 文件 +// 适用于天翼云等支持秒传的网盘 +func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: pieceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// GenerateFromFileWithCAS 从文件路径生成包含 CAS 扩展的 torrent 文件 +func GenerateFromFileWithCAS(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize) +} diff --git a/pkg/torrent/hash_writer.go b/pkg/torrent/hash_writer.go new file mode 100644 index 0000000000..a62f42c648 --- /dev/null +++ b/pkg/torrent/hash_writer.go @@ -0,0 +1,229 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "strings" +) + +// HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash +// 用于在上传过程中一次性计算所有需要的哈希值 +type HashWriter struct { + // 整文件 MD5 + fileMD5 hash.Hash + // 当前分片 MD5 + sliceMD5 hash.Hash + // 当前 piece 的 SHA-1 + pieceSHA1 hash.Hash + + // 分片大小（默认 10MB） + sliceSize int64 + // piece 大小（与 sliceSize 相同，保持对齐） + pieceSize int64 + + // 当前分片已写入字节数 + sliceWritten int64 + // 当前 piece 已写入字节数 + pieceWritten int64 + // 总写入字节数 + totalWritten int64 + + // 每个分片的 MD5（大写十六进制） + sliceMD5Hexs []string + // 所有 piece 的 SHA-1 哈希拼接 + pieceHashes []byte +} + +// NewHashWriter 创建一个新的 HashWriter +// sliceSize: CAS 分片大小（通常 10MB） +// pieceSize: BT piece 大小（设为与 sliceSize 相同以保持对齐） +func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { + if sliceSize <= 0 { + sliceSize = DefaultPieceSize + } + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + return &HashWriter{ + fileMD5: md5.New(), + sliceMD5: md5.New(), + pieceSHA1: sha1.New(), + sliceSize: sliceSize, + pieceSize: pieceSize, + } +} + +// NewDefaultHashWriter 创建默认的 HashWriter（10MB 分片） +func NewDefaultHashWriter() *HashWriter { + return NewHashWriter(DefaultPieceSize, DefaultPieceSize) +} + +// Write 实现 io.Writer 接口 +func (hw *HashWriter) Write(p []byte) (n int, err error) { + total := len(p) + offset := 0 + + for offset < total { + // 计算当前可以写入的字节数（取分片和 piece 剩余空间的最小值） + sliceRemain := hw.sliceSize - hw.sliceWritten + pieceRemain := hw.pieceSize - hw.pieceWritten + canWrite := min64(sliceRemain, pieceRemain) + canWrite = min64(canWrite, int64(total-offset)) + + chunk := p[offset : offset+int(canWrite)] + + // 写入整文件 MD5 + hw.fileMD5.Write(chunk) + // 写入当前分片 MD5 + hw.sliceMD5.Write(chunk) + // 写入当前 piece SHA-1 + hw.pieceSHA1.Write(chunk) + + hw.sliceWritten += canWrite + hw.pieceWritten += canWrite + hw.totalWritten += canWrite + offset += int(canWrite) + + // 检查分片是否完成 + if hw.sliceWritten >= hw.sliceSize { + hw.finishSlice() + } + + // 检查 piece 是否完成 + if hw.pieceWritten >= hw.pieceSize { + hw.finishPiece() + } + } + + return total, nil +} + +// finishSlice 完成当前分片的 MD5 计算 +func (hw *HashWriter) finishSlice() { + md5Hex := strings.ToUpper(hex.EncodeToString(hw.sliceMD5.Sum(nil))) + hw.sliceMD5Hexs = append(hw.sliceMD5Hexs, md5Hex) + hw.sliceMD5.Reset() + hw.sliceWritten = 0 +} + +// finishPiece 完成当前 piece 的 SHA-1 计算 +func (hw *HashWriter) finishPiece() { + hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...) + hw.pieceSHA1.Reset() + hw.pieceWritten = 0 +} + +// Finish 完成所有哈希计算（处理最后不完整的分片/piece） +func (hw *HashWriter) Finish() { + // 处理最后一个不完整的分片 + if hw.sliceWritten > 0 { + hw.finishSlice() + } + // 处理最后一个不完整的 piece + if hw.pieceWritten > 0 { + hw.finishPiece() + } +} + +// GetFileMD5 获取整文件 MD5（大写十六进制） +func (hw *HashWriter) GetFileMD5() string { + return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil))) +} + +// GetSliceMD5s 获取所有分片的 MD5 列表 +func (hw *HashWriter) GetSliceMD5s() []string { + return hw.sliceMD5Hexs +} + +// GetSliceMD5 获取最终的 sliceMD5（用于秒传） +func (hw *HashWriter) GetSliceMD5(fileMD5 string) string { + if len(hw.sliceMD5Hexs) <= 1 { + return fileMD5 + } + joined := strings.Join(hw.sliceMD5Hexs, "\n") + return strings.ToUpper(GetMD5Str(joined)) +} + +// GetPieceHashes 获取所有 piece 的 SHA-1 哈希拼接 +func (hw *HashWriter) GetPieceHashes() []byte { + return hw.pieceHashes +} + +// GetTotalWritten 获取总写入字节数 +func (hw *HashWriter) GetTotalWritten() int64 { + return hw.totalWritten +} + +// BuildTorrent 根据计算结果构建 Torrent 结构 +func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent { + fileMD5 := hw.GetFileMD5() + sliceMD5 := hw.GetSliceMD5(fileMD5) + + t := NewTorrent(fileName, fileSize, fileMD5) + t.SetPieces(hw.GetPieceHashes()) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: hw.GetSliceMD5s(), + SliceSize: hw.sliceSize, + Cloud: "189", + }) + + return t +} + +// BuildTorrentBytes 构建并编码 torrent 文件 +func (hw *HashWriter) BuildTorrentBytes(fileName string, fileSize int64) ([]byte, error) { + t := hw.BuildTorrent(fileName, fileSize) + return t.Encode() +} + +// CopyAndHash 从 reader 读取数据，同时写入 writer 和 HashWriter +func CopyAndHash(dst io.Writer, src io.Reader, hw *HashWriter) (int64, error) { + buf := make([]byte, 32*1024) // 32KB buffer + var written int64 + for { + nr, er := src.Read(buf) + if nr > 0 { + // 写入 HashWriter + hw.Write(buf[:nr]) + // 写入目标 + if dst != nil { + nw, ew := dst.Write(buf[:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = fmt.Errorf("invalid write result") + } + } + written += int64(nw) + if ew != nil { + return written, ew + } + if nr != nw { + return written, io.ErrShortWrite + } + } else { + written += int64(nr) + } + } + if er != nil { + if er == io.EOF { + break + } + return written, er + } + } + return written, nil +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/pkg/torrent/torrent.go b/pkg/torrent/torrent.go new file mode 100644 index 0000000000..8744e6362e --- /dev/null +++ b/pkg/torrent/torrent.go @@ -0,0 +1,439 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "time" +) + +const ( + // DefaultPieceSize 默认分片大小 10MB，与天翼云 CAS 分片大小一致 + DefaultPieceSize int64 = 10 * 1024 * 1024 + + // CASExtensionKey torrent 根字典中的 CAS 扩展 key + CASExtensionKey = "x-cas" + + // CASSliceSizeKey CAS 分片大小 key + CASSliceSizeKey = "slice_size" + // CASSliceMD5sKey 每片 MD5 列表 key + CASSliceMD5sKey = "slice_md5s" + // CASSliceMD5Key 最终 sliceMd5 key + CASSliceMD5Key = "slice_md5" + // CASFileMD5Key 整文件 MD5 key + CASFileMD5Key = "file_md5" + // CASCloudKey 云盘类型 key + CASCloudKey = "cloud" +) + +// CASInfo 天翼云 CAS 秒传所需信息 +type CASInfo struct { + // FileMD5 整文件 MD5（大写十六进制） + FileMD5 string + // SliceMD5 分片 MD5 的摘要（大写十六进制） + SliceMD5 string + // SliceMD5s 每个 10MB 分片的 MD5（大写十六进制） + SliceMD5s []string + // SliceSize 分片大小（字节） + SliceSize int64 + // Cloud 云盘类型标识 + Cloud string +} + +// TorrentFile 表示 torrent 中的单个文件 +type TorrentFile struct { + // Length 文件大小（字节） + Length int64 + // Path 文件路径（多文件模式下的相对路径各段） + Path []string + // MD5Sum 文件的 MD5（可选，BT 标准字段） + MD5Sum string +} + +// TorrentInfo torrent 的 info 字典 +type TorrentInfo struct { + // PieceLength 分片大小 + PieceLength int64 + // Pieces 所有分片的 SHA-1 哈希拼接（每 20 字节一个） + Pieces []byte + // Name 种子名称（单文件模式为文件名，多文件模式为目录名） + Name string + // Length 单文件模式下的文件大小 + Length int64 + // Files 多文件模式下的文件列表 + Files []TorrentFile + // MD5Sum 单文件模式下的文件 MD5（可选） + MD5Sum string +} + +// Torrent 完整的 torrent 文件结构 +type Torrent struct { + // Info info 字典 + Info TorrentInfo + // InfoHash info 字典的 SHA-1 哈希（20 字节） + InfoHash []byte + // Announce tracker URL + Announce string + // AnnounceList tracker 列表 + AnnounceList [][]string + // CreationDate 创建时间 + CreationDate int64 + // Comment 注释 + Comment string + // CreatedBy 创建者 + CreatedBy string + // CAS 天翼云 CAS 扩展信息（存储在 info 字典外部，不影响 info_hash） + CAS *CASInfo +} + +// NewTorrent 创建一个新的 torrent 结构 +func NewTorrent(name string, fileSize int64, fileMD5 string) *Torrent { + return &Torrent{ + Info: TorrentInfo{ + PieceLength: DefaultPieceSize, + Name: name, + Length: fileSize, + MD5Sum: fileMD5, + }, + CreationDate: time.Now().Unix(), + CreatedBy: "OpenList", + Comment: "Generated by OpenList with CAS extension", + } +} + +// SetPieces 设置 SHA-1 分片哈希 +func (t *Torrent) SetPieces(pieces []byte) { + t.Info.Pieces = pieces +} + +// SetCASInfo 设置 CAS 扩展信息 +func (t *Torrent) SetCASInfo(cas *CASInfo) { + t.CAS = cas +} + +// Encode 将 torrent 编码为 bencode 格式的字节数组 +func (t *Torrent) Encode() ([]byte, error) { + // 构建 info 字典 + infoDict := make(map[string]interface{}) + infoDict["piece length"] = int64(t.Info.PieceLength) + infoDict["pieces"] = t.Info.Pieces + infoDict["name"] = t.Info.Name + + if len(t.Info.Files) > 0 { + // 多文件模式 + files := make([]interface{}, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + fileDict := make(map[string]interface{}) + fileDict["length"] = int64(f.Length) + path := make([]interface{}, 0, len(f.Path)) + for _, p := range f.Path { + path = append(path, p) + } + fileDict["path"] = path + if f.MD5Sum != "" { + fileDict["md5sum"] = f.MD5Sum + } + files = append(files, fileDict) + } + infoDict["files"] = files + } else { + // 单文件模式 + infoDict["length"] = int64(t.Info.Length) + if t.Info.MD5Sum != "" { + infoDict["md5sum"] = t.Info.MD5Sum + } + } + + // 编码 info 字典并计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info dict: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 构建根字典 + rootDict := make(map[string]interface{}) + if t.Announce != "" { + rootDict["announce"] = t.Announce + } + if len(t.AnnounceList) > 0 { + announceList := make([]interface{}, 0, len(t.AnnounceList)) + for _, tier := range t.AnnounceList { + tierList := make([]interface{}, 0, len(tier)) + for _, url := range tier { + tierList = append(tierList, url) + } + announceList = append(announceList, tierList) + } + rootDict["announce-list"] = announceList + } + if t.Comment != "" { + rootDict["comment"] = t.Comment + } + if t.CreatedBy != "" { + rootDict["created by"] = t.CreatedBy + } + if t.CreationDate > 0 { + rootDict["creation date"] = t.CreationDate + } + + // info 字典使用原始编码的字节（保证 info_hash 一致） + rootDict["info"] = infoDict + + // CAS 扩展信息（放在 info 外部，不影响 info_hash） + if t.CAS != nil { + casDict := make(map[string]interface{}) + casDict[CASCloudKey] = t.CAS.Cloud + casDict[CASFileMD5Key] = t.CAS.FileMD5 + casDict[CASSliceMD5Key] = t.CAS.SliceMD5 + casDict[CASSliceSizeKey] = t.CAS.SliceSize + + if len(t.CAS.SliceMD5s) > 0 { + md5List := make([]interface{}, 0, len(t.CAS.SliceMD5s)) + for _, md5 := range t.CAS.SliceMD5s { + md5List = append(md5List, md5) + } + casDict[CASSliceMD5sKey] = md5List + } + rootDict[CASExtensionKey] = casDict + } + + return BencodeEncode(rootDict) +} + +// Decode 从 bencode 字节数组解析 torrent +func Decode(data []byte) (*Torrent, error) { + val, err := BencodeDecode(data) + if err != nil { + return nil, fmt.Errorf("bencode decode: %w", err) + } + + rootDict, ok := val.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: root is not a dict") + } + + t := &Torrent{} + + // 解析 announce + if v, ok := rootDict["announce"]; ok { + if b, ok := v.([]byte); ok { + t.Announce = string(b) + } + } + + // 解析 announce-list + if v, ok := rootDict["announce-list"]; ok { + if list, ok := v.([]interface{}); ok { + for _, tier := range list { + if tierList, ok := tier.([]interface{}); ok { + var urls []string + for _, u := range tierList { + if b, ok := u.([]byte); ok { + urls = append(urls, string(b)) + } + } + if len(urls) > 0 { + t.AnnounceList = append(t.AnnounceList, urls) + } + } + } + } + } + + // 解析 comment + if v, ok := rootDict["comment"]; ok { + if b, ok := v.([]byte); ok { + t.Comment = string(b) + } + } + + // 解析 created by + if v, ok := rootDict["created by"]; ok { + if b, ok := v.([]byte); ok { + t.CreatedBy = string(b) + } + } + + // 解析 creation date + if v, ok := rootDict["creation date"]; ok { + if n, ok := v.(int64); ok { + t.CreationDate = n + } + } + + // 解析 info 字典 + if infoVal, ok := rootDict["info"]; ok { + infoDict, ok := infoVal.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: info is not a dict") + } + + // 计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info for hash: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 解析 info 字段 + if v, ok := infoDict["piece length"]; ok { + if n, ok := v.(int64); ok { + t.Info.PieceLength = n + } + } + if v, ok := infoDict["pieces"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Pieces = b + } + } + if v, ok := infoDict["name"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Name = string(b) + } + } + if v, ok := infoDict["length"]; ok { + if n, ok := v.(int64); ok { + t.Info.Length = n + } + } + if v, ok := infoDict["md5sum"]; ok { + if b, ok := v.([]byte); ok { + t.Info.MD5Sum = string(b) + } + } + + // 解析多文件模式 + if v, ok := infoDict["files"]; ok { + if files, ok := v.([]interface{}); ok { + for _, f := range files { + if fileDict, ok := f.(map[string]interface{}); ok { + tf := TorrentFile{} + if l, ok := fileDict["length"]; ok { + if n, ok := l.(int64); ok { + tf.Length = n + } + } + if p, ok := fileDict["path"]; ok { + if pathList, ok := p.([]interface{}); ok { + for _, pp := range pathList { + if b, ok := pp.([]byte); ok { + tf.Path = append(tf.Path, string(b)) + } + } + } + } + if m, ok := fileDict["md5sum"]; ok { + if b, ok := m.([]byte); ok { + tf.MD5Sum = string(b) + } + } + t.Info.Files = append(t.Info.Files, tf) + } + } + } + } + } + + // 解析 CAS 扩展 + if casVal, ok := rootDict[CASExtensionKey]; ok { + if casDict, ok := casVal.(map[string]interface{}); ok { + cas := &CASInfo{} + if v, ok := casDict[CASCloudKey]; ok { + if b, ok := v.([]byte); ok { + cas.Cloud = string(b) + } + } + if v, ok := casDict[CASFileMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.FileMD5 = string(b) + } + } + if v, ok := casDict[CASSliceMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.SliceMD5 = string(b) + } + } + if v, ok := casDict[CASSliceSizeKey]; ok { + if n, ok := v.(int64); ok { + cas.SliceSize = n + } + } + if v, ok := casDict[CASSliceMD5sKey]; ok { + if list, ok := v.([]interface{}); ok { + for _, item := range list { + if b, ok := item.([]byte); ok { + cas.SliceMD5s = append(cas.SliceMD5s, string(b)) + } + } + } + } + t.CAS = cas + } + } + + return t, nil +} + +// GetInfoHashHex 获取 info_hash 的十六进制字符串 +func (t *Torrent) GetInfoHashHex() string { + return hex.EncodeToString(t.InfoHash) +} + +// GetPieceHashes 获取所有分片的 SHA-1 哈希（每个 20 字节） +func (t *Torrent) GetPieceHashes() [][]byte { + if len(t.Info.Pieces) == 0 { + return nil + } + count := len(t.Info.Pieces) / 20 + hashes := make([][]byte, count) + for i := 0; i < count; i++ { + hashes[i] = t.Info.Pieces[i*20 : (i+1)*20] + } + return hashes +} + +// GetTotalSize 获取 torrent 中所有文件的总大小 +func (t *Torrent) GetTotalSize() int64 { + if len(t.Info.Files) > 0 { + var total int64 + for _, f := range t.Info.Files { + total += f.Length + } + return total + } + return t.Info.Length +} + +// HasCASInfo 检查 torrent 是否包含 CAS 扩展信息 +func (t *Torrent) HasCASInfo() bool { + return t.CAS != nil && t.CAS.FileMD5 != "" && t.CAS.SliceMD5 != "" +} + +// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息 +func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo { + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + // 所有分片 MD5 用 \n 拼接后再取 MD5 + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + return &CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + } +} + +// GetMD5Str 计算字符串的 MD5（大写十六进制） +func GetMD5Str(data string) string { + h := md5.New() + h.Write([]byte(data)) + return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) +} diff --git a/server/handles/torrent.go b/server/handles/torrent.go new file mode 100644 index 0000000000..8b6ee1b6bf --- /dev/null +++ b/server/handles/torrent.go @@ -0,0 +1,433 @@ +package handles + +import ( + "encoding/base64" + "fmt" + "io" + "strings" + + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +// maxTorrentBase64Len is the max allowed Base64-encoded torrent size (~10MB decoded) +const maxTorrentBase64Len = 14 * 1024 * 1024 + +// maxTorrentGenFileSize is the max file size allowed for synchronous torrent generation (1GB) +const maxTorrentGenFileSize = 1 * 1024 * 1024 * 1024 + +// validateParsedTorrent checks that basic torrent invariants hold. +func validateParsedTorrent(t *torrent.Torrent) error { + if len(t.Info.Pieces)%20 != 0 { + return fmt.Errorf("torrent pieces 数据无效：长度必须为 20 的整数倍") + } + return nil +} + +// ParseTorrentReq 解析 torrent 文件请求 +type ParseTorrentReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` +} + +// ParseTorrentResp 解析 torrent 文件响应 +type ParseTorrentResp struct { + // Name 种子名称 + Name string `json:"name"` + // TotalSize 总大小 + TotalSize int64 `json:"total_size"` + // PieceLength 分片大小 + PieceLength int64 `json:"piece_length"` + // PieceCount 分片数量 + PieceCount int `json:"piece_count"` + // InfoHash info_hash（十六进制） + InfoHash string `json:"info_hash"` + // Files 文件列表（多文件模式） + Files []TorrentFileInfo `json:"files"` + // HasCAS 是否包含 CAS 扩展信息 + HasCAS bool `json:"has_cas"` + // CAS CAS 扩展信息 + CAS *CASInfoResp `json:"cas,omitempty"` +} + +// TorrentFileInfo torrent 中的文件信息 +type TorrentFileInfo struct { + Path string `json:"path"` + Size int64 `json:"size"` +} + +// CASInfoResp CAS 信息响应 +type CASInfoResp struct { + FileMD5 string `json:"file_md5"` + SliceMD5 string `json:"slice_md5"` + SliceSize int64 `json:"slice_size"` + Cloud string `json:"cloud"` +} + +// ParseTorrent 解析 torrent 文件，返回文件列表等信息 +func ParseTorrent(c *gin.Context) { + var req ParseTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + // 限制 Base64 输入大小（最大 ~10MB decoded） + if len(req.TorrentData) > maxTorrentBase64Len { + common.ErrorResp(c, fmt.Errorf("torrent 数据过大（最大 10MB）"), 400) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + // 单文件模式 + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + common.SuccessResp(c, resp) +} + +// TorrentRapidUploadReq 从 torrent 秒传请求 +type TorrentRapidUploadReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` + // Path 目标路径 + Path string `json:"path" binding:"required"` +} + +// TorrentRapidUpload 从 torrent 文件中提取 CAS 信息尝试秒传到天翼云 +func TorrentRapidUpload(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req TorrentRapidUploadReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + + if !t.HasCASInfo() { + common.ErrorResp(c, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传"), 400) + return + } + + // 获取目标存储 + storage, dstDirActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // 获取目标目录对象 + dstDir, err := op.Get(c.Request.Context(), storage, dstDirActualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取目标目录失败: %w", err), 500) + return + } + if !dstDir.IsDir() { + common.ErrorResp(c, errs.NotFolder, 400) + return + } + + // 检查是否是天翼云 PC 驱动 + cloud189PC, ok := storage.(*_189pc.Cloud189PC) + if !ok { + common.ErrorResp(c, fmt.Errorf("目标存储不是天翼云PC驱动，不支持 CAS 秒传"), 400) + return + } + + // 尝试秒传 + obj, err := cloud189PC.RapidUploadFromTorrent(c.Request.Context(), dstDir, torrentData, true) + if err != nil { + common.ErrorResp(c, fmt.Errorf("秒传失败: %w", err), 400) + return + } + + common.SuccessResp(c, gin.H{ + "message": "秒传成功", + "file_name": obj.GetName(), + "file_size": obj.GetSize(), + }) +} + +// UploadTorrentAndParse 通过文件上传方式解析 torrent +func UploadTorrentAndParse(c *gin.Context) { + file, err := c.FormFile("torrent") + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取上传文件失败: %w", err), 400) + return + } + + // 限制文件大小（最大 10MB） + if file.Size > 10*1024*1024 { + common.ErrorResp(c, fmt.Errorf("torrent 文件过大（最大 10MB）"), 400) + return + } + + f, err := file.Open() + if err != nil { + common.ErrorResp(c, fmt.Errorf("打开文件失败: %w", err), 500) + return + } + defer f.Close() + + torrentData, err := io.ReadAll(f) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + // 同时返回 Base64 编码的 torrent 数据，方便后续使用 + common.SuccessResp(c, gin.H{ + "info": resp, + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + }) +} + +// GenerateTorrentReq 为指定路径的文件生成 torrent 请求 +type GenerateTorrentReq struct { + // Path 文件在 OpenList 中的路径 + Path string `json:"path" binding:"required"` + // WithCAS 是否注入 CAS 扩展信息（仅天翼云需要） + WithCAS bool `json:"with_cas"` +} + +// GenerateTorrentForPath 为指定路径的文件生成 torrent +// 这是一个通用接口，适用于所有驱动 +// 会获取文件内容计算哈希，然后生成 torrent +func GenerateTorrentForPath(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req GenerateTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查读取权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanRead(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // 获取存储和文件信息 + storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // with_cas 仅支持天翼云PC驱动 + if req.WithCAS { + if _, is189pc := storage.(*_189pc.Cloud189PC); !is189pc { + common.ErrorResp(c, fmt.Errorf("CAS 秒传扩展仅支持天翼云PC驱动"), 400) + return + } + } + + // 获取文件对象 + obj, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件失败: %w", err), 500) + return + } + if obj.IsDir() { + common.ErrorResp(c, fmt.Errorf("不支持为目录生成 torrent"), 400) + return + } + + // 限制可生成 torrent 的文件大小 + if obj.GetSize() > maxTorrentGenFileSize { + common.ErrorResp(c, fmt.Errorf("文件过大，无法生成 torrent（最大 1GB）"), 400) + return + } + + // 获取文件下载链接 + link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件链接失败: %w", err), 500) + return + } + defer link.Close() + + // 通过 RangeReader 获取文件内容并计算哈希生成 torrent + if link.RangeReader == nil { + common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取，无法生成 torrent（请先下载文件到本地）"), 400) + return + } + + // 读取整个文件 + rc, err := link.RangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + defer rc.Close() + + var torrentData []byte + if req.WithCAS { + torrentData, err = torrent.GenerateFromReaderWithCAS(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } else { + torrentData, err = torrent.GenerateFromReader(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } + if err != nil { + common.ErrorResp(c, fmt.Errorf("生成 torrent 失败: %w", err), 500) + return + } + + // 解析生成的 torrent 获取 info_hash + t, _ := torrent.Decode(torrentData) + var infoHash string + if t != nil { + infoHash = t.GetInfoHashHex() + } + + common.SuccessResp(c, gin.H{ + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + "info_hash": infoHash, + "file_name": obj.GetName() + ".torrent", + "size": len(torrentData), + "with_cas": req.WithCAS, + }) +} diff --git a/server/router.go b/server/router.go index 57d1166ae7..8e90688248 100644 --- a/server/router.go +++ b/server/router.go @@ -217,6 +217,11 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Torrent 相关接口 + g.POST("/torrent/parse", handles.ParseTorrent) + g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse) + g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) + g.POST("/torrent/generate", handles.GenerateTorrentForPath) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } From ffbbc1ccdf3165a5740ebfeca8422c6540cd1156 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Mon, 25 May 2026 22:10:53 +0800 Subject: [PATCH 013/107] fix(115): fix capacity display and CDN 403 errors (#2510) - Fix #2343: update 115driver to v1.3.3 to handle float size values - Fix #2349, #2356, #2502: use base.UserAgent as fallback when User-Agent header is empty The 115 CDN returns 403 "no cookie value" when the User-Agent header is empty or doesn't match the cookie. This fix ensures a consistent browser User-Agent is used for download link generation. --- drivers/115/driver.go | 4 ++++ drivers/115_share/driver.go | 9 +++++++-- go.mod | 10 +++++----- go.sum | 22 ++++++++++++---------- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index d4f5741d0a..7c7c581a8f 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -5,6 +5,7 @@ import ( "strings" "sync" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -68,6 +69,9 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") + if userAgent == "" { + userAgent = base.UserAgent + } downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index fe8b7733a4..8f8cac01e1 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -2,6 +2,7 @@ package _115_share import ( "context" + "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -96,8 +97,12 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err != nil { return nil, err } - - return &model.Link{URL: downloadInfo.URL.URL}, nil + header := http.Header{} + header.Set("User-Agent", ua) + return &model.Link{ + URL: downloadInfo.URL.URL, + Header: header, + }, nil } func (d *Pan115Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/go.mod b/go.mod index 1df78f4b22..098a75591c 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 github.com/ProtonMail/gopenpgp/v2 v2.9.0 - github.com/SheltonZhu/115driver v1.2.3 + github.com/SheltonZhu/115driver v1.3.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.5 github.com/antchfx/xpath v1.3.5 @@ -67,9 +67,9 @@ require ( github.com/rclone/rclone v1.70.3 github.com/shirou/gopsutil/v4 v4.25.5 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/afero v1.14.0 - github.com/spf13/cobra v1.9.1 - github.com/stretchr/testify v1.10.0 + github.com/spf13/afero v1.15.0 + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 github.com/tchap/go-patricia/v2 v2.3.3 github.com/u2takey/ffmpeg-go v0.5.0 @@ -284,7 +284,7 @@ require ( github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 4ec25fe0b3..8ff7c863db 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2 github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= -github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= +github.com/SheltonZhu/115driver v1.3.3 h1:Bqs86D2MziYPgIOuOJF+HzG4d7GBr71ZhSCrs/U17UU= +github.com/SheltonZhu/115driver v1.3.3/go.mod h1:OujS7azslg1/bn85sPSHnNsp4/WBI9/TiijtZL9kuSQ= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -612,12 +612,13 @@ github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBD github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -632,8 +633,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= @@ -692,6 +693,7 @@ go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= From b28208bd76ee19c49dabc7f68c1f4e863f571fc6 Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Mon, 25 May 2026 22:13:44 +0800 Subject: [PATCH 014/107] fix(189pc): handle numeric res_code in RenameResp to fix JSON unmarshal error (#2489) --- drivers/189pc/driver.go | 2 +- drivers/189pc/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 104832f74d..82aa1c1af5 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -293,7 +293,7 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin req.SetContext(ctx).SetQueryParams(queryParam) }, nil, resp, isFamily) if err != nil { - if resp.ResCode == "FileAlreadyExists" { + if code, ok := resp.ResCode.(string); ok && code == "FileAlreadyExists" { return nil, errs.ObjectAlreadyExists } return nil, err diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index eed447e25b..c05e3867f4 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -441,7 +441,7 @@ type RenameResp struct { ParentID int64 `json:"parentId"` Rev string `json:"rev"` Size int64 `json:"size"` - ResCode string `json:"res_code"` + ResCode any `json:"res_code"` // int or string } func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { From 3b1760e95965b89d84f09cbdb97f9e6682b0de96 Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Tue, 26 May 2026 13:11:21 +0800 Subject: [PATCH 015/107] fix(offline_download): restore SimpleHttp fallback for http links (#2516) - Fixed a regression where SimpleHttp returned the same unsupported scheme error for normal HTTP and HTTPS links. --- internal/offline_download/tool/add.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index e42b08c31c..33128ccc2c 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -69,13 +69,14 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } // try putting url if args.Tool == "SimpleHttp" { + if isSimpleHttpSchemeUnsupported(args.URL) { + return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") + } err = tryPutUrl(ctx, args.DstDirPath, args.URL) if err == nil || !errors.Is(err, errs.NotImplement) { return nil, err } - // SimpleHttp 不支持非 HTTP/HTTPS 协议（如 magnet、ed2k 等） - // tryPutUrl 返回 NotImplement 说明 URL 不是 HTTP/HTTPS - return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") + // Fallback to creating a download task when storage lacks native PutURL support. } // ed2k 链接自动路由：如果当前工具不支持 ed2k，自动尝试使用迅雷系工具 @@ -184,10 +185,6 @@ func tryPutUrl(ctx context.Context, path, urlStr string) error { var dstName string u, err := url.Parse(urlStr) if err == nil { - // 只支持 HTTP/HTTPS 协议，其他协议（magnet、ed2k 等）返回 NotImplement - if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" { - return errors.WithStack(errs.NotImplement) - } dstName = stdpath.Base(u.Path) } else { dstName = "UnnamedURL" @@ -195,6 +192,15 @@ func tryPutUrl(ctx context.Context, path, urlStr string) error { return fs.PutURL(ctx, path, dstName, urlStr) } +func isSimpleHttpSchemeUnsupported(urlStr string) bool { + u, err := url.Parse(strings.TrimSpace(urlStr)) + if err != nil || u.Scheme == "" { + return false + } + scheme := strings.ToLower(u.Scheme) + return scheme != "http" && scheme != "https" +} + // isEd2kURL 检测 URL 是否为 ed2k 协议 func isEd2kURL(urlStr string) bool { return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") From 4c77b9c7bf13bfa404ec547c0420dc86a7c4d83a Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Wed, 27 May 2026 13:26:16 +0800 Subject: [PATCH 016/107] fix(fs): clear skipped names before copy and move tasks (#2520) --- server/handles/fsmanage.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 52bdb20bf1..d97d36d24a 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -119,6 +119,7 @@ func FsMove(c *gin.Context) { req.Names[i] = "" continue } + req.Names[i] = srcPath if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { @@ -129,12 +130,11 @@ func FsMove(c *gin.Context) { if !req.SkipExisting { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return - } else { - continue } + req.Names[i] = "" + continue } } - req.Names[i] = srcPath } // Create all tasks immediately without any synchronous validation @@ -222,6 +222,7 @@ func FsCopy(c *gin.Context) { req.Names[i] = "" continue } + req.Names[i] = srcPath if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { @@ -233,11 +234,11 @@ func FsCopy(c *gin.Context) { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } else if !req.Merge || !res.IsDir() { + req.Names[i] = "" continue } } } - req.Names[i] = srcPath } // Create all tasks immediately without any synchronous validation From 98c32d3be4dde70b71f2a30720e5d914d85cf9dc Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Wed, 27 May 2026 16:49:26 +0800 Subject: [PATCH 017/107] fix(doubao_new): remove temp file after cross-storage put (#2530) --- drivers/doubao_new/driver.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go index 2a90cf16cb..f84828f0e1 100644 --- a/drivers/doubao_new/driver.go +++ b/drivers/doubao_new/driver.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "net/url" + "os" "sort" "strings" "sync" @@ -300,7 +301,10 @@ func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileSt if err != nil { return nil, err } - defer tmpFile.Close() + defer func() { + _ = tmpFile.Close() + _ = os.Remove(tmpFile.Name()) + }() blockSize := uploadPrep.BlockSize totalSize := file.GetSize() From 9cc5dd969b9833c8cb4e14c338c3571dfdbe2108 Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Wed, 27 May 2026 17:16:46 +0800 Subject: [PATCH 018/107] fix(offline_download): block SimpleHttp temp file path traversal via strict filename sanitization * fix(offline_download): prevent path traversal * fix(SimpleHttp): improve filename validation * fix(offline_download): harden SimpleHttp filename and temp path checks * simplify filename validation --------- Co-authored-by: ILoveScratch2 Co-authored-by: j2rong4cn --- internal/offline_download/http/client.go | 17 +++++++++++------ internal/offline_download/http/util.go | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go index 5dcb6d5902..86314031d1 100644 --- a/internal/offline_download/http/client.go +++ b/internal/offline_download/http/client.go @@ -5,7 +5,6 @@ import ( "math/rand/v2" "net/http" "os" - "path" "path/filepath" "strings" "time" @@ -73,11 +72,11 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { } filename, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")) if err != nil { - filename = path.Base(resp.Request.URL.Path) + filename, err = sanitizeFilename(resp.Request.URL.Path) } - filename = strings.Trim(filename, "/") - if len(filename) == 0 { - filename = fmt.Sprintf("%s-%d-%x", strings.ReplaceAll(req.URL.Host, ".", "_"), time.Now().UnixMilli(), rand.Uint32()) + if err != nil { + filename = strings.ReplaceAll(req.URL.Host, ":", "_") + filename = fmt.Sprintf("%s-%d-%x", filename, time.Now().UnixMilli(), rand.Uint32()) } fileSize := resp.ContentLength if streamPut { @@ -91,8 +90,14 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { } task.SetTotalBytes(fileSize) // save to temp dir - _ = os.MkdirAll(task.TempDir, os.ModePerm) + if err := os.MkdirAll(task.TempDir, os.ModePerm); err != nil { + return err + } filePath := filepath.Join(task.TempDir, filename) + cleanTempDir := filepath.Clean(task.TempDir) + string(filepath.Separator) + if !strings.HasPrefix(filepath.Clean(filePath)+string(filepath.Separator), cleanTempDir) { + return fmt.Errorf("filename illegal") + } file, err := os.Create(filePath) if err != nil { return err diff --git a/internal/offline_download/http/util.go b/internal/offline_download/http/util.go index eefefec24a..9622c58bed 100644 --- a/internal/offline_download/http/util.go +++ b/internal/offline_download/http/util.go @@ -3,8 +3,22 @@ package http import ( "fmt" "mime" + "path/filepath" + "strings" ) +func sanitizeFilename(raw string) (string, error) { + raw = strings.TrimSpace(raw) + sep := string(filepath.Separator) + raw = strings.ReplaceAll(raw, "/", sep) + raw = strings.ReplaceAll(raw, "\\", sep) + filename := filepath.Base(filepath.Clean(raw)) + if filename == "." || filename == sep || !filepath.IsLocal(filename) { + return "", fmt.Errorf("invalid filename: %q", raw) + } + return filename, nil +} + func parseFilenameFromContentDisposition(contentDisposition string) (string, error) { if contentDisposition == "" { return "", fmt.Errorf("Content-Disposition is empty") @@ -14,8 +28,15 @@ func parseFilenameFromContentDisposition(contentDisposition string) (string, err return "", err } filename := params["filename"] + if filename == "" { + filename = params["filename*"] + } if filename == "" { return "", fmt.Errorf("filename not found in Content-Disposition: [%s]", contentDisposition) } + filename, err = sanitizeFilename(filename) + if err != nil { + return "", fmt.Errorf("invalid filename in Content-Disposition: [%s]", contentDisposition) + } return filename, nil } From ba6f96ca4ef9cbb8950e682389943dafa56accfa Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Sat, 30 May 2026 10:36:37 +0800 Subject: [PATCH 019/107] fix(driver): json: cannot unmarshal number into Go struct field RenameResp.id of type string (#2526) --- drivers/189pc/types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index c05e3867f4..7952cb78a7 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -432,7 +432,7 @@ type RenameResp struct { ResMsg string `json:"res_message"` CreateDate Time `json:"createDate"` FileCate int `json:"fileCata"` - ID string `json:"id"` + ID String `json:"id"` LastOpTime Time `json:"lastOpTime"` MD5 string `json:"md5"` MediaType int `json:"mediaType"` @@ -446,7 +446,7 @@ type RenameResp struct { func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { return &Cloud189File{ - ID: String(r.ID), + ID: r.ID, Name: r.Name, Size: r.Size, Md5: r.MD5, @@ -458,7 +458,7 @@ func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { func (r *RenameResp) toFolder() *Cloud189Folder { return &Cloud189Folder{ - ID: String(r.ID), + ID: r.ID, Name: r.Name, ParentID: r.ParentID, LastOpTime: r.LastOpTime, From cd6643a642d89075201ff0437197ec2c6ba8711b Mon Sep 17 00:00:00 2001 From: yuxiaoyu8192 <103018181+yuxiaoyu8192@users.noreply.github.com> Date: Sun, 31 May 2026 14:54:20 +0800 Subject: [PATCH 020/107] fix(chaoxing): support multiple uploadDate formats (#2545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update types.go Signed-off-by: yuxiaoyu8192 <103018181+yuxiaoyu8192@users.noreply.github.com> * Update types.go 代码格式化 Signed-off-by: yuxiaoyu8192 <103018181+yuxiaoyu8192@users.noreply.github.com> * Update types.go 时间解析失败返回空 Signed-off-by: yuxiaoyu8192 <103018181+yuxiaoyu8192@users.noreply.github.com> --------- Signed-off-by: yuxiaoyu8192 <103018181+yuxiaoyu8192@users.noreply.github.com> --- drivers/chaoxing/types.go | 108 ++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index ca171c4d50..dcbded0d8a 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -105,44 +105,82 @@ func (ios *int_str) UnmarshalJSON(data []byte) error { return nil } +// 时间戳同样存在字符串/数字两种形式，且毫秒级时间戳会超出 int32 范围，需用 int64 解析 +// 超星 API 返回的 uploadDate 有三种格式： +// 1. 数字时间戳：1780191356415 +// 2. 字符串时间戳："1780191356415" +// 3. 日期字符串："2024-11-06 07:49" +type int64_str int64 + +func (ios *int64_str) UnmarshalJSON(data []byte) error { + // 去除引号 + str := string(bytes.Trim(data, "\"")) + + // 尝试直接解析为数字（处理格式 1 和 2） + intValue, err := strconv.ParseInt(str, 10, 64) + if err == nil { + *ios = int64_str(intValue) + return nil + } + + // 尝试解析日期字符串（处理格式 3） + // 支持格式："2024-11-06 07:49" 或 "2024-11-06 07:49:30" + layouts := []string{ + "2006-01-02 15:04", + "2006-01-02 15:04:05", + } + + for _, layout := range layouts { + if t, err := time.Parse(layout, str); err == nil { + // 转换为毫秒时间戳 + *ios = int64_str(t.UnixMilli()) + return nil + } + } + + // 所有格式都失败，返回 0（避免因时间格式问题导致整个解析失败） + *ios = 0 + return nil +} + type File struct { Cataid int `json:"cataid"` Cfid int `json:"cfid"` Content struct { - Cfid int `json:"cfid"` - Pid int `json:"pid"` - FolderName string `json:"folderName"` - ShareType int `json:"shareType"` - Preview string `json:"preview"` - Filetype string `json:"filetype"` - PreviewURL string `json:"previewUrl"` - IsImg bool `json:"isImg"` - ParentPath string `json:"parentPath"` - Icon string `json:"icon"` - Suffix string `json:"suffix"` - Duration int `json:"duration"` - Pantype string `json:"pantype"` - Puid int_str `json:"puid"` - Filepath string `json:"filepath"` - Crc string `json:"crc"` - Isfile bool `json:"isfile"` - Residstr string `json:"residstr"` - ObjectID string `json:"objectId"` - Extinfo string `json:"extinfo"` - Thumbnail string `json:"thumbnail"` - Creator int `json:"creator"` - ResTypeValue int `json:"resTypeValue"` - UploadDateFormat string `json:"uploadDateFormat"` - DisableOpt bool `json:"disableOpt"` - DownPath string `json:"downPath"` - Sort int `json:"sort"` - Topsort int `json:"topsort"` - Restype string `json:"restype"` - Size int_str `json:"size"` - UploadDate int64 `json:"uploadDate"` - FileSize string `json:"fileSize"` - Name string `json:"name"` - FileID string `json:"fileId"` + Cfid int `json:"cfid"` + Pid int `json:"pid"` + FolderName string `json:"folderName"` + ShareType int `json:"shareType"` + Preview string `json:"preview"` + Filetype string `json:"filetype"` + PreviewURL string `json:"previewUrl"` + IsImg bool `json:"isImg"` + ParentPath string `json:"parentPath"` + Icon string `json:"icon"` + Suffix string `json:"suffix"` + Duration int `json:"duration"` + Pantype string `json:"pantype"` + Puid int_str `json:"puid"` + Filepath string `json:"filepath"` + Crc string `json:"crc"` + Isfile bool `json:"isfile"` + Residstr string `json:"residstr"` + ObjectID string `json:"objectId"` + Extinfo string `json:"extinfo"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + ResTypeValue int `json:"resTypeValue"` + UploadDateFormat string `json:"uploadDateFormat"` + DisableOpt bool `json:"disableOpt"` + DownPath string `json:"downPath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + Restype string `json:"restype"` + Size int_str `json:"size"` + UploadDate int64_str `json:"uploadDate"` + FileSize string `json:"fileSize"` + Name string `json:"name"` + FileID string `json:"fileId"` } `json:"content"` CreatorID int `json:"creatorId"` DesID string `json:"des_id"` @@ -265,7 +303,7 @@ func fileToObj(f File) *model.Object { IsFolder: true, } } - paserTime := time.UnixMilli(f.Content.UploadDate) + paserTime := time.UnixMilli(int64(f.Content.UploadDate)) return &model.Object{ ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), Name: f.Content.Name, From ac80fe19bf471b64d611970e509a85af62579286 Mon Sep 17 00:00:00 2001 From: Akizon <64686663+Akizon77@users.noreply.github.com> Date: Sun, 31 May 2026 14:54:34 +0800 Subject: [PATCH 021/107] feat(drivers): add emby driver (#2270) * feat: add emby driver support with stream and download link methods * refactor: organize codes into separate files Signed-off-by: MadDogOwner * fix: resolve fileID Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner Co-authored-by: MadDogOwner --- drivers/all.go | 1 + drivers/emby/driver.go | 261 +++++++++++++++++++++++++++++++++++++++++ drivers/emby/meta.go | 30 +++++ drivers/emby/types.go | 41 +++++++ drivers/emby/util.go | 91 ++++++++++++++ 5 files changed, 424 insertions(+) create mode 100644 drivers/emby/driver.go create mode 100644 drivers/emby/meta.go create mode 100644 drivers/emby/types.go create mode 100644 drivers/emby/util.go diff --git a/drivers/all.go b/drivers/all.go index fb68d03950..4af88dc00c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" + _ "github.com/OpenListTeam/OpenList/v4/drivers/emby" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/ftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/github" diff --git a/drivers/emby/driver.go b/drivers/emby/driver.go new file mode 100644 index 0000000000..b2406aa6e7 --- /dev/null +++ b/drivers/emby/driver.go @@ -0,0 +1,261 @@ +package emby + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Emby struct { + model.Storage + Addition + + client *http.Client + token string + userID string +} + +func (d *Emby) Config() driver.Config { + return config +} + +func (d *Emby) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Emby) Init(ctx context.Context) error { + d.URL = strings.TrimRight(strings.TrimSpace(d.URL), "/") + if d.URL == "" { + return fmt.Errorf("url is required") + } + + if strings.TrimSpace(d.RootFolderID) == "" { + d.RootFolderID = "1" + } + + d.client = base.HttpClient + d.token = strings.TrimSpace(d.ApiKey) + d.userID = strings.TrimSpace(d.UserID) + + if d.token != "" { + if d.userID == "" { + return fmt.Errorf("user_id is required when api_key is set") + } + op.MustSaveDriverStorage(d) + return nil + } + + if strings.TrimSpace(d.Username) == "" || strings.TrimSpace(d.Password) == "" { + return fmt.Errorf("please provide api_key+user_id or username+password") + } + + if err := d.login(ctx); err != nil { + return err + } + + d.ApiKey = d.token + d.UserID = d.userID + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Emby) Drop(ctx context.Context) error { + return nil +} + +func (d *Emby) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentID := strings.TrimSpace(d.RootFolderID) + if dir != nil && strings.TrimSpace(dir.GetID()) != "" { + parentID = strings.TrimSpace(dir.GetID()) + } + + items, err := d.getItems(ctx, parentID) + if err != nil { + return nil, err + } + + parentPath := "/" + if dir != nil && strings.TrimSpace(dir.GetPath()) != "" { + parentPath = dir.GetPath() + } + + objs := make([]model.Obj, 0, len(items.Items)) + for _, it := range items.Items { + modified := time.Now() + if it.DateCreated != "" { + if t, parseErr := time.Parse(time.RFC3339Nano, it.DateCreated); parseErr == nil { + modified = t + } + } + + name := strings.TrimSpace(it.Name) + id := strings.TrimSpace(it.ID) + displayName := name + if name != "" && id != "" { + if it.IsFolder { + displayName = fmt.Sprintf("%s (ID%s)", name, id) + } else { + ext := path.Ext(strings.TrimSpace(it.Path)) + if ext == "" { + ext = path.Ext(name) + } + + base := strings.TrimSpace(strings.TrimSuffix(name, ext)) + episodeCode := "" + if m := episodeCodeRegexp.FindString(base); m != "" { + episodeCode = strings.ToUpper(m) + } else if it.ParentIndex > 0 && it.IndexNumber > 0 { + episodeCode = fmt.Sprintf("S%02dE%02d", it.ParentIndex, it.IndexNumber) + } + + title := strings.TrimSpace(base) + if episodeCode != "" { + title = strings.TrimSpace(episodeCodeRegexp.ReplaceAllString(title, "")) + title = strings.TrimSpace(strings.Trim(title, "-_:[]() ")) + } + + series := strings.TrimSpace(it.SeriesName) + if series == "" && episodeCode != "" { + if idx := strings.Index(title, " - "); idx > 0 { + series = strings.TrimSpace(title[:idx]) + title = strings.TrimSpace(title[idx+3:]) + } + } + + core := title + if series != "" { + if title == "" || strings.EqualFold(series, title) { + core = series + } else { + core = series + " " + title + } + } + if core == "" { + core = base + } + + if episodeCode != "" { + core = fmt.Sprintf("%s - [%s]", core, episodeCode) + } + if ext == "" { + displayName = fmt.Sprintf("%s (ID%s)", core, id) + } else { + displayName = fmt.Sprintf("%s (ID%s)%s", core, id, ext) + } + } + } + + obj := &model.Object{ + ID: id, + Name: displayName, + Path: path.Join(parentPath, displayName), + Size: it.Size, + Modified: modified, + IsFolder: it.IsFolder, + } + if it.IsFolder { + obj.Size = 0 + } + objs = append(objs, obj) + } + return objs, nil +} + +func (d *Emby) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := strings.TrimSpace(file.GetID()) + if fileID == "" { + return nil, fmt.Errorf("invalid file id") + } + + u, err := url.Parse(d.URL) + if err != nil { + return nil, err + } + linkMethod := strings.ToLower(strings.TrimSpace(d.LinkMethod)) + useDownload := linkMethod == "download" + + mediaSourceID := "" + mediaContainer := "" + if !useDownload { + detailURL, parseErr := url.Parse(d.URL + "/Users/" + d.userID + "/Items/" + fileID) + if parseErr == nil { + q := detailURL.Query() + q.Set("Fields", "MediaSources") + q.Set("api_key", d.token) + detailURL.RawQuery = q.Encode() + + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, detailURL.String(), nil) + if reqErr == nil { + resp, doErr := d.client.Do(req) + if doErr == nil { + func() { + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return + } + var detail itemDetailResp + if decodeErr := json.NewDecoder(resp.Body).Decode(&detail); decodeErr != nil || len(detail.MediaSources) == 0 { + return + } + for i := range detail.MediaSources { + if strings.TrimSpace(detail.MediaSources[i].ID) != "" && detail.MediaSources[i].SupportsDirectStream { + mediaSourceID = strings.TrimSpace(detail.MediaSources[i].ID) + mediaContainer = strings.TrimSpace(detail.MediaSources[i].Container) + return + } + } + for i := range detail.MediaSources { + if strings.TrimSpace(detail.MediaSources[i].ID) != "" { + mediaSourceID = strings.TrimSpace(detail.MediaSources[i].ID) + mediaContainer = strings.TrimSpace(detail.MediaSources[i].Container) + return + } + } + }() + } + } + } + } + + if useDownload { + u.Path = path.Join(u.Path, "/Items", fileID, "Download") + } else { + if mediaContainer != "" { + u.Path = path.Join(u.Path, "/Videos", fileID, "stream."+mediaContainer) + } else { + u.Path = path.Join(u.Path, "/Videos", fileID, "stream") + } + } + q := u.Query() + q.Set("api_key", d.token) + if mediaSourceID != "" { + q.Set("MediaSourceId", mediaSourceID) + } + if !useDownload { + q.Set("Static", "true") + } + u.RawQuery = q.Encode() + + return &model.Link{ + URL: u.String(), + Header: http.Header{ + "User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +var _ driver.Driver = (*Emby)(nil) diff --git a/drivers/emby/meta.go b/drivers/emby/meta.go new file mode 100644 index 0000000000..632c98b7a9 --- /dev/null +++ b/drivers/emby/meta.go @@ -0,0 +1,30 @@ +package emby + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootID + URL string `json:"url" required:"true"` + ApiKey string `json:"api_key"` + UserID string `json:"user_id"` + Username string `json:"username"` + Password string `json:"password"` + LinkMethod string `json:"link_method" type:"select" options:"stream,download" default:"stream"` +} + +var config = driver.Config{ + Name: "Emby", + LocalSort: true, + NoUpload: true, + DefaultRoot: "1", + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Emby{} + }) +} diff --git a/drivers/emby/types.go b/drivers/emby/types.go new file mode 100644 index 0000000000..5f05158296 --- /dev/null +++ b/drivers/emby/types.go @@ -0,0 +1,41 @@ +package emby + +type authReq struct { + Username string `json:"Username"` + Pw string `json:"Pw"` +} + +type authResp struct { + AccessToken string `json:"AccessToken"` + User struct { + ID string `json:"Id"` + } `json:"User"` +} + +type listResp struct { + Items []embyItem `json:"Items"` + TotalRecordCount int `json:"TotalRecordCount"` +} + +type embyItem struct { + Name string `json:"Name"` + ID string `json:"Id"` + Type string `json:"Type"` + Path string `json:"Path"` + SeriesName string `json:"SeriesName"` + IndexNumber int `json:"IndexNumber"` + ParentIndex int `json:"ParentIndexNumber"` + IsFolder bool `json:"IsFolder"` + Size int64 `json:"Size"` + DateCreated string `json:"DateCreated"` +} + +type itemDetailResp struct { + MediaSources []embyMediaSource `json:"MediaSources"` +} + +type embyMediaSource struct { + ID string `json:"Id"` + Container string `json:"Container"` + SupportsDirectStream bool `json:"SupportsDirectStream"` +} diff --git a/drivers/emby/util.go b/drivers/emby/util.go new file mode 100644 index 0000000000..788fb4be43 --- /dev/null +++ b/drivers/emby/util.go @@ -0,0 +1,91 @@ +package emby + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" +) + +var episodeCodeRegexp = regexp.MustCompile(`(?i)\bS\d{1,2}E\d{1,2}\b`) + +func (d *Emby) login(ctx context.Context) error { + payload, err := json.Marshal(authReq{ + Username: d.Username, + Pw: d.Password, + }) + if err != nil { + return err + } + + endpoint := d.URL + "/Users/AuthenticateByName" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Emby-Authorization", `MediaBrowser Client="OpenList", Device="OpenList", DeviceId="openlist-emby", Version="1.0.0"`) + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("emby auth failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var data authResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return err + } + if strings.TrimSpace(data.AccessToken) == "" || strings.TrimSpace(data.User.ID) == "" { + return fmt.Errorf("emby auth response missing access token or user id") + } + + d.token = data.AccessToken + d.userID = data.User.ID + return nil +} + +func (d *Emby) getItems(ctx context.Context, parentID string) (*listResp, error) { + u, err := url.Parse(d.URL + "/Users/" + d.userID + "/Items") + if err != nil { + return nil, err + } + q := u.Query() + q.Set("ParentId", parentID) + q.Set("Recursive", "false") + q.Set("Fields", "Path,Size,DateCreated,SeriesName,IndexNumber,ParentIndexNumber") + q.Set("api_key", d.token) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("emby list failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var data listResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, err + } + return &data, nil +} From 9282e8ef09b3661ff416667cc61f69c914ff6ec9 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 1 Jun 2026 01:50:20 +0800 Subject: [PATCH 022/107] fix(cache,mem): replace finalizers with runtime.AddCleanup (#2535) --- internal/hybrid_cache/hybrid_cache.go | 11 +++++------ internal/mem/utils.go | 14 ++++++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/internal/hybrid_cache/hybrid_cache.go b/internal/hybrid_cache/hybrid_cache.go index d80f768f18..c69147937e 100644 --- a/internal/hybrid_cache/hybrid_cache.go +++ b/internal/hybrid_cache/hybrid_cache.go @@ -18,6 +18,7 @@ type HybridCache struct { memoryOffset uint64 backingStore BackingStore backingOffset uint64 + cleanup runtime.Cleanup } // HybridCache本身是一个大的Block，支持分块成多个小的Block @@ -99,11 +100,15 @@ func (hc *HybridCache) initFileCache() error { if err != nil { return err } + hc.cleanup = runtime.AddCleanup(hc, func(file BackingStore) { + _ = file.Close() + }, file) hc.backingStore = file return nil } func (hc *HybridCache) Close() error { + hc.cleanup.Stop() var err error if hc.memoryStore != nil { err = hc.memoryStore.Free() @@ -228,12 +233,6 @@ func NewHybridCache(blockSize, maxMemorySize uint64) (hc *HybridCache, err error return nil, errors.Join(err, err2) } } - runtime.SetFinalizer(hc, func(hc *HybridCache) { - if hc.backingStore != nil { - _ = hc.backingStore.Close() - hc.backingStore = nil - } - }) return hc, nil } diff --git a/internal/mem/utils.go b/internal/mem/utils.go index 21944066de..b0a46d731f 100644 --- a/internal/mem/utils.go +++ b/internal/mem/utils.go @@ -58,15 +58,16 @@ func NewGuardedMemory(cap, max uint64) (m LinearMemory, err error) { if s, ok := m.(interface{ SetGrowCheck(GrowCheck) }); ok { s.SetGrowCheck(MemoryGrowCheck) } - gm := &guardedMemory{m} - runtime.SetFinalizer(gm, func(gm *guardedMemory) { - gm.Free() - }) + gm := &guardedMemory{LinearMemory: m} + gm.cleanup = runtime.AddCleanup(gm, func(m LinearMemory) { + m.Free() + }, m) return gm, nil } type guardedMemory struct { LinearMemory + cleanup runtime.Cleanup } func (s *guardedMemory) Reallocate(size uint64) (all []byte, err error) { @@ -77,3 +78,8 @@ func (s *guardedMemory) Reallocate(size uint64) (all []byte, err error) { }() return s.LinearMemory.Reallocate(size) } + +func (s *guardedMemory) Free() error { + s.cleanup.Stop() + return s.LinearMemory.Free() +} From 9b587ee16574d033b3ef9981cdbc0e5f8ae6afba Mon Sep 17 00:00:00 2001 From: Jealous Date: Mon, 1 Jun 2026 01:50:47 +0800 Subject: [PATCH 023/107] fix(archive): prefer utf-8 for non-EFS zip names (#2557) --- internal/archive/zip/utils.go | 4 +++ internal/archive/zip/utils_test.go | 41 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 internal/archive/zip/utils_test.go diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index 4f367a6319..e5d8fe20ef 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -5,6 +5,7 @@ import ( "io" "io/fs" "strings" + "unicode/utf8" "github.com/KirCute/zip" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -101,6 +102,9 @@ func decodeName(name string, efs bool) string { if efs { return name } + if utf8.ValidString(name) { + return name + } enc, err := ianaindex.IANA.Encoding(setting.GetStr(conf.NonEFSZipEncoding)) if err != nil { return name diff --git a/internal/archive/zip/utils_test.go b/internal/archive/zip/utils_test.go new file mode 100644 index 0000000000..40611a9fdc --- /dev/null +++ b/internal/archive/zip/utils_test.go @@ -0,0 +1,41 @@ +package zip + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/stretchr/testify/require" +) + +func setNonEFSZipEncoding(value string) { + op.Cache.SetSetting(conf.NonEFSZipEncoding, &model.SettingItem{ + Key: conf.NonEFSZipEncoding, + Value: value, + }) +} + +func TestDecodeNamePrefersValidUTF8WhenEFSDisabled(t *testing.T) { + setNonEFSZipEncoding("IBM437") + + name := "中文.txt" + require.Equal(t, name, decodeName(name, false)) + // Ensure the setting still exists to verify we did not bypass config due to missing setup. + require.Equal(t, "IBM437", setting.GetStr(conf.NonEFSZipEncoding)) +} + +func TestDecodeNameFallsBackToConfiguredEncoding(t *testing.T) { + setNonEFSZipEncoding("GB18030") + + name := string([]byte{0xd6, 0xd0, 0xce, 0xc4, '.', 't', 'x', 't'}) + require.Equal(t, "中文.txt", decodeName(name, false)) +} + +func TestDecodeNameRespectsEFSFlag(t *testing.T) { + setNonEFSZipEncoding("GB18030") + + name := "utf8-name.txt" + require.Equal(t, name, decodeName(name, true)) +} From 054db9f5ebb9fba150e4ec43e3325b200048c065 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Wed, 3 Jun 2026 19:56:55 +0800 Subject: [PATCH 024/107] fix(op): use oldName variable for cache updates in Rename function (#2570) --- internal/op/fs.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/op/fs.go b/internal/op/fs.go index f7b1c45b50..643e699709 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -460,6 +460,7 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) if model.ObjHasMask(srcRawObj, model.NoRename) { return errors.WithStack(errs.PermissionDenied) } + oldName := srcRawObj.GetName() srcObj := model.UnwrapObjName(srcRawObj) var newObj model.Obj @@ -477,19 +478,19 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) dirKey := Key(storage, stdpath.Dir(srcPath)) if !srcRawObj.IsDir() { - Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, oldName)) Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) } if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(dirKey); exist { if srcRawObj.IsDir() { - Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.deleteDirectoryTree(stdpath.Join(dirKey, oldName)) } if newObj == nil { newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } newObj = wrapObjName(storage, newObj) - cache.UpdateObject(srcRawObj.GetName(), newObj) + cache.UpdateObject(oldName, newObj) } } From 9737006a8d7513d03aae48be4322e7d8081db236 Mon Sep 17 00:00:00 2001 From: So <65939241+syscc@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:39:11 +0800 Subject: [PATCH 025/107] feat(onedrive_sharelink): support direct download redirects (#2591) fix(onedrive_sharelink): support direct download redirects Resolve SharePoint item metadata to obtain @content.downloadUrl for redirect downloads. Co-authored-by: syscc Co-authored-by: Codex --- drivers/onedrive_sharelink/driver.go | 108 ++++++++++++++++++++++++++- drivers/onedrive_sharelink/meta.go | 1 - drivers/onedrive_sharelink/types.go | 42 +++++++---- 3 files changed, 134 insertions(+), 17 deletions(-) diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 0bc3e79b53..2b205324c2 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -2,6 +2,7 @@ package onedrive_sharelink import ( "context" + "encoding/json" "fmt" "io" "net/http" @@ -21,7 +22,10 @@ import ( log "github.com/sirupsen/logrus" ) -const headerTTL = 25 * time.Minute +const ( + headerTTL = 25 * time.Minute + directLinkTTL = 20 * time.Minute +) type OnedriveSharelink struct { model.Storage @@ -99,6 +103,18 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model return nil, err } + if args.Redirect { + directURL, err := d.resolveDirectDownloadURL(ctx, file, url, header) + if err != nil { + return nil, err + } + expiration := directLinkTTL + return &model.Link{ + URL: directURL, + Expiration: &expiration, + }, nil + } + return &model.Link{ URL: url, Header: header, @@ -194,6 +210,96 @@ func cloneHeader(header http.Header) http.Header { return header.Clone() } +func (d *OnedriveSharelink) resolveDirectDownloadURL(ctx context.Context, file model.Obj, rawURL string, header http.Header) (string, error) { + var errs []error + if obj, ok := unwrapObject(file); ok { + if obj.SPItemURL != "" { + directURL, err := d.resolveSPItemDownloadURL(ctx, obj.SPItemURL, header) + if err == nil { + return directURL, nil + } + errs = append(errs, err) + } + if obj.ContentDownloadURL != "" { + return obj.ContentDownloadURL, nil + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", err + } + req.Header = cloneHeader(header) + if req.Header == nil { + req.Header = http.Header{} + } + + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + location := resp.Header.Get("Location") + if location == "" { + errs = append(errs, fmt.Errorf("download.aspx returned no redirect location, status code: %d", resp.StatusCode)) + return "", fmt.Errorf("onedrive_sharelink: direct download URL unavailable: %v", errs) + } + u, err := req.URL.Parse(location) + if err != nil { + return "", err + } + return u.String(), nil +} + +type spItemDownloadResp struct { + ContentDownloadURL string `json:"@content.downloadUrl"` +} + +func unwrapObject(obj model.Obj) (*Object, bool) { + for { + switch o := obj.(type) { + case *Object: + return o, true + case model.ObjUnwrap: + obj = o.Unwrap() + default: + return nil, false + } + } +} + +func (d *OnedriveSharelink) resolveSPItemDownloadURL(ctx context.Context, spItemURL string, header http.Header) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, spItemURL, nil) + if err != nil { + return "", err + } + req.Header = cloneHeader(header) + if req.Header == nil { + req.Header = http.Header{} + } + req.Header.Set("Accept", "application/json;odata.metadata=minimal") + + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("sp item metadata request failed, status code: %d", resp.StatusCode) + } + + var data spItemDownloadResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return "", err + } + if data.ContentDownloadURL == "" { + return "", fmt.Errorf("sp item metadata response missing @content.downloadUrl") + } + return data.ContentDownloadURL, nil +} + func (d *OnedriveSharelink) headerSnapshot() http.Header { d.headerMu.RLock() defer d.headerMu.RUnlock() diff --git a/drivers/onedrive_sharelink/meta.go b/drivers/onedrive_sharelink/meta.go index 10bea87af2..453f51e2a0 100644 --- a/drivers/onedrive_sharelink/meta.go +++ b/drivers/onedrive_sharelink/meta.go @@ -19,7 +19,6 @@ type Addition struct { var config = driver.Config{ Name: "Onedrive Sharelink", - OnlyProxy: true, NoUpload: true, DefaultRoot: "/", } diff --git a/drivers/onedrive_sharelink/types.go b/drivers/onedrive_sharelink/types.go index 0b7d746456..02b1f5750a 100644 --- a/drivers/onedrive_sharelink/types.go +++ b/drivers/onedrive_sharelink/types.go @@ -23,30 +23,42 @@ type FolderResp struct { // Item represents an individual item in the folder. type Item struct { - ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. - Name string `json:"FileLeafRef"` // Name is the name of the item. - ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. - Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. - Id string `json:"UniqueId"` // Id is the unique identifier of the item. + ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. + Name string `json:"FileLeafRef"` // Name is the name of the item. + ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. + Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. + Id string `json:"UniqueId"` // Id is the unique identifier of the item. + SPItemURL string `json:".spItemUrl"` // SPItemURL points to the SharePoint item metadata API. + ContentDownloadURL string `json:"@content.downloadUrl"` // ContentDownloadURL is a temporary cookie-free download URL. } -// fileToObj converts an Item to an ObjThumb. -func fileToObj(f Item) *model.ObjThumb { +type Object struct { + model.ObjThumb + SPItemURL string + ContentDownloadURL string +} + +// fileToObj converts an Item to an Object. +func fileToObj(f Item) *Object { // Convert Size from string to int64. size, _ := strconv.ParseInt(f.Size, 10, 64) // Convert ObjType from string to int. objtype, _ := strconv.Atoi(f.ObjType) // Create a new ObjThumb with the converted values. - file := &model.ObjThumb{ - Object: model.Object{ - Name: f.Name, - Modified: f.ModifiedTime, - Size: size, - IsFolder: objtype == 1, // Check if the item is a folder. - ID: f.Id, + file := &Object{ + ObjThumb: model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.ModifiedTime, + Size: size, + IsFolder: objtype == 1, // Check if the item is a folder. + ID: f.Id, + }, + Thumbnail: model.Thumbnail{}, }, - Thumbnail: model.Thumbnail{}, + SPItemURL: f.SPItemURL, + ContentDownloadURL: f.ContentDownloadURL, } return file } From 089315118de0eb4129b7ba46c21682119bd35da5 Mon Sep 17 00:00:00 2001 From: So <65939241+syscc@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:43:26 +0800 Subject: [PATCH 026/107] feat(onedrive_sharelink): support upload and folder sizes (#2595) Co-authored-by: syscc Co-authored-by: OpenAI Codex --- drivers/onedrive_sharelink/driver.go | 464 ++++++++++++++++++++++++++- drivers/onedrive_sharelink/meta.go | 8 +- drivers/onedrive_sharelink/types.go | 13 + drivers/onedrive_sharelink/util.go | 49 +-- 4 files changed, 494 insertions(+), 40 deletions(-) diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 2b205324c2..d9975e7e46 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -1,30 +1,39 @@ package onedrive_sharelink import ( + "bytes" "context" "encoding/json" "fmt" "io" "net/http" + "net/url" stdpath "path" + "regexp" "strings" "sync" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) const ( - headerTTL = 25 * time.Minute - directLinkTTL = 20 * time.Minute + headerTTL = 25 * time.Minute + driveTokenTTL = 20 * time.Minute + directLinkTTL = 20 * time.Minute + uploadSessionChunk = 10 * 1024 * 1024 ) type OnedriveSharelink struct { @@ -82,10 +91,17 @@ func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model. if err != nil { return nil, err } + folderSizes, err := d.driveChildrenFolderSizes(ctx, dir.GetPath()) + if err != nil { + log.Warnf("onedrive_sharelink: failed to get folder sizes for %s: %+v", dir.GetPath(), err) + } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { obj := fileToObj(src) + if size, ok := folderSizes[obj.GetName()]; ok { + obj.Size = size + } obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) return obj, nil }) @@ -125,8 +141,30 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model } func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - // TODO create folder, optional - return errs.NotImplement + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return err + } + apiURL := injectAccessToken(d.drivePathAPIURL(parentDir.GetPath())+"/children", token) + body := map[string]any{ + "name": dirName, + "folder": map[string]any{}, + "@microsoft.graph.conflictBehavior": "fail", + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + return nil + case http.StatusConflict: + return errs.ObjectAlreadyExists + default: + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create folder, status code: %d, body: %s", resp.StatusCode, string(data)) + } } func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -150,8 +188,422 @@ func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error { } func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO upload file, optional - return errs.NotImplement + info, err := d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), stream.GetName()), stream.GetSize()) + if err != nil { + return err + } + if info.ChunkSize == 0 { + return d.uploadContent(ctx, info.UploadURL, stream, up) + } + return d.uploadToSession(ctx, info.UploadURL, stream, up) +} + +func (d *OnedriveSharelink) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if d.DisableDiskUsage { + return nil, errs.NotImplement + } + size, err := d.driveItemSize(ctx, "/") + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: size, + UsedSpace: size, + }, + }, nil +} + +func (d *OnedriveSharelink) driveItemSize(ctx context.Context, path string) (int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return 0, err + } + apiURL := injectAccessToken(d.drivePathAPIURL(path), token) + resp, err := d.doJSON(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("failed to get folder details, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var item struct { + Size int64 `json:"size"` + } + if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { + return 0, err + } + return item.Size, nil +} + +func (d *OnedriveSharelink) driveChildrenFolderSizes(ctx context.Context, path string) (map[string]int64, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + rawURL := d.drivePathAPIURL(path) + "/children?$select=name,size,folder" + sizes := make(map[string]int64) + for rawURL != "" { + resp, err := d.doJSON(ctx, http.MethodGet, injectAccessToken(rawURL, token), nil) + if err != nil { + return nil, err + } + var data struct { + Value []struct { + Name string `json:"name"` + Size int64 `json:"size"` + Folder any `json:"folder"` + } `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + func() { + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + err = fmt.Errorf("failed to get children details, status code: %d, body: %s", resp.StatusCode, string(body)) + return + } + err = json.NewDecoder(resp.Body).Decode(&data) + }() + if err != nil { + return nil, err + } + for _, item := range data.Value { + if item.Folder != nil { + sizes[item.Name] = item.Size + } + } + rawURL = data.NextLink + } + return sizes, nil +} + +func (d *OnedriveSharelink) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveSharelink) GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + if tool != "HttpDirect" { + return nil, errs.NotImplement + } + return d.createUploadInfo(ctx, stdpath.Join(dstDir.GetPath(), fileName), fileSize) +} + +func (d *OnedriveSharelink) createUploadInfo(ctx context.Context, path string, fileSize int64) (*model.HttpDirectUploadInfo, error) { + token, err := d.getValidDriveAccessToken(ctx) + if err != nil { + return nil, err + } + if fileSize == 0 { + return &model.HttpDirectUploadInfo{ + UploadURL: injectAccessToken(d.drivePathAPIURL(path)+"/content", token), + Method: http.MethodPut, + }, nil + } + apiURL := injectAccessToken(d.drivePathAPIURL(path)+"/createUploadSession", token) + body := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + resp, err := d.doJSON(ctx, http.MethodPost, apiURL, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to create upload session, status code: %d, body: %s", resp.StatusCode, string(data)) + } + var data uploadSessionResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return nil, err + } + if data.UploadURL == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: data.UploadURL, + ChunkSize: uploadSessionChunk, + Method: http.MethodPut, + }, nil +} + +func (d *OnedriveSharelink) uploadContent(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: file, + Size: file.GetSize(), + }, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader) + if err != nil { + return err + } + req.ContentLength = file.GetSize() + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to upload content, status code: %d, body: %s", resp.StatusCode, string(data)) + } + return nil +} + +func (d *OnedriveSharelink) uploadToSession(ctx context.Context, uploadURL string, file model.FileStreamer, up driver.UpdateProgress) error { + if up == nil { + up = func(float64) {} + } + if file.GetSize() <= 0 { + return d.uploadSessionChunk(ctx, uploadURL, file, 0, 0, file.GetSize()) + } + ss, err := streamPkg.NewStreamSectionReader(file, uploadSessionChunk, &up) + if err != nil { + return err + } + var finish int64 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, int64(uploadSessionChunk)) + rd, err := ss.GetSectionReader(finish, byteSize) + if err != nil { + return err + } + err = retry.Do( + func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + return d.uploadSessionChunk(ctx, uploadURL, rd, finish, byteSize, file.GetSize()) + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + ss.FreeSectionReader(rd) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + } + return nil +} + +func (d *OnedriveSharelink) uploadSessionChunk(ctx context.Context, uploadURL string, reader io.Reader, start, size, total int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, driver.NewLimitedUploadStream(ctx, reader)) + if err != nil { + return err + } + req.ContentLength = size + if total > 0 { + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, start+size-1, total)) + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch { + case resp.StatusCode >= 500 && resp.StatusCode <= 504: + return fmt.Errorf("server error: %d", resp.StatusCode) + case resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted: + data, _ := io.ReadAll(resp.Body) + return errors.New(string(data)) + default: + return nil + } +} + +func (d *OnedriveSharelink) drivePathAPIURL(path string) string { + drivePath := stdpath.Join(d.driveRootPath, path) + drivePath = utils.FixAndCleanPath(drivePath) + if drivePath == "/" { + return d.DriveURL + "/root" + } + return fmt.Sprintf("%s/root:%s:", d.DriveURL, utils.EncodePath(drivePath, true)) +} + +func injectAccessToken(rawURL, token string) string { + if token == "" { + return rawURL + } + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + values := u.Query() + if strings.HasPrefix(token, "access_token=") { + values.Set("access_token", strings.TrimPrefix(token, "access_token=")) + } else { + values.Set("access_token", token) + } + u.RawQuery = values.Encode() + return u.String() +} + +func (d *OnedriveSharelink) doJSON(ctx context.Context, method, rawURL string, body any) (*http.Response, error) { + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, reader) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json;odata.metadata=minimal") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return base.HttpClient.Do(req) +} + +func (d *OnedriveSharelink) getValidDriveAccessToken(ctx context.Context) (string, error) { + d.headerMu.RLock() + token := d.DriveAccessToken + expired := time.Since(time.Unix(d.DriveTokenTime, 0)) > driveTokenTTL + d.headerMu.RUnlock() + if token != "" && !expired { + return token, nil + } + if err := d.refreshDriveContext(ctx); err != nil { + d.headerMu.RLock() + token = d.DriveAccessToken + d.headerMu.RUnlock() + if token != "" { + log.Warnf("onedrive_sharelink: use cached drive access token after refresh failure: %+v", err) + return token, nil + } + return "", err + } + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return d.DriveAccessToken, nil +} + +func (d *OnedriveSharelink) refreshDriveContext(ctx context.Context) error { + header, err := d.getValidHeaders(ctx) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.ShareLinkURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := NewNoRedirectCLient().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + redirectURL := resp.Header.Get("Location") + if redirectURL == "" && resp.Request != nil && resp.Request.URL != nil { + redirectURL = resp.Request.URL.String() + } + if redirectURL == "" { + return fmt.Errorf("share link did not return redirect URL") + } + return d.refreshDriveContextFromRedirect(ctx, redirectURL, header) +} + +func (d *OnedriveSharelink) refreshDriveContextFromRedirect(ctx context.Context, redirectURL string, header http.Header) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil) + if err != nil { + return err + } + req.Header = cloneHeader(header) + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("onedrive page request failed, status code: %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + ctxInfo, err := parsePageContext(body) + if err != nil { + return err + } + rootPath, err := driveRootPathFromRedirect(redirectURL, ctxInfo.ListURL) + if err != nil { + return err + } + d.headerMu.Lock() + d.DriveURL = ctxInfo.DriveInfo.DriveURL + d.DriveAccessToken = ctxInfo.DriveInfo.DriveAccessToken + d.DriveTokenTime = time.Now().Unix() + d.driveRootPath = rootPath + d.headerMu.Unlock() + return nil +} + +func driveRootPathFromRedirect(redirectURL, listURL string) (string, error) { + u, err := url.Parse(redirectURL) + if err != nil { + return "", err + } + id := u.Query().Get("id") + if id == "" { + return "/", nil + } + if listURL == "" { + return "/", nil + } + if id == listURL { + return "/", nil + } + prefix := strings.TrimRight(listURL, "/") + "/" + if strings.HasPrefix(id, prefix) { + return utils.FixAndCleanPath(strings.TrimPrefix(id, strings.TrimRight(listURL, "/"))), nil + } + return "/", nil +} + +var pageContextRE = regexp.MustCompile(`(?s)var _spPageContextInfo=(\{.*?\});_spPageContextInfo`) + +func parsePageContext(body []byte) (*pageContextInfo, error) { + match := pageContextRE.FindSubmatch(body) + if len(match) < 2 { + return nil, fmt.Errorf("failed to find _spPageContextInfo") + } + var info pageContextInfo + if err := json.Unmarshal(match[1], &info); err != nil { + return nil, err + } + if info.DriveInfo.DriveURL == "" { + return nil, fmt.Errorf("failed to get drive URL from page context") + } + if info.DriveInfo.DriveAccessToken == "" { + return nil, fmt.Errorf("failed to get drive access token from page context") + } + return &info, nil } //func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/onedrive_sharelink/meta.go b/drivers/onedrive_sharelink/meta.go index 453f51e2a0..1aed1b0bad 100644 --- a/drivers/onedrive_sharelink/meta.go +++ b/drivers/onedrive_sharelink/meta.go @@ -11,15 +11,21 @@ type Addition struct { driver.RootPath ShareLinkURL string `json:"url" required:"true"` ShareLinkPassword string `json:"password"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Allow uploading directly to OneDrive without going through OpenList"` IsSharepoint bool downloadLinkPrefix string Headers http.Header HeaderTime int64 + DriveURL string + DriveAccessToken string + DriveTokenTime int64 + driveRootPath string } var config = driver.Config{ Name: "Onedrive Sharelink", - NoUpload: true, + LocalSort: true, DefaultRoot: "/", } diff --git a/drivers/onedrive_sharelink/types.go b/drivers/onedrive_sharelink/types.go index 02b1f5750a..4405324d8a 100644 --- a/drivers/onedrive_sharelink/types.go +++ b/drivers/onedrive_sharelink/types.go @@ -38,6 +38,19 @@ type Object struct { ContentDownloadURL string } +type uploadSessionResp struct { + UploadURL string `json:"uploadUrl"` +} + +type pageContextInfo struct { + ListURL string `json:"listUrl"` + DriveInfo struct { + DriveURL string `json:".driveUrl"` + DriveAccessToken string `json:".driveAccessToken"` + DriveAccessTokenV2 string `json:".driveAccessTokenV21"` + } `json:"driveInfo"` +} + // fileToObj converts an Item to an Object. func fileToObj(f Item) *Object { // Convert Size from string to int64. diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go index d4cd4229ac..13785939fb 100644 --- a/drivers/onedrive_sharelink/util.go +++ b/drivers/onedrive_sharelink/util.go @@ -310,38 +310,25 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, json.NewDecoder(resp.Body).Decode(&graphqlReq) log.Debugln("graphqlReq:", graphqlReq) filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row - if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" { - nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) - log.Debugln("nextHref:", nextHref) - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) + nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + if nextHref != "" { + listViewXml := strings.Replace(graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml, `"`, `\"`, -1) + renderListDataAsStreamVar := strings.Replace( + `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}`, + "REPLACEME", + listViewXml, + -1, + ) - listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml - log.Debugln("listViewXml:", listViewXml) - renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}` - listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1) - renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1) - - graphqlReqNEW := GraphQLNEWRequest{} - postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref - req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) - req.Header = tempHeader - - resp, err := client.Do(req) - if err != nil { - d.Headers, err = d.getHeaders(ctx) - if err != nil { - return nil, err - } - return d.getFiles(ctx, path) - } - defer resp.Body.Close() - json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - for graphqlReqNEW.ListData.NextHref != "" { - graphqlReqNEW = GraphQLNEWRequest{} + for nextHref != "" { + nextHref = nextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" + nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) + log.Debugln("nextHref:", nextHref) + graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) req.Header = tempHeader + resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) @@ -352,13 +339,9 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, } defer resp.Body.Close() json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) - nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" - nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) filesData = append(filesData, graphqlReqNEW.ListData.Row...) + nextHref = graphqlReqNEW.ListData.NextHref } - filesData = append(filesData, graphqlReqNEW.ListData.Row...) - } else { - filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) } return filesData, nil } From bd6cf5bc1e72160c7e24465d53f2503397aff5ad Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Thu, 11 Jun 2026 03:40:26 -0700 Subject: [PATCH 027/107] fix(teldrive): avoid index out of range panic when listing an empty dir (#2585) --- drivers/teldrive/driver.go | 6 ++++++ drivers/teldrive/driver_test.go | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 drivers/teldrive/driver_test.go diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index d420eb4d07..ea0093b7c3 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -68,6 +68,12 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } + // An empty directory reports totalPages == 0, which would make + // pagesData a zero-length slice and panic on pagesData[0] below. + if firstResp.Meta.TotalPages < 1 { + return []model.Obj{}, nil + } + pagesData := make([][]Object, firstResp.Meta.TotalPages) pagesData[0] = firstResp.Items diff --git a/drivers/teldrive/driver_test.go b/drivers/teldrive/driver_test.go new file mode 100644 index 0000000000..a5e4de9dc8 --- /dev/null +++ b/drivers/teldrive/driver_test.go @@ -0,0 +1,38 @@ +package teldrive + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +// An empty directory makes the Teldrive API report totalPages == 0. List used +// to size pagesData by that value and then write pagesData[0], which panicked +// with "index out of range [0] with length 0" on every empty folder. +func TestListEmptyDir(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[],"meta":{"count":0,"totalPages":0,"currentPage":1}}`)) + })) + defer srv.Close() + + oldClient := base.RestyClient + base.RestyClient = resty.New() + defer func() { base.RestyClient = oldClient }() + + d := &Teldrive{} + d.Address = srv.URL + + objs, err := d.List(context.Background(), &model.Object{Path: "/"}, model.ListArgs{}) + if err != nil { + t.Fatalf("List returned error: %v", err) + } + if len(objs) != 0 { + t.Fatalf("expected no entries for an empty dir, got %d", len(objs)) + } +} From 726bbf4bd238339fe6205880fbbe4c2f82d6c1c5 Mon Sep 17 00:00:00 2001 From: So <65939241+syscc@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:46:44 +0800 Subject: [PATCH 028/107] fix(115-open): map SDK not found errors (#2596) --- drivers/115_open/driver.go | 7 ++++++- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 278ae0f7fa..b2559ba66d 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,6 +2,7 @@ package _115_open import ( "context" + "errors" "fmt" "net/http" stdpath "path" @@ -14,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -169,6 +171,9 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { path = stdpath.Join(d.parentPath, path) resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { + if errors.Is(err, sdk.ErrObjectNotFound) { + return nil, errs.ObjectNotFound + } return nil, err } return &Obj{ @@ -218,7 +223,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ - FileID: srcObj.GetID(), + FileID: srcObj.GetID(), FileName: newName, }) if err != nil { diff --git a/go.mod b/go.mod index 098a75591c..9a254b99cd 100644 --- a/go.mod +++ b/go.mod @@ -139,7 +139,7 @@ require ( ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.3 + github.com/OpenListTeam/115-sdk-go v0.2.4 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect diff --git a/go.sum b/go.sum index 8ff7c863db..dcdfc512a3 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/OpenListTeam/115-sdk-go v0.2.4 h1:dVoEz3Pm996n/ZCuRckQvz36w938uNHjPrS7E/SVpBM= +github.com/OpenListTeam/115-sdk-go v0.2.4/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= From 5a92dc2834f128656c6158e16380fd1a0e8a9fea Mon Sep 17 00:00:00 2001 From: Yinan Qin Date: Fri, 12 Jun 2026 01:03:44 +0800 Subject: [PATCH 029/107] chore(workflows): update workflow deps and rm 3rd-party actions (#2600) Signed-off-by: Elysia --- .github/workflows/beta_release.yml | 35 +++++++++++++++------------- .github/workflows/build.yml | 6 ++--- .github/workflows/release.yml | 14 +++++------ .github/workflows/release_docker.yml | 8 +++---- .github/workflows/test_docker.yml | 4 ++-- 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index d5817e6b4d..7827f1dab1 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -40,15 +40,18 @@ jobs: npx changelogithub --output CHANGELOG.md - name: Upload assets to beta release - uses: softprops/action-gh-release@v2 - with: - body_path: CHANGELOG.md - files: CHANGELOG.md - prerelease: true - tag_name: beta + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if ! gh release view beta; then + gh release create beta --prerelease --notes-file CHANGELOG.md + else + gh release edit beta --prerelease --notes-file CHANGELOG.md + fi + gh release upload beta "CHANGELOG.md" --clobber - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -110,7 +113,7 @@ jobs: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -121,7 +124,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + uses: OpenListTeam/cgo-actions@v1.2.5 with: targets: ${{ matrix.target }} flags: ${{ matrix.flags || '-ldflags=' }} @@ -164,13 +167,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # See above - name: Upload assets to beta release - uses: softprops/action-gh-release@v2 - with: - files: build/compress/* - prerelease: true - tag_name: beta + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in build/compress/*; do + gh release upload beta "$file" --clobber + done - name: Clean illegal characters from matrix.target id: clean_target_name @@ -182,7 +185,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3a501ffa4..58cc3ef6f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: id: short-sha - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -42,7 +42,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + uses: OpenListTeam/cgo-actions@v1.2.5 with: targets: ${{ matrix.target }} flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} @@ -69,7 +69,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fa13af18c..3eed1faf6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '1.25.0' @@ -69,9 +69,9 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload assets - uses: softprops/action-gh-release@v2 - with: - files: build/compress/* - prerelease: false - tag_name: ${{ github.event.release.tag_name }} - + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in build/compress/*; do + gh release upload "${{ github.event.release.tag_name }}" "$file" --clobber + done diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 80bdf9e3c8..1badaf45a9 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -45,7 +45,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -85,7 +85,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 16c2994018..5448adc56d 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -34,7 +34,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' @@ -58,7 +58,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true From 88d256de7393795b522415bd16518012d6dabab5 Mon Sep 17 00:00:00 2001 From: So <65939241+syscc@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:59:34 +0800 Subject: [PATCH 030/107] feat(s3): support direct transfer redirects (#2598) Redirect S3 GET object requests to direct links when the underlying storage can provide one and proxying is not required. Redirect eligible S3 PUT object requests to HttpDirect single PUT upload URLs with 307, while falling back to the existing gofakes3 stream path for unsupported uploads. Allow OneDrive Sharelink direct upload info to use simple content PUT URLs for files within Microsoft's single-upload limit. Co-authored-by: syscc Co-authored-by: Codex --- drivers/onedrive_sharelink/driver.go | 3 +- server/s3/redirect.go | 169 +++++++++++++++++++++++++++ server/s3/redirect_test.go | 74 ++++++++++++ server/s3/server.go | 5 +- 4 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 server/s3/redirect.go create mode 100644 server/s3/redirect_test.go diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index d9975e7e46..a975772ac0 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -33,6 +33,7 @@ const ( headerTTL = 25 * time.Minute driveTokenTTL = 20 * time.Minute directLinkTTL = 20 * time.Minute + simpleUploadLimit = 250 * 1024 * 1024 uploadSessionChunk = 10 * 1024 * 1024 ) @@ -302,7 +303,7 @@ func (d *OnedriveSharelink) createUploadInfo(ctx context.Context, path string, f if err != nil { return nil, err } - if fileSize == 0 { + if fileSize >= 0 && fileSize <= simpleUploadLimit { return &model.HttpDirectUploadInfo{ UploadURL: injectAccessToken(d.drivePathAPIURL(path)+"/content", token), Method: http.MethodPut, diff --git a/server/s3/redirect.go b/server/s3/redirect.go new file mode 100644 index 0000000000..4ca822333e --- /dev/null +++ b/server/s3/redirect.go @@ -0,0 +1,169 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for openlist +package s3 + +import ( + "context" + "net" + "net/http" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/itsHenry35/gofakes3/signature" +) + +func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if u, ok := directObjectURL(r, authPairs); ok { + w.Header().Set("Location", u) + w.WriteHeader(http.StatusFound) + return + } + if u, ok := directUploadURL(r, authPairs); ok { + w.Header().Set("Location", u) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusTemporaryRedirect) + return + } + next.ServeHTTP(w, r) + }) +} + +func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool) { + if r.Method != http.MethodGet { + return "", false + } + if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) { + return "", false + } + bucketName, objectName, ok := parseObjectPath(r.URL.Path) + if !ok { + return "", false + } + bucket, err := getBucketByName(bucketName) + if err != nil { + return "", false + } + reqPath := path.Join(bucket.Path, objectName) + meta, _ := op.GetNearestMeta(reqPath) + ctx := context.WithValue(r.Context(), conf.MetaKey, meta) + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil || common.ShouldProxy(storage, path.Base(reqPath)) { + return "", false + } + link, file, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: clientIP(r), + Header: r.Header, + Redirect: true, + }) + if err != nil { + return "", false + } + defer link.Close() + if file == nil || file.IsDir() || link.URL == "" || link.RangeReader != nil { + return "", false + } + return link.URL, true +} + +func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool) { + if r.Method != http.MethodPut || r.ContentLength < 0 { + return "", false + } + if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) { + return "", false + } + if r.Header.Get("X-Amz-Copy-Source") != "" || + r.Header.Get("X-Amz-Content-Sha256") == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" { + return "", false + } + bucketName, objectName, ok := parseObjectPath(r.URL.Path) + if !ok || strings.HasSuffix(objectName, "/") { + return "", false + } + bucket, err := getBucketByName(bucketName) + if err != nil { + return "", false + } + reqPath := path.Join(bucket.Path, objectName) + storage, dstDirActualPath, err := op.GetStorageAndActualPath(path.Dir(reqPath)) + if err != nil || storage.Config().NoUpload { + return "", false + } + info, err := op.GetDirectUploadInfo(r.Context(), "HttpDirect", storage, dstDirActualPath, path.Base(reqPath), r.ContentLength) + if err != nil { + return "", false + } + httpInfo, ok := asHTTPDirectUploadInfo(info) + if !ok { + return "", false + } + method := httpInfo.Method + if method == "" { + method = http.MethodPut + } + if !strings.EqualFold(method, http.MethodPut) || httpInfo.UploadURL == "" || + httpInfo.ChunkSize > 0 || len(httpInfo.Headers) > 0 { + return "", false + } + return httpInfo.UploadURL, true +} + +func asHTTPDirectUploadInfo(info any) (*model.HttpDirectUploadInfo, bool) { + switch v := info.(type) { + case *model.HttpDirectUploadInfo: + return v, v != nil + case model.HttpDirectUploadInfo: + return &v, true + default: + return nil, false + } +} + +func parseObjectPath(rawPath string) (bucket, object string, ok bool) { + parts := strings.SplitN(strings.TrimPrefix(rawPath, "/"), "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + return parts[0], parts[1], true +} + +func hasNonObjectQuery(r *http.Request) bool { + query := r.URL.Query() + for key := range query { + lower := strings.ToLower(key) + if strings.HasPrefix(lower, "x-amz-") || + lower == "awsaccesskeyid" || + lower == "signature" || + lower == "expires" || + lower == "x-id" { + continue + } + return true + } + return false +} + +func s3RequestAuthorized(r *http.Request, authPairs map[string]string) bool { + if len(authPairs) == 0 { + return true + } + result := signature.V4SignVerify(r) + if result == signature.ErrUnsupportAlgorithm { + result = signature.V2SignVerify(r) + } + return result == signature.ErrNone +} + +func clientIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} diff --git a/server/s3/redirect_test.go b/server/s3/redirect_test.go new file mode 100644 index 0000000000..3b9d27391c --- /dev/null +++ b/server/s3/redirect_test.go @@ -0,0 +1,74 @@ +package s3 + +import ( + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestParseObjectPath(t *testing.T) { + tests := []struct { + name string + path string + wantBucket string + wantObject string + wantOK bool + }{ + {name: "object", path: "/bucket/path/to/file.txt", wantBucket: "bucket", wantObject: "path/to/file.txt", wantOK: true}, + {name: "missing object", path: "/bucket", wantOK: false}, + {name: "empty bucket", path: "//file.txt", wantOK: false}, + {name: "empty object", path: "/bucket/", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bucket, object, ok := parseObjectPath(tt.path) + if ok != tt.wantOK || bucket != tt.wantBucket || object != tt.wantObject { + t.Fatalf("parseObjectPath(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.path, bucket, object, ok, tt.wantBucket, tt.wantObject, tt.wantOK) + } + }) + } +} + +func TestHasNonObjectQuery(t *testing.T) { + tests := []struct { + name string + raw string + want bool + }{ + {name: "no query", raw: "/bucket/object", want: false}, + {name: "aws auth query", raw: "/bucket/object?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc", want: false}, + {name: "legacy auth query", raw: "/bucket/object?AWSAccessKeyId=ak&Signature=sig&Expires=1", want: false}, + {name: "sdk operation marker", raw: "/bucket/object?x-id=GetObject", want: false}, + {name: "list query", raw: "/bucket/object?list-type=2", want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest("GET", tt.raw, nil) + if got := hasNonObjectQuery(req); got != tt.want { + t.Fatalf("hasNonObjectQuery(%q) = %v, want %v", tt.raw, got, tt.want) + } + }) + } +} + +func TestAsHTTPDirectUploadInfo(t *testing.T) { + info := model.HttpDirectUploadInfo{UploadURL: "https://example.com/upload"} + got, ok := asHTTPDirectUploadInfo(info) + if !ok || got == nil || got.UploadURL != info.UploadURL { + t.Fatalf("asHTTPDirectUploadInfo(value) = (%v, %v), want upload info", got, ok) + } + + got, ok = asHTTPDirectUploadInfo(&info) + if !ok || got == nil || got.UploadURL != info.UploadURL { + t.Fatalf("asHTTPDirectUploadInfo(pointer) = (%v, %v), want upload info", got, ok) + } + + got, ok = asHTTPDirectUploadInfo(struct{}{}) + if ok || got != nil { + t.Fatalf("asHTTPDirectUploadInfo(unsupported) = (%v, %v), want nil false", got, ok) + } +} diff --git a/server/s3/server.go b/server/s3/server.go index 360eb8752e..d33b63ddde 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -13,15 +13,16 @@ import ( // Make a new S3 Server to serve the remote func NewServer(ctx context.Context) (h http.Handler, err error) { var newLogger logger + authPairs := authlistResolver() faker := gofakes3.New( newBackend(), // gofakes3.WithHostBucket(!opt.pathBucketMode), gofakes3.WithLogger(newLogger), gofakes3.WithRequestID(rand.Uint64()), gofakes3.WithoutVersioning(), - gofakes3.WithV4Auth(authlistResolver()), + gofakes3.WithV4Auth(authPairs), gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) - return faker.Server(), nil + return redirectHandler(faker.Server(), authPairs), nil } From 64d39114ee14e08f854a78198922df988eef4979 Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:38:49 +0800 Subject: [PATCH 031/107] chore(doc): add sponsors (#2531) --- README.md | 9 +++++++++ README_cn.md | 8 ++++++++ README_ja.md | 8 ++++++++ README_nl.md | 8 ++++++++ 4 files changed, 33 insertions(+) diff --git a/README.md b/README.md index 4a94dbfcf6..bcecd83578 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,14 @@ Please refer to [*Discussions*](https://github.com/OpenListTeam/OpenList/discuss [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## Donors + +Thanks to the following donors for their generous support: + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## License The `OpenList` is open-source software licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) license. @@ -163,3 +171,4 @@ We sincerely thank the author [Xhofe](https://github.com/Xhofe) of the original Thanks goes to these wonderful people: [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) + diff --git a/README_cn.md b/README_cn.md index 55fe221060..47a8482bc3 100644 --- a/README_cn.md +++ b/README_cn.md @@ -134,6 +134,14 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## 捐赠者 + +感谢以下捐赠者的慷慨支持： + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## 许可证 `OpenList` 是基于 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证的开源软件。 diff --git a/README_ja.md b/README_ja.md index 261223de3a..2e4dbfb5af 100644 --- a/README_ja.md +++ b/README_ja.md @@ -134,6 +134,14 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## 寄付者 + +以下の方々の寛大なご支援に感謝いたします： + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## ライセンス 「OpenList」は [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスの下で公開されているオープンソースソフトウェアです。 diff --git a/README_nl.md b/README_nl.md index d3be2703f9..a7ae49e1c7 100644 --- a/README_nl.md +++ b/README_nl.md @@ -134,6 +134,14 @@ Stel algemene vragen in [*Discussions*](https://github.com/OpenListTeam/OpenList [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") +## Donatie Bijdragers + +Dank aan de volgende donateurs voor hun steun: + +- [HisAtri](https://github.com/HisAtri) +- 爱发电用户_7jTh +- suka + ## Licentie `OpenList` is open-source software onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. From d7c332ca133505cb60c3ce247c6d1a3b2605a270 Mon Sep 17 00:00:00 2001 From: So <65939241+syscc@users.noreply.github.com> Date: Fri, 12 Jun 2026 17:46:51 +0800 Subject: [PATCH 032/107] fix(s3): only skip direct redirect for true sub-resource queries (#2604) --- server/s3/redirect.go | 34 ++++++++++++++++------------------ server/s3/redirect_test.go | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/server/s3/redirect.go b/server/s3/redirect.go index 4ca822333e..2157efcf74 100644 --- a/server/s3/redirect.go +++ b/server/s3/redirect.go @@ -4,7 +4,6 @@ package s3 import ( "context" - "net" "net/http" "path" "strings" @@ -13,6 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/itsHenry35/gofakes3/signature" ) @@ -20,11 +20,14 @@ import ( func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if u, ok := directObjectURL(r, authPairs); ok { + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") w.Header().Set("Location", u) w.WriteHeader(http.StatusFound) return } if u, ok := directUploadURL(r, authPairs); ok { + w.Header().Set("Referrer-Policy", "no-referrer") w.Header().Set("Location", u) w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusTemporaryRedirect) @@ -57,7 +60,7 @@ func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool return "", false } link, file, err := fs.Link(ctx, reqPath, model.LinkArgs{ - IP: clientIP(r), + IP: utils.ClientIP(r), Header: r.Header, Redirect: true, }) @@ -133,17 +136,20 @@ func parseObjectPath(rawPath string) (bucket, object string, ok bool) { return parts[0], parts[1], true } +// hasNonObjectQuery reports whether the request carries a query parameter that +// makes gofakes3 route it to a sub-resource handler instead of plain object +// download (see gofakes3 routing.go). Only those keys force server-side +// handling; every other query parameter (response-content-disposition, +// response-content-type, etc.) is irrelevant to route selection and must not +// block a direct redirect. func hasNonObjectQuery(r *http.Request) bool { query := r.URL.Query() - for key := range query { - lower := strings.ToLower(key) - if strings.HasPrefix(lower, "x-amz-") || - lower == "awsaccesskeyid" || - lower == "signature" || - lower == "expires" || - lower == "x-id" { - continue + for _, key := range []string{"uploadId", "uploads", "versioning", "versions", "location"} { + if _, ok := query[key]; ok { + return true } + } + if versionID := query.Get("versionId"); versionID != "" && versionID != "null" { return true } return false @@ -159,11 +165,3 @@ func s3RequestAuthorized(r *http.Request, authPairs map[string]string) bool { } return result == signature.ErrNone } - -func clientIP(r *http.Request) string { - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return r.RemoteAddr - } - return host -} diff --git a/server/s3/redirect_test.go b/server/s3/redirect_test.go index 3b9d27391c..9584c136a0 100644 --- a/server/s3/redirect_test.go +++ b/server/s3/redirect_test.go @@ -42,7 +42,20 @@ func TestHasNonObjectQuery(t *testing.T) { {name: "aws auth query", raw: "/bucket/object?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc", want: false}, {name: "legacy auth query", raw: "/bucket/object?AWSAccessKeyId=ak&Signature=sig&Expires=1", want: false}, {name: "sdk operation marker", raw: "/bucket/object?x-id=GetObject", want: false}, - {name: "list query", raw: "/bucket/object?list-type=2", want: true}, + {name: "response content disposition", raw: "/bucket/object?response-content-disposition=attachment%3Bfilename%3Dx", want: false}, + {name: "response content type", raw: "/bucket/object?response-content-type=text%2Fplain", want: false}, + // list-type is not a routing key in gofakes3: a path with an object + // segment is dispatched to getObject regardless, so it must not block + // a direct redirect. + {name: "list query", raw: "/bucket/object?list-type=2", want: false}, + {name: "multipart uploads", raw: "/bucket/object?uploads", want: true}, + {name: "multipart upload id", raw: "/bucket/object?uploadId=123", want: true}, + {name: "versioning", raw: "/bucket/object?versioning", want: true}, + {name: "versions", raw: "/bucket/object?versions", want: true}, + {name: "bucket location", raw: "/bucket/object?location", want: true}, + {name: "version id", raw: "/bucket/object?versionId=abc", want: true}, + {name: "version id null", raw: "/bucket/object?versionId=null", want: false}, + {name: "version id empty", raw: "/bucket/object?versionId=", want: false}, } for _, tt := range tests { From b7f9e8ef0a0a9a7fcdb16d1e284d7a3914daff09 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 13:40:57 +0800 Subject: [PATCH 033/107] fix(sftp): add connection timeout (#2626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 为 SFTP SSH 连接设置启动超时 - 避免远端不可达时长时间阻塞存储初始化 Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- drivers/sftp/util.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go index 293df8fa14..4822089992 100644 --- a/drivers/sftp/util.go +++ b/drivers/sftp/util.go @@ -3,6 +3,7 @@ package sftp import ( "fmt" "path" + "time" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/pkg/sftp" @@ -39,6 +40,7 @@ func (d *SFTP) _initClient() error { User: d.Username, Auth: []ssh.AuthMethod{auth}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 10 * time.Second, } conn, err := ssh.Dial("tcp", d.Address, config) if err != nil { From e7bccc636a8ad12ff14a707747caf7e6ba34b70f Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 13:42:31 +0800 Subject: [PATCH 034/107] docs(agents): add agent workflow instructions (#2629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加 PR 创建前读取模板的 agent 规则 - 添加提交消息格式和签名相关规则 - 明确不得编造测试、验证或提交内容 Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- AGENTS.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..606cf1178d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# Agent Instructions + +## Issues + +Before creating an issue, review the available issue templates in the `.github` directory. + +When drafting the issue: + +- Use the most appropriate template. +- Follow the template structure. +- Fill in all required sections. +- Remove sections that the template explicitly marks as optional or not applicable. +- Do not invent reproduction steps, logs, screenshots, or expected behavior. + +## Pull Requests + +Before creating a pull request, read `.github/PULL_REQUEST_TEMPLATE.md`. + +When drafting the pull request: + +- Follow the template structure. +- Use the title format required by the template. +- Fill in or remove each section according to the template guidance. +- Include testing details, or explain why testing was not run. +- Do not invent testing results. +- Do not claim validation, verification, or review steps that were not actually performed. + +## Git Commits + +When creating commits, follow the repository `git-commit` skill rules: + +- Use Conventional Commits title format: `type(scope): subject`. +- Allowed types: `feat`, `fix`, `refactor`, `perf`, `docs`, `style`, `test`, `build`, `ci`, `chore`, `revert`. +- Use a meaningful scope based on the main module, package, or feature. +- Write the subject in imperative mood and describe the actual change. +- Use a concise Markdown list in the commit body, with each item describing one key change. +- Do not invent changes that are not present in the diff. +- Do not describe behavior, refactors, fixes, or tests that are not reflected in the commit. + +Include at most one `Co-authored-by` trailer that matches the AI assistant actually used to produce the change. + +Examples: + +- `Co-authored-by: Codex <267193182+codex@users.noreply.github.com>` +- `Co-authored-by: GitHub Copilot ` +- `Co-authored-by: Claude <81847+claude@users.noreply.github.com>` + +If you are not one of the listed assistants, do not add a `Co-authored-by` trailer. + +Instead, ask the human collaborator to provide the exact `Co-authored-by` trailer to use. Do not invent, infer, or generate one yourself. From 7e55f11e4a41fd67d11128307e9c8b947d209588 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 13:43:43 +0800 Subject: [PATCH 035/107] fix(index/meilisearch): refresh progress after update (#2628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(index): refresh progress after update - 手动更新索引完成后刷新索引进度状态 - 保留现有对象计数避免引入计数差值风险 - 记录更新失败时的错误信息 Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- server/handles/index.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/server/handles/index.go b/server/handles/index.go index cd270f6f9f..78096ddb7f 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -2,6 +2,7 @@ package handles import ( "context" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -55,18 +56,38 @@ func UpdateIndex(c *gin.Context) { } go func() { ctx := context.Background() + progress, err := search.Progress() + if err != nil { + log.Errorf("get index progress error: %+v", err) + progress = &model.IndexProgress{} + } for _, path := range req.Paths { - err := search.Del(ctx, path) + err = search.Del(ctx, path) if err != nil { log.Errorf("delete index on %s error: %+v", path, err) return } } - err := search.BuildIndex(context.Background(), req.Paths, + err = search.BuildIndex(context.Background(), req.Paths, conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, false) if err != nil { log.Errorf("update index error: %+v", err) + now := time.Now() + search.WriteProgress(&model.IndexProgress{ + ObjCount: progress.ObjCount, + IsDone: true, + LastDoneTime: &now, + Error: err.Error(), + }) + return } + now := time.Now() + search.WriteProgress(&model.IndexProgress{ + ObjCount: progress.ObjCount, + IsDone: true, + LastDoneTime: &now, + Error: "", + }) }() common.SuccessResp(c) } From ffe3f3cfa378cffe0396c663150a7ac91a9edf12 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 13:45:15 +0800 Subject: [PATCH 036/107] fix(local): remove thumbnail cache on delete (#2627) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 复用本地缩略图缓存路径生成逻辑 - 删除本地文件成功后清理对应缩略图缓存 - 增加本地文件删除缓存清理测试 Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- drivers/local/driver.go | 3 +++ drivers/local/remove_test.go | 44 ++++++++++++++++++++++++++++++++++++ drivers/local/util.go | 23 +++++++++++++++---- 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 drivers/local/remove_test.go diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 7189648a7a..ba7c0850b7 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -389,6 +389,9 @@ func (d *Local) Remove(ctx context.Context, obj model.Obj) error { if err != nil { return err } + if !obj.IsDir() { + d.removeThumbCache(obj.GetPath()) + } if obj.IsDir() { if d.directoryMap.Has(obj.GetPath()) { d.directoryMap.DeleteDirNode(obj.GetPath()) diff --git a/drivers/local/remove_test.go b/drivers/local/remove_test.go new file mode 100644 index 0000000000..86ec510819 --- /dev/null +++ b/drivers/local/remove_test.go @@ -0,0 +1,44 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestRemoveDeletesThumbCache(t *testing.T) { + root := t.TempDir() + cacheDir := filepath.Join(root, "thumbs") + if err := os.Mkdir(cacheDir, 0o755); err != nil { + t.Fatal(err) + } + filePath := filepath.Join(root, "photo.jpg") + if err := os.WriteFile(filePath, []byte("image"), 0o666); err != nil { + t.Fatal(err) + } + + driver := &Local{ + Addition: Addition{ + ThumbCacheFolder: cacheDir, + }, + } + thumbPath := driver.thumbCachePath(filePath) + if err := os.WriteFile(thumbPath, []byte("thumb"), 0o666); err != nil { + t.Fatal(err) + } + + err := driver.Remove(context.Background(), &model.Object{ + Path: filePath, + Name: filepath.Base(filePath), + }) + if err != nil { + t.Fatal(err) + } + + if _, err = os.Stat(thumbPath); !os.IsNotExist(err) { + t.Fatalf("expected thumb cache to be removed, got %v", err) + } +} diff --git a/drivers/local/util.go b/drivers/local/util.go index be86d9c9b0..f9f4f4c45f 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -24,6 +24,8 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) +const thumbPrefix = "openlist_thumb_" + func isSymlinkDir(f fs.FileInfo, path string) bool { if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular == os.ModeIrregular) { // os.ModeIrregular is Junction bit in Windows @@ -110,16 +112,29 @@ func readDir(dirname string) ([]fs.FileInfo, error) { return list, nil } +func (d *Local) thumbCachePath(fullPath string) string { + if d.ThumbCacheFolder == "" { + return "" + } + return filepath.Join(d.ThumbCacheFolder, thumbPrefix+utils.GetMD5EncodeStr(fullPath)+".png") +} + +func (d *Local) removeThumbCache(fullPath string) { + thumbPath := d.thumbCachePath(fullPath) + if thumbPath == "" { + return + } + _ = os.Remove(thumbPath) +} + func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() - thumbPrefix := "openlist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { return nil, &fullPath, nil } - thumbPath := filepath.Join(d.ThumbCacheFolder, thumbName) + thumbPath := d.thumbCachePath(fullPath) if utils.Exists(thumbPath) { return nil, &thumbPath, nil } @@ -151,7 +166,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { return nil, nil, err } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) + err = os.WriteFile(d.thumbCachePath(fullPath), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } From b05a51a0624cda4f2b46c787a0cd3a220575d82b Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 13:46:17 +0800 Subject: [PATCH 037/107] feat(archive): support livp as zip archive (#2624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(archive): support livp as zip archive - 将 .livp 注册为 ZIP 压缩包识别格式 - 复用现有 ZIP 解析逻辑支持 LIVP 预览和解压 - 补充 .livp 扩展名注册测试 Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- internal/archive/zip/utils_test.go | 6 ++++++ internal/archive/zip/zip.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/archive/zip/utils_test.go b/internal/archive/zip/utils_test.go index 40611a9fdc..efd2480302 100644 --- a/internal/archive/zip/utils_test.go +++ b/internal/archive/zip/utils_test.go @@ -3,6 +3,7 @@ package zip import ( "testing" + "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -10,6 +11,11 @@ import ( "github.com/stretchr/testify/require" ) +func TestZipAcceptsLivpExtension(t *testing.T) { + _, ok := tool.Tools[".livp"] + require.True(t, ok) +} + func setNonEFSZipEncoding(value string) { op.Cache.SetSetting(conf.NonEFSZipEncoding, &model.SettingItem{ Key: conf.NonEFSZipEncoding, diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index 1dcb904d74..7adb5a988a 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -17,7 +17,7 @@ type Zip struct { } func (z *Zip) AcceptedExtensions() []string { - return []string{} + return []string{".livp"} } func (z *Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { From da84a800ab935a4d299a0450aba01600cf9aa331 Mon Sep 17 00:00:00 2001 From: hcrgm Date: Tue, 16 Jun 2026 13:49:24 +0800 Subject: [PATCH 038/107] feat(baidu): support link cache lasting one hour (#2033) --- drivers/baidu_netdisk/util.go | 2 ++ drivers/baidu_photo/types.go | 2 +- drivers/baidu_photo/utils.go | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 0e27fb3053..7916f9b508 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -215,11 +215,13 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li u = res.Header().Get("location") //} + exp := time.Hour return &model.Link{ URL: u, Header: http.Header{ "User-Agent": []string{"pan.baidu.com"}, }, + Expiration: &exp, }, nil } diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index 5d218ace9b..bee9ebb266 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -72,7 +72,7 @@ func (c *File) Thumb() string { } func (c *File) GetHash() utils.HashInfo { - return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) + return utils.HashInfo{} } /*相册部分*/ diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index f47eb1748b..a7cee84b84 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "time" "unicode" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -411,12 +412,14 @@ func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkAr // } // location := resp.Header().Get("Location") + exp := time.Hour link := &model.Link{ URL: downloadUrl.Dlink, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, }, + Expiration: &exp, } return link, nil } From eadf03a4f88c5fa6f70602eea17fccb3f3a43f8c Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Tue, 16 Jun 2026 14:45:43 +0800 Subject: [PATCH 039/107] fix(upload): respect overwrite for direct uploads (#2625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(upload): respect overwrite for direct uploads - 将 Direct Upload 的 overwrite 参数传递到后端检查逻辑 - 覆盖上传时允许已存在目标继续生成直传信息 - 非覆盖上传时按完整目标路径返回 file exists Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(upload): validate direct upload target checks - 校验直传文件名只能是单个路径段 - 在非覆盖直传检查中返回非 object not found 错误 - 在底层直传信息生成前保留真实探测错误 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(upload): preserve direct upload conflict status - 将非覆盖直传的并发已存在错误返回为 403 - 保持直传存在性检查的前端错误文案不暴露路径 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> --------- Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- internal/fs/fs.go | 4 ++-- internal/fs/put.go | 4 ++-- internal/op/fs.go | 14 ++++++++++---- server/handles/direct_upload.go | 20 ++++++++++++++++++-- server/s3/redirect.go | 2 +- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 3d459c6bed..67a1ac065e 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -199,8 +199,8 @@ func PutURL(ctx context.Context, path, dstName, urlStr string) error { return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } -func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { - info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) +func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64, overwrite bool) (any, error) { + info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize, overwrite) if err != nil { log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) } diff --git a/internal/fs/put.go b/internal/fs/put.go index be829ae472..0b905be08c 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -110,10 +110,10 @@ func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return op.Put(ctx, storage, dstDirActualPath, file, nil) } -func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { +func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64, overwrite bool) (any, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } - return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) + return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize, overwrite) } diff --git a/internal/op/fs.go b/internal/op/fs.go index 643e699709..f82a3ca8f8 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -779,7 +779,7 @@ func GetDirectUploadTools(storage driver.Driver) []string { return du.GetDirectUploadTools() } -func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { +func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64, overwrite bool) (any, error) { du, ok := storage.(driver.DirectUploader) if !ok { return nil, errors.WithStack(errs.NotImplement) @@ -789,9 +789,15 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver } dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, dstName) - _, err := Get(ctx, storage, dstPath) - if err == nil { - return nil, errors.WithStack(errs.ObjectAlreadyExists) + var err error + if !overwrite { + _, err = Get(ctx, storage, dstPath) + if err == nil { + return nil, errors.WithStack(errs.ObjectAlreadyExists) + } + if !errs.IsObjectNotFound(err) { + return nil, errors.WithMessage(err, "failed to check if object exists") + } } err = MakeDir(ctx, storage, dstDirPath) if err != nil { diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go index 69cfd2fa81..d77c3044cb 100644 --- a/server/handles/direct_upload.go +++ b/server/handles/direct_upload.go @@ -2,8 +2,10 @@ package handles import ( "net/url" + stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/server/common" @@ -38,15 +40,29 @@ func FsGetDirectUploadInfo(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if err := checkRelativePath(req.FileName); err != nil { + common.ErrorResp(c, err, 403) + return + } overwrite := c.GetHeader("Overwrite") != "false" + dstPath := stdpath.Join(path, req.FileName) if !overwrite { - if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + res, err := fs.Get(c.Request.Context(), dstPath, &fs.GetArgs{NoLog: true}) + if err != nil && !errs.IsObjectNotFound(err) { + common.ErrorResp(c, err, 500) + return + } + if res != nil { common.ErrorStrResp(c, "file exists", 403) return } } - directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) + directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite) if err != nil { + if !overwrite && errs.IsObjectAlreadyExists(err) { + common.ErrorStrResp(c, "file exists", 403) + return + } common.ErrorResp(c, err, 500) return } diff --git a/server/s3/redirect.go b/server/s3/redirect.go index 2157efcf74..4031e171d9 100644 --- a/server/s3/redirect.go +++ b/server/s3/redirect.go @@ -98,7 +98,7 @@ func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool if err != nil || storage.Config().NoUpload { return "", false } - info, err := op.GetDirectUploadInfo(r.Context(), "HttpDirect", storage, dstDirActualPath, path.Base(reqPath), r.ContentLength) + info, err := op.GetDirectUploadInfo(r.Context(), "HttpDirect", storage, dstDirActualPath, path.Base(reqPath), r.ContentLength, true) if err != nil { return "", false } From ace7482bc2c59e91144d0122c09e4087913ea46b Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Thu, 18 Jun 2026 23:16:16 +0800 Subject: [PATCH 040/107] fix(offline-download): block cloud metadata endpoints (#2487) --- internal/net/serve.go | 28 ++++++++++++++++++- internal/offline_download/115/client.go | 8 ++---- internal/offline_download/115_open/client.go | 8 ++---- internal/offline_download/123/client.go | 7 ++--- internal/offline_download/123_open/client.go | 7 ++--- internal/offline_download/http/client.go | 4 +-- internal/offline_download/pikpak/pikpak.go | 8 ++---- internal/offline_download/thunder/thunder.go | 8 ++---- .../thunder_browser/thunder_browser.go | 10 +++---- .../offline_download/thunderx/thunderx.go | 8 ++---- internal/offline_download/tool/base.go | 3 ++ internal/offline_download/tool/download.go | 1 + .../offline_download/transmission/client.go | 15 +++++++--- 13 files changed, 69 insertions(+), 46 deletions(-) diff --git a/internal/net/serve.go b/internal/net/serve.go index 6a20460b1b..89a209d881 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "mime/multipart" + gonet "net" "net/http" "strconv" "strings" @@ -292,6 +293,31 @@ func NewHttpClient() *http.Client { return &http.Client{ Timeout: time.Hour * 48, - Transport: transport, + Transport: &safeTransport{base: transport}, } } + +type safeTransport struct { + base http.RoundTripper +} + +func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + host := req.URL.Hostname() + addrs, err := gonet.DefaultResolver.LookupIPAddr(req.Context(), host) + if err != nil || len(addrs) == 0 { + return nil, errors.Wrapf(err, "failed to resolve host: %s", host) + } + for _, addr := range addrs { + if isCloudMetadataIP(addr.IP) { + return nil, ErrCloudMetadataEndpoint + } + } + return t.base.RoundTrip(req) +} + +var ErrCloudMetadataEndpoint = errors.New("access to cloud metadata endpoint is not allowed") + +func isCloudMetadataIP(ip gonet.IP) bool { + ip = ip.To4() + return ip != nil && ip[0] == 169 && ip[1] == 254 && ip[2] == 169 && ip[3] == 254 +} diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index 9e9f702d55..d31971d7cb 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -62,18 +62,16 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - hashs, err := driver115.OfflineDownload(ctx, []string{args.Url}, parentDir) + hashs, err := driver115.OfflineDownload(args.Ctx, []string{args.Url}, parentDir) if err != nil || len(hashs) < 1 { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index d12e02ec59..6f4c7fd046 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -58,18 +58,16 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) + hashs, err := driver115Open.OfflineDownload(args.Ctx, []string{args.Url}, parentDir) if err != nil || len(hashs) < 1 { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/123/client.go b/internal/offline_download/123/client.go index 2c4f470489..6138608909 100644 --- a/internal/offline_download/123/client.go +++ b/internal/offline_download/123/client.go @@ -58,15 +58,14 @@ func (*Pan123) AddURL(args *tool.AddUrlArgs) (string, error) { if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") } - ctx := context.Background() - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - taskID, err := driver123.OfflineDownload(ctx, args.Url, parentDir) + taskID, err := driver123.OfflineDownload(args.Ctx, args.Url, parentDir) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/123_open/client.go b/internal/offline_download/123_open/client.go index ce1453c321..997eeabefa 100644 --- a/internal/offline_download/123_open/client.go +++ b/internal/offline_download/123_open/client.go @@ -56,16 +56,15 @@ func (*Open123) AddURL(args *tool.AddUrlArgs) (string, error) { if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") } - ctx := context.Background() - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } cb := setting.GetStr(conf.Pan123OpenOfflineDownloadCallbackUrl) - taskID, err := driver123Open.OfflineDownload(ctx, args.Url, parentDir, cb) + taskID, err := driver123Open.OfflineDownload(args.Ctx, args.Url, parentDir, cb) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go index 86314031d1..6b3c45a5fd 100644 --- a/internal/offline_download/http/client.go +++ b/internal/offline_download/http/client.go @@ -11,13 +11,13 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type SimpleHttp struct { - client http.Client } func (s SimpleHttp) Name() string { @@ -62,7 +62,7 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { if streamPut { req.Header.Set("Range", "bytes=0-") } - resp, err := s.client.Do(req) + resp, err := net.HttpClient().Do(req) if err != nil { return err } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index f48ed99519..8a6693091f 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -63,18 +63,16 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "") + t, err := pikpakDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go index 7cbff89df0..1a134f34a7 100644 --- a/internal/offline_download/thunder/thunder.go +++ b/internal/offline_download/thunder/thunder.go @@ -64,18 +64,16 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") + task, err := thunderDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/thunder_browser/thunder_browser.go b/internal/offline_download/thunder_browser/thunder_browser.go index 9324d7a760..d1c7671221 100644 --- a/internal/offline_download/thunder_browser/thunder_browser.go +++ b/internal/offline_download/thunder_browser/thunder_browser.go @@ -63,13 +63,11 @@ func (t *ThunderBrowser) AddURL(args *tool.AddUrlArgs) (string, error) { return "", err } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } @@ -77,9 +75,9 @@ func (t *ThunderBrowser) AddURL(args *tool.AddUrlArgs) (string, error) { var task *thunder_browser.OfflineTask switch v := storage.(type) { case *thunder_browser.ThunderBrowser: - task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") + task, err = v.OfflineDownload(args.Ctx, args.Url, parentDir, "") case *thunder_browser.ThunderBrowserExpert: - task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") + task, err = v.OfflineDownload(args.Ctx, args.Url, parentDir, "") default: return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderBrowser is supported") } diff --git a/internal/offline_download/thunderx/thunderx.go b/internal/offline_download/thunderx/thunderx.go index 3ba4a36800..9ba4c7eb87 100644 --- a/internal/offline_download/thunderx/thunderx.go +++ b/internal/offline_download/thunderx/thunderx.go @@ -58,18 +58,16 @@ func (t *ThunderX) AddURL(args *tool.AddUrlArgs) (string, error) { return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderX is supported") } - ctx := context.Background() - - if err := op.MakeDir(ctx, storage, actualPath); err != nil { + if err := op.MakeDir(args.Ctx, storage, actualPath); err != nil { return "", err } - parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + parentDir, err := op.GetUnwrap(args.Ctx, storage, actualPath) if err != nil { return "", err } - task, err := thunderXDriver.OfflineDownload(ctx, args.Url, parentDir, "") + task, err := thunderXDriver.OfflineDownload(args.Ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 6de05e5b2f..823bac5266 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -1,6 +1,8 @@ package tool import ( + "context" + "github.com/OpenListTeam/OpenList/v4/internal/model" ) @@ -9,6 +11,7 @@ type AddUrlArgs struct { UID string TempDir string Signal chan int + Ctx context.Context } type Status struct { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 5ee6ef4ffd..b7f7a1a9df 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -54,6 +54,7 @@ func (t *DownloadTask) Run() error { t.Signal = nil }() gid, err := t.tool.AddURL(&AddUrlArgs{ + Ctx: t.Ctx(), Url: t.Url, UID: t.ID, TempDir: t.TempDir, diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index f86390fb8c..49e48aa68f 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -9,9 +9,11 @@ import ( "net/url" "strconv" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -84,7 +86,12 @@ func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { } // http url for .torrent file if endpoint.Scheme == "http" || endpoint.Scheme == "https" { - resp, err := http.Get(args.Url) + resp, err := net.RequestHttp( + args.Ctx, + http.MethodGet, + http.Header{"User-Agent": []string{base.UserAgent}}, + args.Url, + ) if err != nil { return "", errors.Wrap(err, "failed to get .torrent file") } @@ -106,7 +113,7 @@ func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { rpcPayload.Filename = &args.Url } - torrent, err := t.client.TorrentAdd(context.TODO(), rpcPayload) + torrent, err := t.client.TorrentAdd(args.Ctx, rpcPayload) if err != nil { return "", err } @@ -123,7 +130,7 @@ func (t *Transmission) Remove(task *tool.DownloadTask) error { if err != nil { return err } - err = t.client.TorrentRemove(context.TODO(), transmissionrpc.TorrentRemovePayload{ + err = t.client.TorrentRemove(task.Ctx(), transmissionrpc.TorrentRemovePayload{ IDs: []int64{gid}, DeleteLocalData: false, }) @@ -135,7 +142,7 @@ func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { if err != nil { return nil, err } - infos, err := t.client.TorrentGetAllFor(context.TODO(), []int64{gid}) + infos, err := t.client.TorrentGetAllFor(task.Ctx(), []int64{gid}) if err != nil { return nil, err } From d34976561745e90778042735a4f709f4f331306e Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Thu, 18 Jun 2026 23:17:24 +0800 Subject: [PATCH 041/107] fix(drivers/thunder): handle missing token errors (#2623) --- drivers/thunder/driver.go | 6 ++++++ drivers/thunder/driver_test.go | 18 ++++++++++++++++++ drivers/thunder_browser/driver.go | 7 +++++++ drivers/thunder_browser/driver_test.go | 18 ++++++++++++++++++ drivers/thunderx/driver.go | 8 +++++++- drivers/thunderx/driver_test.go | 18 ++++++++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 drivers/thunder/driver_test.go create mode 100644 drivers/thunder_browser/driver_test.go create mode 100644 drivers/thunderx/driver_test.go diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 492b9814f5..df4113c71d 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -88,6 +88,9 @@ func (x *Thunder) Init(ctx context.Context) (err error) { // 清空 信任密钥 x.Addition.CreditKey = "" } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -521,6 +524,9 @@ func (xc *XunLeiCommon) SetCoreTokenResp(tr *CoreLoginResp) { // 携带Authorization和CaptchaToken的请求 func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), diff --git a/drivers/thunder/driver_test.go b/drivers/thunder/driver_test.go new file mode 100644 index 0000000000..5cea7f4719 --- /dev/null +++ b/drivers/thunder/driver_test.go @@ -0,0 +1,18 @@ +package thunder + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index e60522e529..da1844afa7 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -12,6 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" @@ -87,6 +88,9 @@ func (x *ThunderBrowser) Init(ctx context.Context) (err error) { // 清空 信任密钥 x.Addition.CreditKey = "" } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -639,6 +643,9 @@ func (xc *XunLeiBrowserCommon) SetSpaceTokenResp(spaceToken string) { // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiBrowserCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.GetToken(), diff --git a/drivers/thunder_browser/driver_test.go b/drivers/thunder_browser/driver_test.go new file mode 100644 index 0000000000..edf004927e --- /dev/null +++ b/drivers/thunder_browser/driver_test.go @@ -0,0 +1,18 @@ +package thunder_browser + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiBrowserCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index acbb825133..457a9c5367 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -69,13 +69,16 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) - if token.UserID != "" { + if token != nil && token.UserID != "" { x.SetUserID(token.UserID) x.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) } op.MustSaveDriverStorage(x) } } + if token == nil { + return err + } x.SetTokenResp(token) return err }, @@ -496,6 +499,9 @@ func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if xc.TokenResp == nil { + return nil, errs.EmptyToken + } data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), diff --git a/drivers/thunderx/driver_test.go b/drivers/thunderx/driver_test.go new file mode 100644 index 0000000000..d3f9ef9c2c --- /dev/null +++ b/drivers/thunderx/driver_test.go @@ -0,0 +1,18 @@ +package thunderx + +import ( + "errors" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +func TestRequestReturnsErrorWhenTokenIsMissing(t *testing.T) { + xc := &XunLeiXCommon{} + + _, err := xc.Request(API_URL+"/about", http.MethodGet, nil, nil) + if !errors.Is(err, errs.EmptyToken) { + t.Fatalf("expected EmptyToken, got %v", err) + } +} From dcfefce85d858eb58f3fffcb44059647be10d6dc Mon Sep 17 00:00:00 2001 From: EzraRT Date: Fri, 19 Jun 2026 00:04:25 +0800 Subject: [PATCH 042/107] fix(webauthn): enable discoverable keys (#2451) Set `WithResidentKeyRequirement` option during registration to enable discoverable keys. Existing keys do not require migration. They won't appear without entering username, but work normally after username input, legacy flows remain unaffected. --- server/handles/webauthn.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index b2a0fbfb47..6848020fd0 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -133,7 +133,10 @@ func BeginAuthnRegistration(c *gin.Context) { return } - options, sessionData, err := authnInstance.BeginRegistration(user) + options, sessionData, err := authnInstance.BeginRegistration( + user, + webauthn.WithResidentKeyRequirement(protocol.ResidentKeyRequirementRequired), + ) if err != nil { common.ErrorResp(c, err, 400) From f45473357ea404b4415da52e5402a162c98f9311 Mon Sep 17 00:00:00 2001 From: Castronaut <3116935098@qq.com> Date: Fri, 19 Jun 2026 00:11:38 +0800 Subject: [PATCH 043/107] fix(drivers/webdav): normalize missing path error (#2611) Normalize WebDAV 404 responses from `Stat` to `errs.ObjectNotFound`. This fixes the mkdir pre-check path: when the target folder does not exist, `op.MakeDir` can now treat the lookup as a normal missing object and continue to issue `MKCOL` through the WebDAV driver. Co-authored-by: ChatGPT --- drivers/webdav/driver.go | 4 ++ drivers/webdav/driver_test.go | 89 +++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 drivers/webdav/driver_test.go diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 61e4a6160f..778d810b43 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -10,6 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/gowebdav" @@ -130,6 +131,9 @@ func (d *WebDav) Get(ctx context.Context, _path string) (model.Obj, error) { _path = path.Join(d.GetRootPath(), _path) info, err := d.client.Stat(_path) if err != nil { + if gowebdav.IsErrNotFound(err) { + return nil, errs.ObjectNotFound + } return nil, err } diff --git a/drivers/webdav/driver_test.go b/drivers/webdav/driver_test.go new file mode 100644 index 0000000000..6251f7ff28 --- /dev/null +++ b/drivers/webdav/driver_test.go @@ -0,0 +1,89 @@ +package webdav + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +func TestGetMapsMissingPathToObjectNotFound(t *testing.T) { + d, cleanup := newTestDriver(t, nil) + defer cleanup() + + _, err := d.Get(context.Background(), "/missing") + if !errs.IsObjectNotFound(err) { + t.Fatalf("expected object not found, got %v", err) + } +} + +func TestMakeDirAfterMissingWebDAVStat(t *testing.T) { + var mkcolCount int32 + d, cleanup := newTestDriver(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.Method == "MKCOL" && (r.URL.Path == "/new" || r.URL.Path == "/new/") { + atomic.AddInt32(&mkcolCount, 1) + w.WriteHeader(http.StatusCreated) + return true + } + return false + }) + defer cleanup() + + if err := op.MakeDir(context.Background(), d, "/new"); err != nil { + t.Fatalf("MakeDir failed: %v", err) + } + if got := atomic.LoadInt32(&mkcolCount); got != 1 { + t.Fatalf("expected one MKCOL request, got %d", got) + } +} + +func newTestDriver(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) (*WebDav, func()) { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if extra != nil && extra(w, r) { + return + } + switch r.Method { + case "PROPFIND": + if r.URL.Path == "/" { + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + _, _ = w.Write([]byte(` + + + / + + + / + + + HTTP/1.1 200 OK + + +`)) + return + } + http.NotFound(w, r) + case "MKCOL": + w.WriteHeader(http.StatusConflict) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + + d := &WebDav{Addition: Addition{Address: srv.URL, RootPath: driver.RootPath{RootFolderPath: "/"}}} + if err := d.Init(context.Background()); err != nil { + srv.Close() + t.Fatalf("init driver: %v", err) + } + return d, func() { + _ = d.Drop(context.Background()) + srv.Close() + } +} From a42e367660a19ce19fbc21eda4adbcac5c765943 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Fri, 19 Jun 2026 00:12:18 +0800 Subject: [PATCH 044/107] fix(drivers/github_releases): skip broken repos instead of returning 500 (#2494) * fix(drivers/github_releases): skip broken repos instead of returning 500 When multiple repos are configured and one fails (404, rate limit, etc.), the driver now logs a warning and skips that repo instead of panicking with a nil pointer dereference that caused a 500 error for the entire storage. Added proper error propagation from GetRequest through RequestRelease/RequestReleases/GetOtherFile, and nil checks on all Release/Releases dereferences. Instead of hiding broken repos entirely, still display them as directory entries in parent listings. Return an error only when the user navigates into a broken repo. Co-authored-by: Claude Code Co-authored-by: Mimo <208276378+mimo@users.noreply.github.com> * fix(drivers/github_releases): apply patches Co-authored-by: GitHub Copilot Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner Co-authored-by: Claude Code Co-authored-by: Mimo <208276378+mimo@users.noreply.github.com> Co-authored-by: GitHub Copilot --- drivers/github_releases/driver.go | 65 +++++++++++++++++++++++---- drivers/github_releases/types.go | 74 ++++++++++++++++++++++++------- drivers/github_releases/util.go | 3 +- 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index 8a8025c5aa..92d94b4dcf 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" ) type GithubReleases struct { @@ -45,12 +46,25 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis point := &d.points[i] if !d.Addition.ShowAllVersion { // latest - point.RequestRelease(d.GetRequest, args.Refresh) + err := point.RequestRelease(d.GetRequest, args.Refresh) + if err != nil { + log.Warnf("failed to request release for %s: %v", point.Repo, err) + } if point.Point == path { // 与仓库路径相同 + if point.Release == nil { + if err != nil { + return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get release for %s: unknown error", point.Repo) + } files = append(files, point.GetLatestRelease()...) if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) + if err != nil { + return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + } + files = append(files, otherFiles...) } if d.Addition.ShowSourceCode { files = append(files, point.GetSourceCode()...) @@ -60,6 +74,9 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if nextDir == "" { continue } + if err != nil { + return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) + } hasSameDir := false for index := range files { @@ -70,30 +87,51 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } } if !hasSameDir { + var updateAt, createAt string + if point.Release != nil { + updateAt = point.Release.PublishedAt + createAt = point.Release.CreatedAt + } files = append(files, File{ Path: stdpath.Join(path, nextDir), FileName: nextDir, Size: point.GetLatestSize(), - UpdateAt: point.Release.PublishedAt, - CreateAt: point.Release.CreatedAt, + UpdateAt: updateAt, + CreateAt: createAt, Type: "dir", Url: "", }) } } } else { // all version - point.RequestReleases(d.GetRequest, args.Refresh) + err := point.RequestReleases(d.GetRequest, args.Refresh) + if err != nil { + log.Warnf("failed to request releases for %s: %v", point.Repo, err) + } if point.Point == path { // 与仓库路径相同 + if point.Releases == nil { + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) + } files = append(files, point.GetAllVersion()...) if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) + if err != nil { + return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + } + files = append(files, otherFiles...) } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } hasSameDir := false for index := range files { @@ -104,12 +142,17 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } } if !hasSameDir { + var updateAt, createAt string + if point.Releases != nil && len(*point.Releases) > 0 { + updateAt = (*point.Releases)[0].PublishedAt + createAt = (*point.Releases)[0].CreatedAt + } files = append(files, File{ FileName: nextDir, Path: stdpath.Join(path, nextDir), Size: point.GetAllVersionSize(), - UpdateAt: (*point.Releases)[0].PublishedAt, - CreateAt: (*point.Releases)[0].CreatedAt, + UpdateAt: updateAt, + CreateAt: createAt, Type: "dir", Url: "", }) @@ -119,6 +162,12 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if tagName == "" { continue } + if point.Releases == nil { + if err != nil { + return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + } + return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) + } files = append(files, point.GetReleaseByTagName(tagName)...) diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 663aec77e2..abc024cc8d 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -19,33 +19,54 @@ type MountPoint struct { } // 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) error { if m.Repo == "" { - return + return nil } if m.Release == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") - m.Release = new(Release) - json.Unmarshal(resp.Body(), m.Release) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") + if err != nil { + m.Release = nil + return err + } + release := new(Release) + if err := json.Unmarshal(resp.Body(), release); err != nil { + m.Release = nil + return err + } + m.Release = release } + return nil } // 请求所有版本 -func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) error { if m.Repo == "" { - return + return nil } if m.Releases == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases") - m.Releases = new([]Release) - json.Unmarshal(resp.Body(), m.Releases) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases") + if err != nil { + m.Releases = nil + return err + } + releases := new([]Release) + if err := json.Unmarshal(resp.Body(), releases); err != nil { + m.Releases = nil + return err + } + m.Releases = releases } + return nil } // 获取最新版本 func (m *MountPoint) GetLatestRelease() []File { + if m.Release == nil { + return nil + } files := make([]File, 0, len(m.Release.Assets)) for _, asset := range m.Release.Assets { files = append(files, File{ @@ -63,6 +84,9 @@ func (m *MountPoint) GetLatestRelease() []File { // 获取最新版本大小 func (m *MountPoint) GetLatestSize() int64 { + if m.Release == nil { + return 0 + } size := int64(0) for _, asset := range m.Release.Assets { size += asset.Size @@ -72,6 +96,9 @@ func (m *MountPoint) GetLatestSize() int64 { // 获取所有版本 func (m *MountPoint) GetAllVersion() []File { + if m.Releases == nil { + return nil + } files := make([]File, 0) for _, release := range *m.Releases { file := File{ @@ -93,6 +120,9 @@ func (m *MountPoint) GetAllVersion() []File { // 根据版本号获取版本 func (m *MountPoint) GetReleaseByTagName(tagName string) []File { + if m.Releases == nil { + return nil + } for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) @@ -145,6 +175,9 @@ func (m *MountPoint) GetAllVersionSize() int64 { } func (m *MountPoint) GetSourceCode() []File { + if m.Release == nil { + return nil + } files := make([]File, 0) // 无法获取文件大小，此处设为 1 @@ -171,6 +204,9 @@ func (m *MountPoint) GetSourceCode() []File { } func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { + if m.Releases == nil { + return nil + } for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) @@ -198,11 +234,19 @@ func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { return nil } -func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { +func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) ([]File, error) { if m.OtherFile == nil || refresh { - resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") - m.OtherFile = new([]FileInfo) - json.Unmarshal(resp.Body(), m.OtherFile) + resp, err := get("https://api.github.com/repos/" + m.Repo + "/contents") + if err != nil { + m.OtherFile = nil + return nil, err + } + otherFile := new([]FileInfo) + if err := json.Unmarshal(resp.Body(), otherFile); err != nil { + m.OtherFile = nil + return nil, err + } + m.OtherFile = otherFile } files := make([]File, 0) @@ -220,7 +264,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), }) } } - return files + return files, nil } type File struct { diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index 91a4b3a231..a6401e6cd7 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -7,7 +7,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) // 发送 GET 请求 @@ -23,7 +22,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { return nil, err } if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) + return nil, fmt.Errorf("github api error: status %d", res.StatusCode()) } return res, nil } From 3b017c2b5cbc94743f3852d9df6d0c15d94aa57b Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Fri, 19 Jun 2026 00:32:00 +0800 Subject: [PATCH 045/107] feat(mcp): add mcp endpoint for whole openlist (#2485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(mcp): add initial MCP tools support - 新增 MCP 路由与 session 初始化流程 - 接入 tools/list 与 tools/call，并开放 openlist.fs.list - 补充 MCP 相关协议与路由测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * refactor(mcp): move MCP implementation into server/mcp - 将 MCP 实现与测试迁移到 server/mcp 目录 - 保留 server/mcp.go 作为路由接入包装入口 - 对齐 webdav、s3、ftp、sftp 的目录组织风格 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): address endpoint review issues - 收紧 Streamable HTTP Accept 头校验 - 为 MCP session 增加过期清理和数量上限 - 简化 fs.list 参数解析并修复分页边界 - 使用独立 Server 实例隔离 MCP 测试状态 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * feat(mcp): add fs get and link tools - 新增 openlist.fs.get 和 openlist.fs.link 工具 - 复用文件详情、代理链接和权限校验逻辑 - 补充工具列表和参数解析测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): improve protocol negotiation and proxy handling - Remove WebDAV proxy URL policy from MCP proxy link detection - Return server protocol version during initialize negotiation - Return JSON-RPC error body for invalid MCP Accept header - Add tests for MCP proxy and HTTP negotiation behavior Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): set Allow header for GET requests - 为 MCP GET 405 响应添加 Allow 头 - 补充 GET 405 响应头断言 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * refactor(mcp): use mutex for session store - 将 MCP session 锁改为 Mutex - 让锁类型匹配会更新 lastUsedAt 的访问路径 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): negotiate initialize protocol version - 添加 MCP 协议版本协商函数 - 不支持客户端版本时返回服务端支持版本 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * revert(mcp): drop protocol version negotiation wrapper - 撤回 MCP 协议版本协商包装函数 - 恢复 initialize 直接返回服务端协议版本 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): limit and reuse sessions - 将 MCP 全局 session 上限调整为 128 - 添加单用户 16 个 session 上限 - initialize 复用同用户已有 session 标识 - 补充 session 复用和上限裁剪测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): require path for fs list - 拒绝 openlist.fs.list 的空参数和 null 参数 - 校验 fs.list 缺失 path 时返回 -32602 - 添加 fs.list 参数解析测试覆盖错误文案 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): enforce protocol version header - 校验非 initialize 请求的 MCP-Protocol-Version 头 - 拒绝缺失或不支持协议版本的后续 POST 请求 - 添加协议版本头缺失和不匹配测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * test(mcp): serialize setting cache mutation - 为修改全局设置缓存的测试添加包级互斥锁 - 保留 ClearAll 清理逻辑，避免并发测试互相影响 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * feat(mcp): add config switch - Add MCP config section with disabled default - Register MCP routes only when enabled Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): handle missing session explicitly - 区分缺失 MCP session 与未知 MCP session - 为未知 session 返回 not found 错误 - 添加 MCP session 错误处理测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * feat(mcp): support latest protocol negotiation - Upgrade default MCP protocol version to 2025-11-25 - Preserve 2025-06-18 compatibility through initialize negotiation - Validate subsequent request protocol version against the negotiated session version - Add tests for latest and older protocol negotiation paths Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * fix(mcp): relax streamable http compatibility - 允许缺失协议版本头时使用会话协商版本 - 放宽 Accept 头兼容 JSON、SSE 和通配类型 - 补充 MCP 初始化和协议版本兼容性测试 Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> --------- Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --- internal/conf/config.go | 8 + server/mcp.go | 18 ++ server/mcp/call.go | 88 ++++++ server/mcp/call_test.go | 183 ++++++++++++ server/mcp/fs_get.go | 168 +++++++++++ server/mcp/fs_link.go | 200 +++++++++++++ server/mcp/fs_list.go | 182 ++++++++++++ server/mcp/get_test.go | 39 +++ server/mcp/handler.go | 536 ++++++++++++++++++++++++++++++++++ server/mcp/handler_test.go | 569 +++++++++++++++++++++++++++++++++++++ server/mcp/link_test.go | 143 ++++++++++ server/mcp/list_test.go | 54 ++++ server/mcp/tools.go | 122 ++++++++ server/router.go | 1 + 14 files changed, 2311 insertions(+) create mode 100644 server/mcp.go create mode 100644 server/mcp/call.go create mode 100644 server/mcp/call_test.go create mode 100644 server/mcp/fs_get.go create mode 100644 server/mcp/fs_link.go create mode 100644 server/mcp/fs_list.go create mode 100644 server/mcp/get_test.go create mode 100644 server/mcp/handler.go create mode 100644 server/mcp/handler_test.go create mode 100644 server/mcp/link_test.go create mode 100644 server/mcp/list_test.go create mode 100644 server/mcp/tools.go diff --git a/internal/conf/config.go b/internal/conf/config.go index 4a11728158..0f66761168 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -106,6 +106,10 @@ type SFTP struct { Listen string `json:"listen" env:"LISTEN"` } +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -131,6 +135,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` LastLaunchedVersion string `json:"last_launched_version"` ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"` } @@ -244,6 +249,9 @@ func DefaultConfig(dataDir string) *Config { Enable: false, Listen: ":5222", }, + MCP: MCP{ + Enable: false, + }, LastLaunchedVersion: "", ProxyAddress: "", } diff --git a/server/mcp.go b/server/mcp.go new file mode 100644 index 0000000000..c036ce389a --- /dev/null +++ b/server/mcp.go @@ -0,0 +1,18 @@ +package server + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/mcp" + "github.com/gin-gonic/gin" +) + +func MCP(g *gin.RouterGroup) { + if !conf.Conf.MCP.Enable { + g.Any("/mcp", func(c *gin.Context) { + common.ErrorStrResp(c, "MCP server is not enabled", 403) + }) + return + } + mcp.Register(g) +} diff --git a/server/mcp/call.go b/server/mcp/call.go new file mode 100644 index 0000000000..5a19b9b4b2 --- /dev/null +++ b/server/mcp/call.go @@ -0,0 +1,88 @@ +package mcp + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" +) + +type toolCallParams struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` +} + +type toolResultContent struct { + Type string `json:"type"` + Text string `json:"text"` +} + +func (s *Server) handleToolsCall(c *gin.Context, req request) (int, response) { + var params toolCallParams + if len(req.Params) == 0 { + return http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/call params"}, + } + } + if err := json.Unmarshal(req.Params, ¶ms); err != nil || params.Name == "" { + return http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/call params"}, + } + } + + var ( + result any + err *rpcError + ) + switch params.Name { + case "openlist.fs.list": + result, err = s.callFSList(c, params.Arguments) + case "openlist.fs.get": + result, err = s.callFSGet(c, params.Arguments) + case "openlist.fs.link": + result, err = s.callFSLink(c, params.Arguments) + default: + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32601, Message: "unknown tool"}, + } + } + + if err != nil { + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "content": []toolResultContent{ + {Type: "text", Text: err.Message}, + }, + "isError": true, + }, + } + } + + resultJSON, marshalErr := json.Marshal(result) + if marshalErr != nil { + return http.StatusInternalServerError, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32603, Message: "failed to encode tool result"}, + } + } + + return http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "content": []toolResultContent{ + {Type: "text", Text: string(resultJSON)}, + }, + "structuredContent": result, + }, + } +} diff --git a/server/mcp/call_test.go b/server/mcp/call_test.go new file mode 100644 index 0000000000..b922ffd52c --- /dev/null +++ b/server/mcp/call_test.go @@ -0,0 +1,183 @@ +package mcp + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +func TestToolsListRequiresInitializedSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32002 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestToolsListSuccess(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s2": {id: "s2", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":2, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s2") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + tools, ok := result["tools"].([]any) + if !ok || len(tools) != 3 { + t.Fatalf("unexpected tools payload: %#v", result["tools"]) + } + names := map[string]bool{} + for _, rawTool := range tools { + currentTool, ok := rawTool.(map[string]any) + if !ok { + t.Fatalf("unexpected tool payload: %#v", rawTool) + } + name, _ := currentTool["name"].(string) + names[name] = true + } + if !names["openlist.fs.list"] || !names["openlist.fs.get"] || !names["openlist.fs.link"] { + t.Fatalf("unexpected tool names: %#v", names) + } +} + +func TestToolsCallUnknownTool(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s3": {id: "s3", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":3, + "method":"tools/call", + "params":{"name":"openlist.fs.unknown","arguments":{"path":"/"}} + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s3") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32601 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestToolsCallInvalidParams(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s4": {id: "s4", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":4, + "method":"tools/call", + "params":{"name":"openlist.fs.list","arguments":"bad"} + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s4") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("expected tool error result, got protocol error: %+v", resp.Error) + } +} + +func decodeResponse(t *testing.T, w *httptest.ResponseRecorder) response { + t.Helper() + + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + return resp +} diff --git a/server/mcp/fs_get.go b/server/mcp/fs_get.go new file mode 100644 index 0000000000..550cc036fc --- /dev/null +++ b/server/mcp/fs_get.go @@ -0,0 +1,168 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/handles" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsGetArgs struct { + Path string `json:"path"` + Password string `json:"password"` +} + +func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSGetArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{ + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + rawURL, provider, err := buildFSGetRawURL(ctx, c, reqPath, obj, meta) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + parentPath := stdpath.Dir(reqPath) + var related []model.Obj + sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelatedObjs(sameLevelFiles, obj) + } + + parentMeta, _ := op.GetNearestMeta(parentPath) + thumb, _ := model.GetThumb(obj) + mountDetails, _ := model.GetStorageDetails(obj) + return handles.FsGetResp{ + ObjResp: handles.ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Thumb: thumb, + Type: utils.GetFileType(obj.GetName()), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + MountDetails: mountDetails, + }, + RawURL: rawURL, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Provider: provider, + Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)), + }, nil +} + +func parseFSGetArgs(raw json.RawMessage) (*fsGetArgs, *rpcError) { + args := &fsGetArgs{} + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.get arguments"} + } + + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.get arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + return args, nil +} + +func buildFSGetRawURL(ctx context.Context, c *gin.Context, reqPath string, obj model.Obj, meta *model.Meta) (string, string, error) { + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + provider, ok := model.GetProvider(obj) + if !ok && storageErr == nil { + provider = storage.Config().Name + } + if obj.IsDir() { + return "", provider, nil + } + if storageErr != nil { + return "", provider, storageErr + } + + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + rawURL := common.GenerateDownProxyURL(storage.GetStorage(), reqPath) + if rawURL != "" { + return rawURL, provider, nil + } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + return fmt.Sprintf("%s/p%s%s", common.GetApiUrl(ctx), utils.EncodePath(reqPath, true), query), provider, nil + } + + if url, ok := model.GetUrl(obj); ok { + return url, provider, nil + } + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Redirect: true, + }) + if err != nil { + return "", provider, err + } + defer link.Close() + return link.URL, provider, nil +} + +func filterRelatedObjs(objs []model.Obj, obj model.Obj) []model.Obj { + related := make([]model.Obj, 0) + nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + for _, current := range objs { + if current.GetName() == obj.GetName() { + continue + } + if strings.HasPrefix(current.GetName(), nameWithoutExt) { + related = append(related, current) + } + } + return related +} diff --git a/server/mcp/fs_link.go b/server/mcp/fs_link.go new file mode 100644 index 0000000000..43ddad8522 --- /dev/null +++ b/server/mcp/fs_link.go @@ -0,0 +1,200 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + stdpath "path" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsLinkArgs struct { + Path string `json:"path"` + Password string `json:"password"` + Type string `json:"type"` +} + +type fsLinkResp struct { + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Provider string `json:"provider"` + URL string `json:"url"` + URLType string `json:"url_type"` + DirectURL string `json:"direct_url,omitempty"` + ProxyURL string `json:"proxy_url,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + Header http.Header `json:"header,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + PartSize int `json:"part_size,omitempty"` +} + +func (s *Server) callFSLink(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSLinkArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{ + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if obj.IsDir() { + return nil, &rpcError{Code: -32003, Message: "path is a directory"} + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + linkInfo, err := buildFSLinkInfo(ctx, c, reqPath, args, obj, meta, storage) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + return linkInfo, nil +} + +func parseFSLinkArgs(raw json.RawMessage) (*fsLinkArgs, *rpcError) { + args := &fsLinkArgs{} + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.link arguments"} + } + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.link arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + return args, nil +} + +func buildFSLinkInfo(ctx context.Context, c *gin.Context, reqPath string, args *fsLinkArgs, obj model.Obj, meta *model.Meta, storage driver.Driver) (*fsLinkResp, error) { + provider, ok := model.GetProvider(obj) + if !ok { + provider = storage.Config().Name + } + + resp := &fsLinkResp{ + Path: reqPath, + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Provider: provider, + DownloadURL: signedFileURL(ctx, "/d", reqPath, meta, args.Type), + } + + if canProxyFile(storage, stdpath.Base(reqPath)) { + proxyURL := proxyFileURL(ctx, reqPath, meta, storage.GetStorage(), args.Type) + resp.ProxyURL = proxyURL + } + + if common.ShouldProxy(storage, stdpath.Base(reqPath)) { + resp.URL = resp.ProxyURL + resp.URLType = "proxy" + return resp, nil + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Type: args.Type, + Redirect: true, + }) + if err != nil { + return nil, err + } + defer link.Close() + + resp.DirectURL = link.URL + resp.URL = link.URL + resp.URLType = "direct" + resp.Header = link.Header + resp.ContentLength = link.ContentLength + resp.Concurrency = link.Concurrency + resp.PartSize = link.PartSize + return resp, nil +} + +func canProxyFile(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) { + return true + } + return false +} + +func proxyFileURL(ctx context.Context, reqPath string, meta *model.Meta, storage *model.Storage, linkType string) string { + if url := common.GenerateDownProxyURL(storage, reqPath); url != "" { + return url + } + return signedFileURL(ctx, "/p", reqPath, meta, linkType) +} + +func signedFileURL(ctx context.Context, prefix, reqPath string, meta *model.Meta, linkType string) string { + query := url.Values{} + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query.Set("sign", sign.Sign(reqPath)) + } + if linkType != "" { + query.Set("type", linkType) + } + rawQuery := "" + if encoded := query.Encode(); encoded != "" { + rawQuery = "?" + encoded + } + return fmt.Sprintf("%s%s%s%s", + common.GetApiUrl(ctx), + prefix, + utils.EncodePath(reqPath, true), + rawQuery, + ) +} diff --git a/server/mcp/fs_list.go b/server/mcp/fs_list.go new file mode 100644 index 0000000000..ac28cf7561 --- /dev/null +++ b/server/mcp/fs_list.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "encoding/json" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/handles" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type fsListArgs struct { + Path string `json:"path"` + Password string `json:"password"` + Refresh bool `json:"refresh"` + Page int `json:"page"` + PerPage int `json:"per_page"` +} + +func (s *Server) callFSList(c *gin.Context, raw json.RawMessage) (any, *rpcError) { + args, mcpErr := parseFSListArgs(raw) + if mcpErr != nil { + return nil, mcpErr + } + + user, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || user == nil { + return nil, &rpcError{Code: -32603, Message: "missing user context"} + } + if user.IsGuest() && user.Disabled { + return nil, &rpcError{Code: -32001, Message: "guest user is disabled"} + } + + reqPath, err := user.JoinPath(args.Path) + if err != nil { + return nil, &rpcError{Code: -32003, Message: err.Error()} + } + + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + if !common.CanAccess(user, meta, reqPath, args.Password) { + return nil, &rpcError{Code: -32003, Message: "password is incorrect or you have no permission"} + } + + write := common.CanWrite(user, meta, reqPath) + writeContentBypass := common.CanWriteContentBypassUserPerms(meta, reqPath) + canWriteContentAtPath := write && (user.CanWriteContent() || writeContentBypass) + if args.Refresh && !canWriteContentAtPath { + return nil, &rpcError{Code: -32003, Message: "refresh without permission"} + } + + ctx := context.WithValue(c.Request.Context(), conf.MetaKey, meta) + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{ + Refresh: args.Refresh, + WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), + }) + if err != nil { + return nil, &rpcError{Code: -32603, Message: err.Error()} + } + + total, paged := paginateObjs(objs, args.Page, args.PerPage) + return handles.FsListResp{ + Content: toObjResp(paged, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Write: write, + WriteContentBypass: writeContentBypass, + Provider: "unknown", + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + }, nil +} + +func parseFSListArgs(raw json.RawMessage) (*fsListArgs, *rpcError) { + args := &fsListArgs{ + Page: 1, + PerPage: model.MaxInt, + } + if len(raw) == 0 || string(raw) == "null" { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.list arguments"} + } + + if err := json.Unmarshal(raw, args); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid openlist.fs.list arguments"} + } + if args.Path == "" { + return nil, &rpcError{Code: -32602, Message: "path is required"} + } + normalizeFSListArgs(args) + return args, nil +} + +func normalizeFSListArgs(args *fsListArgs) { + pageReq := model.PageReq{ + Page: args.Page, + PerPage: args.PerPage, + } + pageReq.Validate() + args.Page = pageReq.Page + args.PerPage = pageReq.PerPage +} + +func paginateObjs(objs []model.Obj, page, perPage int) (int, []model.Obj) { + total := len(objs) + if page < 1 { + page = 1 + } + if perPage < 1 { + perPage = model.MaxInt + } + offset := page - 1 + if offset > total/perPage { + return total, []model.Obj{} + } + start := offset * perPage + if start > total { + return total, []model.Obj{} + } + end := total + if perPage <= total-start { + end = start + perPage + } + if end > total { + end = total + } + return total, objs[start:end] +} + +func toObjResp(objs []model.Obj, parent string, encrypt bool) []handles.ObjResp { + resp := make([]handles.ObjResp, 0, len(objs)) + for _, obj := range objs { + thumb, _ := model.GetThumb(obj) + mountDetails, _ := model.GetStorageDetails(obj) + resp = append(resp, handles.ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + MountDetails: mountDetails, + }) + } + return resp +} + +func getReadme(meta *model.Meta, path string) string { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.RSub) { + return meta.Readme + } + return "" +} + +func getHeader(meta *model.Meta, path string) string { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.HeaderSub) { + return meta.Header + } + return "" +} + +func isEncrypt(meta *model.Meta, path string) bool { + if common.IsStorageSignEnabled(path) { + return true + } + if meta == nil || meta.Password == "" { + return false + } + return common.MetaCoversPath(meta.Path, path, meta.PSub) +} diff --git a/server/mcp/get_test.go b/server/mcp/get_test.go new file mode 100644 index 0000000000..81fbe39006 --- /dev/null +++ b/server/mcp/get_test.go @@ -0,0 +1,39 @@ +package mcp + +import ( + "encoding/json" + "testing" +) + +func TestParseFSGetArgsRequiresPath(t *testing.T) { + _, err := parseFSGetArgs(json.RawMessage(`{"password":"secret"}`)) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} + +func TestParseFSGetArgs(t *testing.T) { + args, err := parseFSGetArgs(json.RawMessage(`{"path":"/movie.mkv","password":"secret"}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/movie.mkv" { + t.Fatalf("unexpected path: %q", args.Path) + } + if args.Password != "secret" { + t.Fatalf("unexpected password: %q", args.Password) + } +} + +func TestParseFSGetArgsRejectsInvalidJSON(t *testing.T) { + _, err := parseFSGetArgs(json.RawMessage(`"bad"`)) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} diff --git a/server/mcp/handler.go b/server/mcp/handler.go new file mode 100644 index 0000000000..191977eb43 --- /dev/null +++ b/server/mcp/handler.go @@ -0,0 +1,536 @@ +package mcp + +import ( + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/OpenList/v4/server/middlewares" + "github.com/gin-gonic/gin" +) + +const ( + ProtocolVersion = "2025-11-25" + ProtocolVersionHeader = "MCP-Protocol-Version" + SessionHeader = "MCP-Session-Id" + sessionTTL = 30 * time.Minute + maxSessions = 128 + maxUserSessions = 16 +) + +type session struct { + id string + userID uint + protocolVersion string + initialized bool + createdAt time.Time + lastUsedAt time.Time +} + +type Server struct { + mu sync.Mutex + sessions map[string]*session +} + +type request struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id,omitempty"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type response struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type initializeParams struct { + ProtocolVersion string `json:"protocolVersion"` + Capabilities map[string]any `json:"capabilities"` + ClientInfo map[string]any `json:"clientInfo"` +} + +var defaultServer = &Server{ + sessions: map[string]*session{}, +} + +var supportedProtocolVersions = map[string]struct{}{ + "2025-11-25": {}, + "2025-06-18": {}, +} + +func Register(g *gin.RouterGroup) { + mcpGroup := g.Group("/mcp", middlewares.Auth(false), middlewares.AuthAdmin) + mcpGroup.GET("", defaultServer.handleGet) + mcpGroup.POST("", defaultServer.handlePost) + mcpGroup.DELETE("", defaultServer.handleDelete) +} + +func (s *Server) handleGet(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + c.Header("Allow", "POST, DELETE") + c.Status(http.StatusMethodNotAllowed) +} + +func (s *Server) handlePost(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + if !acceptsStreamableHTTP(c.GetHeader("Accept")) { + c.JSON(http.StatusNotAcceptable, response{ + JSONRPC: "2.0", + Error: &rpcError{ + Code: -32000, + Message: "Not Acceptable: client must accept both application/json and text/event-stream", + }, + }) + return + } + + body, err := io.ReadAll(io.LimitReader(c.Request.Body, 1<<20)) + if err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + Error: &rpcError{Code: -32700, Message: "failed to read request body"}, + }) + return + } + + var req request + if err := json.Unmarshal(body, &req); err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + Error: &rpcError{Code: -32700, Message: "parse error"}, + }) + return + } + if req.JSONRPC != "2.0" { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32600, Message: "invalid request"}, + }) + return + } + + if req.Method == "initialize" { + s.handleInitialize(c, req) + return + } + sessionID := c.GetHeader(SessionHeader) + if sessionID == "" { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32000, Message: "missing MCP session"}, + }) + return + } + + currentSession, ok := s.getSession(sessionID) + if !ok { + c.JSON(http.StatusNotFound, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32001, Message: "session not found"}, + }) + return + } + + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if currentSession.userID != user.ID { + c.JSON(http.StatusNotFound, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32001, Message: "session not found"}, + }) + return + } + + if !s.validateRequestProtocolVersion(c.GetHeader(ProtocolVersionHeader), currentSession.protocolVersion) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32000, Message: "missing or unsupported MCP protocol version"}, + }) + return + } + + switch req.Method { + case "ping": + c.JSON(http.StatusOK, response{JSONRPC: "2.0", ID: req.ID, Result: map[string]any{}}) + case "notifications/initialized": + s.markSessionInitialized(sessionID) + c.Status(http.StatusAccepted) + case "tools/list": + if !s.sessionInitialized(sessionID) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32002, Message: "MCP session not initialized"}, + }) + return + } + c.JSON(http.StatusOK, s.handleToolsList(req)) + case "tools/call": + if !s.sessionInitialized(sessionID) { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32002, Message: "MCP session not initialized"}, + }) + return + } + status, resp := s.handleToolsCall(c, req) + c.JSON(status, resp) + default: + c.JSON(http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32601, Message: fmt.Sprintf("method %q not implemented yet", req.Method)}, + }) + } +} + +func (s *Server) handleInitialize(c *gin.Context, req request) { + var params initializeParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + c.JSON(http.StatusBadRequest, response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid initialize params"}, + }) + return + } + } + + protocolVersion := negotiateProtocolVersion(params.ProtocolVersion) + currentSession := s.initializeSession( + c.Request.Context().Value(conf.UserKey).(*model.User).ID, + c.GetHeader(SessionHeader), + protocolVersion, + ) + c.Header(SessionHeader, currentSession.id) + c.JSON(http.StatusOK, response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "protocolVersion": protocolVersion, + "capabilities": map[string]any{ + "tools": map[string]any{ + "listChanged": false, + }, + }, + "serverInfo": map[string]any{ + "name": "OpenList MCP", + "version": conf.Version, + }, + "instructions": "Complete initialization with notifications/initialized, then use tools/list and tools/call. Available tools include openlist.fs.list, openlist.fs.get, and openlist.fs.link.", + }, + }) +} + +func (s *Server) initializeSession(userID uint, requestedID string, protocolVersion string) *session { + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + s.pruneExpiredSessionsLocked(now) + if requestedID != "" { + currentSession, ok := s.sessions[requestedID] + if ok && currentSession != nil && currentSession.userID == userID { + currentSession.initialized = false + currentSession.protocolVersion = protocolVersion + currentSession.lastUsedAt = now + return currentSession + } + } + + return s.createSessionLocked(userID, protocolVersion, now) +} + +func (s *Server) handleDelete(c *gin.Context) { + if !validateOrigin(c.Request) { + c.Status(http.StatusForbidden) + return + } + + currentSession, ok := s.getSession(c.GetHeader(SessionHeader)) + if !ok { + c.Status(http.StatusNotFound) + return + } + + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if currentSession.userID != user.ID { + c.Status(http.StatusNotFound) + return + } + + s.deleteSession(currentSession.id) + c.Status(http.StatusNoContent) +} + +func (s *Server) createSession(userID uint) *session { + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + s.pruneExpiredSessionsLocked(now) + return s.createSessionLocked(userID, ProtocolVersion, now) +} + +func (s *Server) createSessionLocked(userID uint, protocolVersion string, now time.Time) *session { + s.pruneLeastRecentlyUsedUserSessionsLocked(userID, max(0, s.countUserSessionsLocked(userID)-maxUserSessions+1)) + s.pruneLeastRecentlyUsedSessionsLocked(max(0, len(s.sessions)-maxSessions+1)) + currentSession := &session{ + id: random.Token(), + userID: userID, + protocolVersion: protocolVersion, + createdAt: now, + lastUsedAt: now, + } + s.sessions[currentSession.id] = currentSession + return currentSession +} + +func (s *Server) validateRequestProtocolVersion(requestedVersion string, negotiatedVersion string) bool { + if requestedVersion == "" { + _, ok := supportedProtocolVersions[negotiatedVersion] + return ok + } + if _, ok := supportedProtocolVersions[requestedVersion]; !ok { + return false + } + return negotiatedVersion == "" || requestedVersion == negotiatedVersion +} + +func negotiateProtocolVersion(requestedVersion string) string { + if _, ok := supportedProtocolVersions[requestedVersion]; ok { + return requestedVersion + } + return ProtocolVersion +} + +func (s *Server) getSession(id string) (session, bool) { + if id == "" { + return session{}, false + } + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return session{}, false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return session{}, false + } + currentSession.lastUsedAt = now + return *currentSession, true +} + +func (s *Server) markSessionInitialized(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return false + } + currentSession.initialized = true + currentSession.lastUsedAt = now + return true +} + +func (s *Server) sessionInitialized(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + currentSession, ok := s.sessions[id] + if !ok || currentSession == nil { + return false + } + now := time.Now() + if sessionExpired(currentSession, now) { + delete(s.sessions, id) + return false + } + currentSession.lastUsedAt = now + return currentSession.initialized +} + +func (s *Server) deleteSession(id string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.sessions, id) +} + +func (s *Server) pruneExpiredSessionsLocked(now time.Time) { + for id, currentSession := range s.sessions { + if currentSession == nil || sessionExpired(currentSession, now) { + delete(s.sessions, id) + } + } +} + +func (s *Server) pruneLeastRecentlyUsedSessionsLocked(count int) { + for range count { + var ( + oldestID string + oldest time.Time + ) + for id, currentSession := range s.sessions { + if currentSession == nil { + oldestID = id + break + } + lastUsedAt := sessionLastUsedAt(currentSession) + if oldestID == "" || lastUsedAt.Before(oldest) { + oldestID = id + oldest = lastUsedAt + } + } + if oldestID == "" { + return + } + delete(s.sessions, oldestID) + } +} + +func (s *Server) countUserSessionsLocked(userID uint) int { + count := 0 + for _, currentSession := range s.sessions { + if currentSession != nil && currentSession.userID == userID { + count++ + } + } + return count +} + +func (s *Server) pruneLeastRecentlyUsedUserSessionsLocked(userID uint, count int) { + for range count { + var ( + oldestID string + oldest time.Time + ) + for id, currentSession := range s.sessions { + if currentSession == nil { + continue + } + if currentSession.userID != userID { + continue + } + lastUsedAt := sessionLastUsedAt(currentSession) + if oldestID == "" || lastUsedAt.Before(oldest) { + oldestID = id + oldest = lastUsedAt + } + } + if oldestID == "" { + return + } + delete(s.sessions, oldestID) + } +} + +func sessionExpired(currentSession *session, now time.Time) bool { + lastUsedAt := sessionLastUsedAt(currentSession) + return !lastUsedAt.IsZero() && now.Sub(lastUsedAt) > sessionTTL +} + +func sessionLastUsedAt(currentSession *session) time.Time { + if currentSession.lastUsedAt.IsZero() { + return currentSession.createdAt + } + return currentSession.lastUsedAt +} + +func acceptsStreamableHTTP(accept string) bool { + if accept == "" { + return false + } + hasJSON := false + hasSSE := false + for part := range strings.SplitSeq(accept, ",") { + mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(part)) + if err != nil { + continue + } + if q, ok := params["q"]; ok { + quality, err := strconv.ParseFloat(q, 64) + if err == nil && quality == 0 { + continue + } + } + switch mediaType { + case "*/*": + hasJSON = true + hasSSE = true + case "application/*", "application/json": + hasJSON = true + case "text/*", "text/event-stream": + hasSSE = true + } + } + return hasJSON || hasSSE +} + +func validateOrigin(r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin == "" { + return true + } + + originURL, err := url.Parse(origin) + if err != nil || originURL.Host == "" { + return false + } + if strings.EqualFold(originURL.Host, r.Host) { + return strings.EqualFold(originURL.Scheme, requestScheme(r)) + } + + siteURL := common.GetApiUrlFromRequest(r) + if siteURL == "" { + return false + } + siteParsed, err := url.Parse(siteURL) + if err != nil { + return false + } + return strings.EqualFold(originURL.Host, siteParsed.Host) && strings.EqualFold(originURL.Scheme, siteParsed.Scheme) +} + +func requestScheme(r *http.Request) string { + if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { + return "https" + } + return "http" +} diff --git a/server/mcp/handler_test.go b/server/mcp/handler_test.go new file mode 100644 index 0000000000..d3369576b2 --- /dev/null +++ b/server/mcp/handler_test.go @@ -0,0 +1,569 @@ +package mcp + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +func TestInitializeCreatesSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-11-25", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(SessionHeader); got == "" { + t.Fatal("expected session header to be set") + } + + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != ProtocolVersion { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestInitializeNegotiatesSupportedOlderProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-06-18", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != "2025-06-18" { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestInitializeReusesExistingSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + currentSession := srv.createSession(1) + currentSession.initialized = true + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2025-06-18", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, currentSession.id) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(SessionHeader); got != currentSession.id { + t.Fatalf("unexpected session header: got %q want %q", got, currentSession.id) + } + if len(srv.sessions) != 1 { + t.Fatalf("unexpected session count: got %d want %d", len(srv.sessions), 1) + } + reusedSession, ok := srv.getSession(currentSession.id) + if !ok { + t.Fatal("expected existing session to be reused") + } + if reusedSession.initialized { + t.Fatal("expected reused session to require initialized notification again") + } + if reusedSession.protocolVersion != "2025-06-18" { + t.Fatalf("unexpected protocol version: got %q want %q", reusedSession.protocolVersion, "2025-06-18") + } +} + +func TestInitializeNegotiatesUnsupportedProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize", + "params":{ + "protocolVersion":"2026-01-01", + "capabilities":{}, + "clientInfo":{"name":"test-client","version":"1.0.0"} + } + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } + result, ok := resp.Result.(map[string]any) + if !ok { + t.Fatalf("unexpected result type: %T", resp.Result) + } + if result["protocolVersion"] != ProtocolVersion { + t.Fatalf("unexpected protocol version: %v", result["protocolVersion"]) + } +} + +func TestCreateSessionPrunesUserSessionLimit(t *testing.T) { + srv := newTestServer(nil) + var firstSessionID string + for i := range maxUserSessions { + currentSession := srv.createSession(1) + if i == 0 { + firstSessionID = currentSession.id + } + } + srv.createSession(2) + srv.createSession(1) + + if _, ok := srv.sessions[firstSessionID]; ok { + t.Fatal("expected oldest user session to be pruned") + } + if got := countSessionsForUser(srv, 1); got != maxUserSessions { + t.Fatalf("unexpected user session count: got %d want %d", got, maxUserSessions) + } + if got := len(srv.sessions); got != maxUserSessions+1 { + t.Fatalf("unexpected total session count: got %d want %d", got, maxUserSessions+1) + } +} + +func TestCreateSessionPrunesGlobalSessionLimit(t *testing.T) { + srv := newTestServer(nil) + var firstSessionID string + for i := range maxSessions { + currentSession := srv.createSession(uint(i + 1)) + if i == 0 { + firstSessionID = currentSession.id + } + } + srv.createSession(uint(maxSessions + 1)) + + if _, ok := srv.sessions[firstSessionID]; ok { + t.Fatal("expected oldest global session to be pruned") + } + if got := len(srv.sessions); got != maxSessions { + t.Fatalf("unexpected session count: got %d want %d", got, maxSessions) + } +} + +func TestPostAcceptsJSONOnly(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "application/json") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostAcceptsWildcardAccept(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "*/*") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostInvalidAcceptReturnsJSONRPCError(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"initialize" + }`)) + req.Header.Set("Accept", "image/png") + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotAcceptable { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNotAcceptable) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostUsesNegotiatedProtocolVersionWhenHeaderMissing(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, protocolVersion: "2025-06-18", initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusOK) + } + resp := decodeResponse(t, w) + if resp.Error != nil { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsUnsupportedProtocolVersion(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, "2025-03-26") + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsProtocolVersionMismatch(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(map[string]*session{ + "s1": {id: "s1", userID: 1, protocolVersion: "2025-06-18", initialized: true}, + }) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "s1") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsMissingSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusBadRequest) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32000 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestPostRejectsUnknownSessionWithNotFound(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + r := gin.New() + r.POST("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handlePost(c) + }) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/mcp", strings.NewReader(`{ + "jsonrpc":"2.0", + "id":1, + "method":"tools/list" + }`)) + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Origin", "http://example.com") + req.Header.Set(ProtocolVersionHeader, ProtocolVersion) + req.Header.Set(SessionHeader, "unknown") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNotFound) + } + resp := decodeResponse(t, w) + if resp.Error == nil || resp.Error.Code != -32001 { + t.Fatalf("unexpected error response: %+v", resp.Error) + } +} + +func TestDeleteRemovesSession(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := newTestServer(nil) + + currentSession := srv.createSession(1) + r := gin.New() + r.DELETE("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + srv.handleDelete(c) + }) + + req := httptest.NewRequest(http.MethodDelete, "http://example.com/mcp", nil) + req.Header.Set("Origin", "http://example.com") + req.Header.Set(SessionHeader, currentSession.id) + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNoContent { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusNoContent) + } + if _, ok := srv.getSession(currentSession.id); ok { + t.Fatal("expected session to be deleted") + } +} + +func TestGetReturnsMethodNotAllowed(t *testing.T) { + gin.SetMode(gin.TestMode) + + r := gin.New() + r.GET("/mcp", func(c *gin.Context) { + common.GinAppendValues(c, conf.UserKey, &model.User{ID: 1, Role: model.ADMIN}) + defaultServer.handleGet(c) + }) + + req := httptest.NewRequest(http.MethodGet, "http://example.com/mcp", nil) + req.Header.Set("Origin", "http://example.com") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusMethodNotAllowed { + t.Fatalf("unexpected status: got %d want %d", w.Code, http.StatusMethodNotAllowed) + } + if allow := w.Header().Get("Allow"); allow != "POST, DELETE" { + t.Fatalf("unexpected Allow header: got %q want %q", allow, "POST, DELETE") + } +} + +func newTestServer(sessions map[string]*session) *Server { + if sessions == nil { + sessions = map[string]*session{} + } + now := time.Now() + for _, currentSession := range sessions { + if currentSession.createdAt.IsZero() { + currentSession.createdAt = now + } + if currentSession.lastUsedAt.IsZero() { + currentSession.lastUsedAt = now + } + } + return &Server{sessions: sessions} +} + +func countSessionsForUser(srv *Server, userID uint) int { + count := 0 + for _, currentSession := range srv.sessions { + if currentSession != nil && currentSession.userID == userID { + count++ + } + } + return count +} diff --git a/server/mcp/link_test.go b/server/mcp/link_test.go new file mode 100644 index 0000000000..10896d80ca --- /dev/null +++ b/server/mcp/link_test.go @@ -0,0 +1,143 @@ +package mcp + +import ( + "context" + "encoding/json" + "net/http/httptest" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/gin-gonic/gin" +) + +var settingCacheMu sync.Mutex + +func TestParseFSLinkArgs(t *testing.T) { + args, err := parseFSLinkArgs(json.RawMessage(`{"path":"/file.txt","password":"pw","type":"thumb"}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/file.txt" || args.Password != "pw" || args.Type != "thumb" { + t.Fatalf("unexpected args: %+v", args) + } +} + +func TestParseFSLinkArgsRequiresPath(t *testing.T) { + _, err := parseFSLinkArgs(json.RawMessage(`{"type":"thumb"}`)) + if err == nil || err.Code != -32602 { + t.Fatalf("unexpected error: %+v", err) + } +} + +func TestCanProxyFile(t *testing.T) { + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test"}, + storage: model.Storage{ + MountPath: "/", + }, + } + if canProxyFile(storage, "file.bin") { + t.Fatal("unexpected proxy support") + } + storage.config.OnlyProxy = true + if !canProxyFile(storage, "file.bin") { + t.Fatal("expected proxy support") + } +} + +func TestCanProxyFileIgnoresWebdavProxyURLPolicy(t *testing.T) { + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test"}, + storage: model.Storage{ + MountPath: "/", + Proxy: model.Proxy{ + WebdavPolicy: "use_proxy_url", + }, + }, + } + + if canProxyFile(storage, "file.bin") { + t.Fatal("webdav proxy url policy should not enable MCP proxy support") + } +} + +func TestBuildFSLinkInfoUsesProxyWhenStorageRequiresProxy(t *testing.T) { + gin.SetMode(gin.TestMode) + settingCacheMu.Lock() + t.Cleanup(func() { + op.Cache.ClearAll() + settingCacheMu.Unlock() + }) + op.Cache.SetSetting(conf.LinkExpiration, &model.SettingItem{Key: conf.LinkExpiration, Value: "0"}) + op.Cache.SetSetting(conf.Token, &model.SettingItem{Key: conf.Token, Value: "test-token"}) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest("POST", "http://example.com/mcp", nil) + ctx := context.WithValue(c.Request.Context(), conf.ApiUrlKey, "http://openlist.test") + storage := &fsLinkTestDriver{ + config: driver.Config{Name: "Test", OnlyProxy: true}, + storage: model.Storage{ + MountPath: "/", + Proxy: model.Proxy{ + DownProxyURL: "http://proxy.test", + DisableProxySign: true, + }, + }, + } + obj := &model.ObjectURL{ + Object: model.Object{Name: "file.txt", Size: 12}, + Url: model.Url{Url: "http://direct.test/file.txt"}, + } + + meta := &model.Meta{Path: "/file.txt", Password: "secret"} + resp, err := buildFSLinkInfo(ctx, c, "/file.txt", &fsLinkArgs{Path: "/file.txt"}, obj, meta, storage) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if resp.URL != "http://proxy.test/file.txt" || resp.URLType != "proxy" { + t.Fatalf("unexpected selected link: %+v", resp) + } + if resp.DirectURL != "" { + t.Fatalf("direct link should not be resolved for proxy storage: %+v", resp) + } +} + +type fsLinkTestDriver struct { + config driver.Config + storage model.Storage +} + +func (d *fsLinkTestDriver) Config() driver.Config { + return d.config +} + +func (d *fsLinkTestDriver) GetStorage() *model.Storage { + return &d.storage +} + +func (d *fsLinkTestDriver) SetStorage(storage model.Storage) { + d.storage = storage +} + +func (d *fsLinkTestDriver) GetAddition() driver.Additional { + return nil +} + +func (d *fsLinkTestDriver) Init(context.Context) error { + return nil +} + +func (d *fsLinkTestDriver) Drop(context.Context) error { + return nil +} + +func (d *fsLinkTestDriver) List(context.Context, model.Obj, model.ListArgs) ([]model.Obj, error) { + return nil, nil +} + +func (d *fsLinkTestDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + return nil, nil +} diff --git a/server/mcp/list_test.go b/server/mcp/list_test.go new file mode 100644 index 0000000000..8079245aa5 --- /dev/null +++ b/server/mcp/list_test.go @@ -0,0 +1,54 @@ +package mcp + +import ( + "encoding/json" + "testing" +) + +func TestParseFSListArgsRequiresPath(t *testing.T) { + assertFSListPathRequired(t, json.RawMessage(`{"refresh":true}`)) +} + +func TestParseFSListArgsRejectsEmptyArguments(t *testing.T) { + for _, raw := range []json.RawMessage{nil, json.RawMessage(`null`)} { + assertFSListInvalidArguments(t, raw) + } +} + +func TestParseFSListArgsRejectsInvalidJSON(t *testing.T) { + assertFSListInvalidArguments(t, json.RawMessage(`"bad"`)) +} + +func TestParseFSListArgs(t *testing.T) { + args, err := parseFSListArgs(json.RawMessage(`{"path":"/movies","password":"secret","refresh":true,"page":2,"per_page":10}`)) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if args.Path != "/movies" || args.Password != "secret" || !args.Refresh || args.Page != 2 || args.PerPage != 10 { + t.Fatalf("unexpected args: %+v", args) + } +} + +func assertFSListPathRequired(t *testing.T, raw json.RawMessage) { + t.Helper() + + _, err := parseFSListArgs(raw) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 || err.Message != "path is required" { + t.Fatalf("unexpected error: %+v", err) + } +} + +func assertFSListInvalidArguments(t *testing.T, raw json.RawMessage) { + t.Helper() + + _, err := parseFSListArgs(raw) + if err == nil { + t.Fatal("expected error") + } + if err.Code != -32602 || err.Message != "invalid openlist.fs.list arguments" { + t.Fatalf("unexpected error: %+v", err) + } +} diff --git a/server/mcp/tools.go b/server/mcp/tools.go new file mode 100644 index 0000000000..188c674ce6 --- /dev/null +++ b/server/mcp/tools.go @@ -0,0 +1,122 @@ +package mcp + +import "encoding/json" + +type tool struct { + Name string `json:"name"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + InputSchema toolInputSchema `json:"inputSchema"` +} + +type toolInputSchema struct { + Type string `json:"type"` + Properties map[string]schemaProperty `json:"properties,omitempty"` + Required []string `json:"required,omitempty"` +} + +type schemaProperty struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` +} + +type toolsListParams struct { + Cursor string `json:"cursor,omitempty"` +} + +var openListTools = []tool{ + { + Name: "openlist.fs.list", + Title: "OpenList FS List", + Description: "List files and directories under a mount path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "Mount path to list, for example \"/\" or \"/movies\".", + }, + "refresh": { + Type: "boolean", + Description: "Refresh the directory listing before returning results.", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + "page": { + Type: "integer", + Description: "1-based page number.", + }, + "per_page": { + Type: "integer", + Description: "Page size.", + }, + }, + Required: []string{"path"}, + }, + }, + { + Name: "openlist.fs.get", + Title: "OpenList FS Get", + Description: "Get file or directory details for a mount path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "Mount path to inspect, for example \"/movies/demo.mp4\".", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + }, + Required: []string{"path"}, + }, + }, + { + Name: "openlist.fs.link", + Title: "OpenList FS Link", + Description: "Return usable link information for a file path that the current user can access.", + InputSchema: toolInputSchema{ + Type: "object", + Properties: map[string]schemaProperty{ + "path": { + Type: "string", + Description: "File mount path, for example \"/movies/demo.mp4\".", + }, + "password": { + Type: "string", + Description: "Optional password for protected paths.", + }, + "type": { + Type: "string", + Description: "Optional link type forwarded to storage drivers.", + }, + }, + Required: []string{"path"}, + }, + }, +} + +func (s *Server) handleToolsList(req request) response { + var params toolsListParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return response{ + JSONRPC: "2.0", + ID: req.ID, + Error: &rpcError{Code: -32602, Message: "invalid tools/list params"}, + } + } + } + + return response{ + JSONRPC: "2.0", + ID: req.ID, + Result: map[string]any{ + "tools": openListTools, + }, + } +} diff --git a/server/router.go b/server/router.go index 8e90688248..7330bd2c33 100644 --- a/server/router.go +++ b/server/router.go @@ -41,6 +41,7 @@ func Init(e *gin.Engine) { } WebDav(g.Group("/dav")) S3(g.Group("/s3")) + MCP(g) downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) signCheck := middlewares.Down(sign.Verify) From 3abdf7990b4f65134c24ef1d83ac399a439cf277 Mon Sep 17 00:00:00 2001 From: Sorubedo <102888681+sorubedo@users.noreply.github.com> Date: Fri, 19 Jun 2026 22:44:30 +0800 Subject: [PATCH 046/107] fix(drivers/ftp): add cwd_list option and filter path-separated entries (#2621) 1. Add `cwd_list` option to switch LIST command logic Use CWD + LIST "" instead of LIST to fix garbled non-UTF8 paths on legacy Chinese FTP servers, eliminate fake duplicate folders. 2. Filter entries with "/" in filename Skip cdir marker entries carrying full path names, compliant with FTP filename specification. Co-authored-by: Claude <81847+claude@users.noreply.github.com> --- drivers/ftp/driver.go | 27 +++++++++++++++++++++++++-- drivers/ftp/meta.go | 1 + 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 4845337861..0d3e803e03 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -5,6 +5,7 @@ import ( "errors" "io" stdpath "path" + "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -51,13 +52,35 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if err := d.login(); err != nil { return nil, err } - entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + + path := encode(dir.GetPath(), d.Encoding) + + var entries []*ftp.Entry + var err error + + if d.CwdList { + origDir, cwdErr := d.conn.CurrentDir() + if cwdErr != nil { + return nil, cwdErr + } + if cwdErr = d.conn.ChangeDir(path); cwdErr != nil { + return nil, cwdErr + } + entries, err = d.conn.List("") + if restoreErr := d.conn.ChangeDir(origDir); restoreErr != nil { + d.conn = nil + } + } else { + entries, err = d.conn.List(path) + } + if err != nil { return nil, err } + res := make([]model.Obj, 0) for _, entry := range entries { - if entry.Name == "." || entry.Name == ".." { + if entry.Name == "." || entry.Name == ".." || strings.Contains(entry.Name, "/") { continue } name := decode(entry.Name, d.Encoding) diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 0ec0e735eb..6dc0d2cb61 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -27,6 +27,7 @@ type Addition struct { Encoding string `json:"encoding" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` + CwdList bool `json:"cwd_list" type:"bool" default:"false" help:"enter directory before listing"` driver.RootPath } From aa70b6f86a591455847f3a2e2ee8228f0a8473c8 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:16 +0800 Subject: [PATCH 047/107] feat(stream): link refresh, self-healing reader, seekable prefetch, and upload hash rework - Add link refresh capability with RefreshableRangeReader for expired download links - Implement self-healing reader for interrupted stream reconnection - Add seekable prefetch window with strict tests - Enhance hash calculation and upload logic for various stream types - Fix link expiry detection to avoid treating context cancellation as expired - Optimize link cache logic, remove SyncClosers dependency - Support all 4xx client errors in expired link check - Fix local file 'file already closed' error - Unify hash calculation progress weight to 100 - Add slow network support with adjusted timeout and retry --- internal/model/args.go | 8 + internal/net/serve.go | 12 +- internal/op/fs.go | 46 +++- internal/op/storage.go | 5 +- internal/stream/stream.go | 12 +- internal/stream/util.go | 401 +++++++++++++++++++++++++++++++++++ internal/stream/util_test.go | 160 ++++++++++++++ pkg/utils/hash.go | 6 + server/handles/fsup.go | 12 ++ 9 files changed, 647 insertions(+), 15 deletions(-) create mode 100644 internal/stream/util_test.go diff --git a/internal/model/args.go b/internal/model/args.go index 073c94a63a..d165908fb3 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -25,6 +25,10 @@ type LinkArgs struct { Redirect bool } +// LinkRefresher is a callback function type for refreshing download links +// It returns a new Link and the associated object, or an error +type LinkRefresher func(ctx context.Context) (*Link, Obj, error) + type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) @@ -37,6 +41,10 @@ type Link struct { PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 + // Refresher is a callback to refresh the link when it expires during long downloads + // This field is not serialized and is optional - if nil, no refresh will be attempted + Refresher LinkRefresher `json:"-"` + utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` diff --git a/internal/net/serve.go b/internal/net/serve.go index 89a209d881..70b2ed587c 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,7 +7,7 @@ import ( "fmt" "io" "mime/multipart" - gonet "net" + gonet "net" // 标准库 net 包：用于 Dialer 与 safeTransport 的云元数据 IP 拦截 "net/http" "strconv" "strings" @@ -287,12 +287,20 @@ func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + // 快速连接超时：10秒建立连接，失败快速重试 + DialContext: (&gonet.Dialer{ + Timeout: 10 * time.Second, // TCP握手超时 + KeepAlive: 30 * time.Second, // TCP keep-alive + }).DialContext, + // 响应头超时：15秒等待服务器响应头（平衡API调用与下载检测） + ResponseHeaderTimeout: 15 * time.Second, + // 允许长时间读取数据（无 IdleConnTimeout 限制） } SetProxyIfConfigured(transport) return &http.Client{ - Timeout: time.Hour * 48, + Timeout: time.Hour * 48, // 总超时保持48小时（允许大文件慢速下载） Transport: &safeTransport{base: transport}, } } diff --git a/internal/op/fs.go b/internal/op/fs.go index f82a3ca8f8..0148521203 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -149,6 +149,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, + HashInfo: utils.NewHashInfo(nil, ""), }, nil case driver.IRootPath: return &model.Object{ @@ -157,6 +158,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") @@ -246,6 +248,8 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } + // SyncClosers 已关闭（文件句柄已关闭），删除缓存条目，重新获取 + Cache.linkCache.DeleteKey(key) } fn := func() (*objWithLink, error) { @@ -261,11 +265,37 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } + + // Set up link refresher for automatic refresh on expiry during long downloads + // This enables all download scenarios to handle link expiration gracefully + if link.Refresher == nil { + storageCopy := storage + pathCopy := path + argsCopy := args + link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + log.Infof("Refreshing download link for: %s", pathCopy) + // Get fresh link directly from storage, bypassing cache + file, err := GetUnwrap(refreshCtx, storageCopy, pathCopy) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file for refresh") + } + newLink, err := storageCopy.Link(refreshCtx, file, argsCopy) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to refresh link") + } + return newLink, file, nil + } + } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) - } else { + } else if link.RequireReference { + // 本地文件等需要引用计数的链接，缓存与文件句柄生命周期绑定 Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) + } else { + // 不需要引用计数（如云盘链接无过期时间），使用默认 TTL，多客户端复用 + Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } @@ -325,9 +355,16 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + if errs.IsObjectNotFound(err) { + // Retry once after a short delay (handles cloud storage API sync delay) + log.Debugf("[op] parent dir [%s] not found immediately after creation, retrying...", parentPath) + time.Sleep(100 * time.Millisecond) + parentDir, err = GetUnwrap(ctx, storage, parentPath) + } + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } } if !parentDir.IsDir() { return nil, errs.NotFolder @@ -360,6 +397,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) @@ -685,6 +723,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -753,6 +792,7 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e31b..2e93bf5693 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -368,7 +368,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, }(d) select { case r := <-resultChan: - ret.StorageDetails = r + if r != nil { + ret.StorageDetails = r + } case <-time.After(time.Second): } return ret @@ -419,6 +421,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver Name: name, Modified: v.GetStorage().Modified, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), } if !found { idx := len(files) diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 9ae8313f0c..d7a0936ac4 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -254,15 +254,9 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - if _, ok := rr.(*model.FileRangeReader); ok { - var rc io.ReadCloser - rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err - } - fs.Reader = rc - fs.Add(rc) - } + // IMPORTANT: Do NOT create Reader early for FileRangeReader! + // Let generateReader() create it on-demand when actually needed for reading + // This prevents the Reader from being consumed by intermediate operations like hash calculation fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil diff --git a/internal/stream/util.go b/internal/stream/util.go index 2947fcbc2b..a10772d54e 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "net/http" + "strings" "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -20,13 +22,298 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + // 链接刷新相关常量 + MAX_LINK_REFRESH_COUNT = 50 // 下载链接最大刷新次数（支持长时间传输） + + // RangeRead 重试相关常量 + MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead 最大重试次数（从3增加到5） +) + type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + + // Don't treat context cancellation as link expiration + // This happens when user pauses/seeks video or cancels download + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // All 4xx client errors may indicate expired/invalid links + // 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, 410 Gone, etc. + if code >= 400 && code < 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + // Wrap the ReadCloser with self-healing capability to detect 0-byte reads + // This handles cases where cloud providers return 200 OK but empty body for expired links + return &selfHealingReadCloser{ + ReadCloser: rc, + refresher: r, + ctx: ctx, + httpRange: httpRange, + firstRead: false, + closed: false, + }, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= MAX_LINK_REFRESH_COUNT { + return fmt.Errorf("max refresh attempts (%d) reached", MAX_LINK_REFRESH_COUNT) + } + + log.Infof("Link expired, attempting to refresh...") + // Use independent context for refresh to prevent cancellation from affecting link refresh + refreshCtx := context.WithoutCancel(ctx) + newLink, _, refreshErr := r.link.Refresher(refreshCtx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + +// selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link +// if the upstream reader dies before the requested range is fully delivered. +type selfHealingReadCloser struct { + io.ReadCloser + refresher *RefreshableRangeReader + ctx context.Context + httpRange http_range.Range + firstRead bool + bytesRead int64 + closed bool + mu sync.Mutex +} + +func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return 0, errors.New("read from closed reader") + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + wasFirstRead := !s.firstRead + s.firstRead = true + + // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) + if s.shouldReconnectAfterRead(wasFirstRead, n, err) { + if reconnectErr := s.reconnectFromCurrentOffsetLocked(); reconnectErr != nil { + log.Errorf("Failed to refresh link after interrupted read: %v", reconnectErr) + return n, err + } + + if n > 0 { + return n, nil + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + return n, err + } + + return n, err +} + +func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n int, err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if s.remainingBytes() <= 0 { + return false + } + + if wasFirstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + return true + } + + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + errStr := strings.ToLower(err.Error()) + if strings.Contains(errStr, "connection reset by peer") { + log.Warnf("Detected upstream connection reset after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + return false +} + +func (s *selfHealingReadCloser) reconnectFromCurrentOffsetLocked() error { + nextRange := s.httpRange + nextRange.Start += s.bytesRead + if nextRange.Length >= 0 { + nextRange.Length -= s.bytesRead + } + + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + if refreshErr != nil { + s.refresher.mu.Unlock() + return refreshErr + } + + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + if getErr != nil { + return getErr + } + + newRc, rangeErr := reader.RangeRead(s.ctx, nextRange) + if rangeErr != nil { + return rangeErr + } + + _ = s.ReadCloser.Close() + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected from offset %d", nextRange.Start) + return nil +} + +func (s *selfHealingReadCloser) remainingBytes() int64 { + length := s.httpRange.Length + if length < 0 || s.httpRange.Start+length > s.refresher.size { + length = s.refresher.size - s.httpRange.Start + } + remaining := length - s.bytesRead + if remaining < 0 { + return 0 + } + return remaining +} + +func (s *selfHealingReadCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + return s.ReadCloser.Close() +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -173,6 +460,120 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// file: 文件流 +// buf: 目标缓冲区 +// off: 读取的起始偏移量 +// 返回值: 实际读取的字节数和错误 +// 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） +// 注意：链接刷新现在由 RefreshableRangeReader 内部的 selfHealingReadCloser 自动处理 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { + length := int64(len(buf)) + var lastErr error + + // 重试最多 MAX_RANGE_READ_RETRY_COUNT 次 + for retry := 0; retry < MAX_RANGE_READ_RETRY_COUNT; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return 0, err + } + lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) + log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) + // 快速重试：1秒、2秒、3秒、4秒、5秒（连接失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) + continue + } + + n, err := io.ReadFull(reader, buf) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + + if err == nil { + return n, nil + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return n, err + } + + lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + + // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + // 注意：0字节读取导致的链接过期现在由 selfHealingReadCloser 自动处理 + time.Sleep(time.Duration(retry+1) * time.Second) + } + + return 0, lastErr +} + +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + // 如果已经有完整缓存文件，直接使用 + if cache := file.GetFile(); cache != nil { + hashFunc := hashType.NewFunc() + cache.Seek(0, io.SeekStart) + _, err := io.Copy(hashFunc, cache) + if err != nil { + return "", err + } + if up != nil && progressWeight > 0 { + (*up)(progressWeight) + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil + } + + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + + var n int + var err error + + // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader + // 这样后续发送时 Reader 还能正常工作 + if _, ok := file.(*SeekableStream); ok { + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } else { + // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） + n, err = io.ReadFull(file, buf[:readSize]) + if err != nil { + // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } + } + + if err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + hashFunc.Write(buf[:n]) + offset += int64(n) + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReader interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go new file mode 100644 index 0000000000..6dc4c8a09c --- /dev/null +++ b/internal/stream/util_test.go @@ -0,0 +1,160 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != int64(len(data)-5) { + t.Fatalf("resumed range length = %d, want %d", resumedRanges[0].Length, len(data)-5) + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != -1 { + t.Fatalf("resumed range length = %d, want -1", resumedRanges[0].Length) + } +} + +type flakyReadCloser struct { + data []byte + failAfter int + failErr error + failed bool +} + +func newFlakyReadCloser(data []byte, failAfter int, failErr error) *flakyReadCloser { + return &flakyReadCloser{ + data: data, + failAfter: failAfter, + failErr: failErr, + } +} + +func (f *flakyReadCloser) Read(p []byte) (int, error) { + if f.failed { + return 0, io.EOF + } + if f.failAfter >= len(f.data) { + f.failed = true + n := copy(p, f.data) + return n, io.EOF + } + + n := copy(p, f.data[:f.failAfter]) + f.failed = true + return n, f.failErr +} + +func (f *flakyReadCloser) Close() error { + return nil +} + +func sliceForRange(data []byte, httpRange http_range.Range) []byte { + start := int(httpRange.Start) + length := int(httpRange.Length) + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > int64(len(data)) { + length = len(data) - start + } + return data[start : start+length] +} diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 596e61e541..c4b4e735f2 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -90,6 +90,12 @@ var ( // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) + + // SHA1_128K is SHA1 of first 128KB, used by 115 driver for rapid upload + SHA1_128K = RegisterHash("sha1_128k", "SHA1-128K", 40, sha1.New) + + // PRE_HASH is SHA1 of first 1024 bytes, used by Aliyundrive for rapid upload + PRE_HASH = RegisterHash("pre_hash", "PRE-HASH", 40, sha1.New) ) // HashData get hash of one hashType diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cdf..54cdb4feeb 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -93,6 +93,12 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) @@ -190,6 +196,12 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) From 8871230fd286976f1aa52a5ba4ea43a79652f023 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:24 +0800 Subject: [PATCH 048/107] feat(google_drive): duplicate filename handling, folder lock, retry, and MD5 checksum - Add duplicate filename handling to ensure file name uniqueness - Add folder creation lock mechanism and retry logic - Increase MakeDir wait time for Google Drive API sync delay - Add retry mechanism for small file upload read errors - Update Put method to support MD5 checksum on seekable and non-seekable streams --- drivers/google_drive/driver.go | 163 +++++++++++++++++++++++++++++---- drivers/google_drive/util.go | 51 +++++++++++ 2 files changed, 197 insertions(+), 17 deletions(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 94ef854f2f..1e2e476ce5 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -3,17 +3,29 @@ package google_drive import ( "context" "fmt" + "io" "net/http" "strconv" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) +// mkdirLocks prevents race conditions when creating folders with the same name +// Google Drive allows duplicate folder names, so we need application-level locking +var mkdirLocks sync.Map // map[string]*sync.Mutex - key is parentID + "/" + dirName + type GoogleDrive struct { model.Storage Addition @@ -67,15 +79,76 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // Use per-folder lock to prevent concurrent creation of same folder + // This is critical because Google Drive allows duplicate folder names + lockKey := parentDir.GetID() + "/" + dirName + lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() + + // Check if folder already exists with retry to handle API eventual consistency + escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") + query := map[string]string{ + "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), + "fields": "files(id)", + } + + var existingFiles Files + err := retry.Do(func() error { + var checkErr error + _, checkErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + return checkErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(200*time.Millisecond), + ) + + // If query succeeded and folder exists, return success (idempotent) + if err == nil && len(existingFiles.Files) > 0 { + log.Debugf("[google_drive] Folder '%s' already exists in parent %s, skipping creation", dirName, parentDir.GetID()) + return nil + } + // If query failed, return error to prevent duplicate creation + if err != nil { + return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) + } + + // Create new folder (only when confirmed folder doesn't exist) data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { - req.SetBody(data) - }, nil) - return err + + var createErr error + err = retry.Do(func() error { + _, createErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return createErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) + + if err != nil { + return err + } + + // Wait for API eventual consistency before releasing lock + // This helps prevent race conditions where a concurrent request + // checks for folder existence before the newly created folder is visible + // 500ms is needed because Google Drive API has significant sync delay + time.Sleep(500 * time.Millisecond) + + return nil } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -111,8 +184,44 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj := stream.GetExist() +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + // 1. 准备MD5（用于完整性校验） + md5Hash := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验（Google Drive会自动校验） + } + } else { + // 不可重复读取的流（如 HTTP body） + if len(md5Hash) != utils.MD5.Width { + // 缓存整个文件并计算 MD5 + var err error + _, md5Hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验 + } else if file.GetFile() == nil { + // 有 MD5 但没有缓存，需要缓存以支持后续 RangeRead + _, err := file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + } + + // 2. 初始化可恢复上传会话 + obj := file.GetExist() var ( e Error url string @@ -125,7 +234,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi data = base.Json{} } else { data = base.Json{ - "name": stream.GetName(), + "name": file.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" @@ -133,8 +242,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, - "X-Upload-Content-Type": stream.GetMimetype(), - "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + "X-Upload-Content-Type": file.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(file.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { @@ -151,20 +260,40 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi if err != nil { return err } - return d.Put(ctx, dstDir, stream, up) + return d.Put(ctx, dstDir, file, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } + + // 3. 上传文件内容 putUrl := res.Header().Get("location") - if stream.GetSize() < d.ChunkSize*1024*1024 { - _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, stream)) - }, nil) + if file.GetSize() < d.ChunkSize*1024*1024 { + // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） + err = retry.Do(func() error { + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) + }, nil) + return err + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + return err } else { - err = d.chunkUpload(ctx, stream, putUrl, up) + // 大文件分片上传 + return d.chunkUpload(ctx, file, putUrl, up) } - return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa44..1fc68c3359 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -296,9 +296,60 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { res = append(res, resp.Files...) } + + // Handle duplicate filenames by adding suffixes like (1), (2), etc. + // Google Drive allows multiple files with the same name in one folder, + // but OpenList uses path-based file system which requires unique names + res = handleDuplicateNames(res) + return res, nil } +// handleDuplicateNames adds suffixes to duplicate filenames to make them unique +// For example: file.txt, file (1).txt, file (2).txt +func handleDuplicateNames(files []File) []File { + if len(files) <= 1 { + return files + } + + // Track how many files with each name we've seen + nameCount := make(map[string]int) + + // First pass: count occurrences of each name + for _, file := range files { + nameCount[file.Name]++ + } + + // Second pass: add suffixes to duplicates + nameIndex := make(map[string]int) + for i := range files { + name := files[i].Name + if nameCount[name] > 1 { + index := nameIndex[name] + nameIndex[name]++ + + if index > 0 { + // Add suffix for all except the first occurrence + // Split name into base and extension + ext := "" + base := name + for j := len(name) - 1; j >= 0; j-- { + if name[j] == '.' { + ext = name[j:] + base = name[:j] + break + } + } + + // Add (1), (2), etc. suffix + files[i].Name = fmt.Sprintf("%s (%d)%s", base, index, ext) + } + } + } + + return files +} + // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File From beacb2feb75702d122795d08edff0eebaefe3259 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:32 +0800 Subject: [PATCH 049/107] feat(115_open): permanent delete, proxy_range, offline task fixes, and error handling - Add permanent delete feature with recycle-bin retry logic - Expose proxy_range option in driver meta - Fix duplicate link error handling via error code 10008 detection - Fix folder delete cid JSON number deserialization error - Move offline task Remove before Transfer - Cleanup completed offline download tasks --- drivers/115_open/driver.go | 328 ++++++++- drivers/115_open/driver_test.go | 639 ++++++++++++++++++ drivers/115_open/meta.go | 8 +- drivers/115_open/upload.go | 44 +- internal/offline_download/115_open/client.go | 503 +++++++++++++- .../offline_download/115_open/client_test.go | 616 +++++++++++++++++ 6 files changed, 2106 insertions(+), 32 deletions(-) create mode 100644 drivers/115_open/driver_test.go create mode 100644 internal/offline_download/115_open/client_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index b2559ba66d..36cc469ba2 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,6 +2,7 @@ package _115_open import ( "context" + "encoding/json" "errors" "fmt" "net/http" @@ -21,6 +22,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -32,6 +34,12 @@ type Open115 struct { parentPath string } +var ( + // 回收站列表存在短暂最终一致性延迟，永久删除 fallback 查找增加短重试。 + recycleBinLookupMaxAttempts = 4 + recycleBinLookupRetryDelay = 300 * time.Millisecond +) + func (d *Open115) Config() driver.Config { return config } @@ -101,13 +109,20 @@ func (d *Open115) Drop(ctx context.Context) error { } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + start := time.Now() + log.Infof("[115] List request started for dir: %s (ID: %s)", dir.GetName(), dir.GetID()) + var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) + pageCount := 0 + for { if err := d.WaitLimit(ctx); err != nil { return nil, err } + + pageStart := time.Now() resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -117,7 +132,12 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) // Cur: 1, ShowDir: true, }) + pageDuration := time.Since(pageStart) + pageCount++ + log.Infof("[115] GetFiles page %d took: %v (offset=%d, limit=%d)", pageCount, pageDuration, offset, pageSize) + if err != nil { + log.Errorf("[115] GetFiles page %d failed after %v: %v", pageCount, pageDuration, err) return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { @@ -129,10 +149,17 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } offset += pageSize } + + totalDuration := time.Since(start) + log.Infof("[115] List request completed in %v (%d pages, %d files)", totalDuration, pageCount, len(res)) + return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + start := time.Now() + log.Infof("[115] Link request started for file: %s", file.GetName()) + if err := d.WaitLimit(ctx); err != nil { return nil, err } @@ -148,14 +175,25 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc + + apiStart := time.Now() + log.Infof("[115] Calling DownURL API...") resp, err := d.client.DownURL(ctx, pc, ua) + apiDuration := time.Since(apiStart) + log.Infof("[115] DownURL API took: %v", apiDuration) + if err != nil { + log.Errorf("[115] DownURL API failed after %v: %v", apiDuration, err) return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } + + totalDuration := time.Since(start) + log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) + return &model.Link{ URL: u.URL.URL, Header: http.Header{ @@ -259,13 +297,164 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if !ok { return fmt.Errorf("can't convert obj") } - _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + resp, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } + if d.RemoveWay != "delete" { + return nil + } + return d.removePermanently(ctx, _obj, resp) +} + +func (d *Open115) removePermanently(ctx context.Context, obj *Obj, deleteResp []string) error { + var directDeleteErr error + for _, tid := range deleteResp { + tid = strings.TrimSpace(tid) + if tid == "" { + continue + } + if err := d.deleteRecycleBinEntry(ctx, tid); err == nil { + return nil + } else if directDeleteErr == nil { + directDeleteErr = err + } + } + + recycleEntry, err := d.findRecycleBinEntryWithRetry(ctx, obj) + if err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin candidate: %w; fallback lookup failed: %v", directDeleteErr, err) + } + return err + } + if err := d.deleteRecycleBinEntry(ctx, recycleEntry.ID); err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin entry %s after candidate delete error %v: %w", recycleEntry.ID, directDeleteErr, err) + } + return err + } + return nil +} + +func (d *Open115) deleteRecycleBinEntry(ctx context.Context, tid string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _, err := d.client.RbDelete(ctx, tid) + return err +} + +func (d *Open115) findRecycleBinEntry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 200 + } else if pageSize > 1150 { + pageSize = 1150 + } + + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.RbList(ctx, pageSize, offset) + if err != nil { + return nil, err + } + if entry := matchRecycleBinEntry(obj, resp.Files); entry != nil { + return entry, nil + } + + count, err := strconv.ParseInt(resp.Count, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse recycle bin count %q: %w", resp.Count, err) + } + offset += pageSize + if offset >= count || len(resp.Files) == 0 { + break + } + } + + return nil, fmt.Errorf("recycle bin entry not found for object id=%s name=%s parent=%s", obj.GetID(), obj.GetName(), obj.Pid) +} + +func isRecycleBinEntryNotFoundErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "recycle bin entry not found") +} + +func (d *Open115) findRecycleBinEntryWithRetry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + attempts := recycleBinLookupMaxAttempts + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for i := 0; i < attempts; i++ { + entry, err := d.findRecycleBinEntry(ctx, obj) + if err == nil { + return entry, nil + } + + lastErr = err + if !isRecycleBinEntryNotFoundErr(err) || i == attempts-1 { + break + } + + wait := recycleBinLookupRetryDelay * time.Duration(i+1) + if wait <= 0 { + continue + } + + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + return nil, lastErr +} + +func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *sdk.RbListResp_FileInfo { + if len(files) == 0 { + return nil + } + if entry, ok := files[obj.GetID()]; ok { + matched := entry + return &matched + } + + size := strconv.FormatInt(obj.GetSize(), 10) + for _, entry := range files { + if entry.ID == obj.GetID() { + matched := entry + return &matched + } + cid := string(entry.CID) + if obj.IsDir() { + if entry.FileName == obj.GetName() && cid == obj.Pid { + matched := entry + return &matched + } + continue + } + if obj.Sha1 != "" && entry.SHA1 != "" && strings.EqualFold(entry.SHA1, obj.Sha1) { + if entry.FileName == obj.GetName() || cid == obj.Pid { + matched := entry + return &matched + } + } + if entry.FileName == obj.GetName() && cid == obj.Pid && entry.FileSize == size { + matched := entry + return &matched + } + } return nil } @@ -274,27 +463,97 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } + sha1 := file.GetHash().GetHash(utils.SHA1) - if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + sha1128k := file.GetHash().GetHash(utils.SHA1_128K) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) if err != nil { return err } + if resp.Status == 2 { + up(100) + return nil + } + // 秒传失败，继续后续流程 } - const PreHashSize int64 = 128 * utils.KB - hashSize := PreHashSize - if file.GetSize() < PreHashSize { - hashSize = file.GetSize() - } - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) - if err != nil { - return err - } - sha1128k, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return err + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(sha1) != utils.SHA1.Width { + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + if err != nil { + return err + } + } + // 计算 sha1_128k（如果没有预计算） + if len(sha1128k) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 如果有预计算的 hash，上面已经尝试过秒传了 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + // 秒传失败，需要缓存文件进行实际上传 + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } else { + // 没有预计算的 hash，缓存整个文件并计算 + if len(sha1) != utils.SHA1.Width { + _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + if err != nil { + return err + } + } else if file.GetFile() == nil { + // 有 SHA1 但没有缓存，需要缓存以支持后续 RangeRead + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + // 计算 sha1_128k + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } } - // 1. Init + + // 1. Init（SeekableStream 或已缓存的 FileStream） resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -320,11 +579,11 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } - reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) + signReader, err := file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } - signVal, err := utils.HashReader(utils.SHA1, reader) + signVal, err := utils.HashReader(utils.SHA1, signReader) if err != nil { return err } @@ -362,15 +621,48 @@ func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir mod return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } +func (d *Open115) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + var envelope sdk.Resp[[]sdk.AddOfflineTaskURIsResp] + response, err := d.client.AuthRequestRaw(ctx, sdk.ApiAddOffline, http.MethodPost, nil, sdk.ReqWithForm(sdk.Form{ + "urls": strings.Join(uris, "\n"), + "wp_path_id": dstDir.GetID(), + })) + if response != nil { + _ = json.Unmarshal(response.Bytes(), &envelope) + } + hashes := make([]string, 0, len(envelope.Data)) + for _, item := range envelope.Data { + if item.State && item.InfoHash != "" { + hashes = append(hashes, item.InfoHash) + } + } + rawResponse := "" + if response != nil { + rawResponse = response.String() + } + return hashes, envelope.Data, rawResponse, err +} + func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 获取第一页 resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } + // 如果有多页，获取所有页面的任务 + if resp.PageCount > 1 { + for page := 2; page <= resp.PageCount; page++ { + pageResp, err := d.client.OfflineTaskList(ctx, int64(page)) + if err != nil { + return nil, err + } + resp.Tasks = append(resp.Tasks, pageResp.Tasks...) + } + } return resp, nil } diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go new file mode 100644 index 0000000000..1d96646952 --- /dev/null +++ b/drivers/115_open/driver_test.go @@ -0,0 +1,639 @@ +package _115_open + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + "time" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type recordedRequest struct { + Path string + Form url.Values +} + +type rewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = t.target.Scheme + cloned.URL.Host = t.target.Host + return t.base.RoundTrip(cloned) +} + +func TestOpen115RemoveTrashUsesDelFileOnly(t *testing.T) { + driver, requests := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, []string{"rb-123"}) + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete") + assertFormValue(t, requests()[0].Form, "file_ids", "file-1") + assertFormValue(t, requests()[0].Form, "parent_id", "dir-1") +} + +func TestOpen115RemoveDeleteUsesDelFileResponseIDWhenAvailable(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/del": + writeSDKSuccess(t, w, []string{"rb-123"}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del") + assertFormValue(t, requests()[1].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteFallsBackToRecycleBinLookup(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "file_size": "123", + "cid": "dir-1", + "sha1": "sha-demo", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing(t *testing.T) { + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(context.Background(), obj) + if err == nil { + t.Fatalf("expected Remove to fail when recycle-bin entry is missing") + } + if !strings.Contains(err.Error(), "recycle bin entry not found") { + t.Fatalf("expected recycle-bin lookup error, got: %v", err) + } +} + +func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { + info, ok := op.GetDriverInfoMap()["115 Open"] + if !ok { + t.Fatalf("115 Open driver info was not registered") + } + + for _, item := range info.Additional { + if item.Name != "remove_way" { + continue + } + if item.Type != "select" { + t.Fatalf("unexpected remove_way type: %q", item.Type) + } + if item.Options != "trash,delete" { + t.Fatalf("unexpected remove_way options: %q", item.Options) + } + if item.Default != "trash" { + t.Fatalf("unexpected remove_way default: %q", item.Default) + } + if !item.Required { + t.Fatalf("expected remove_way to be required") + } + return + } + + t.Fatalf("remove_way item not found in 115 Open driver info") +} + +func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + + var ( + mu sync.Mutex + requests []recordedRequest + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + }) + mu.Unlock() + responder(w, r) + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + client := sdk.New(sdk.WithAccessToken("test-token")) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + + return &Open115{ + Addition: Addition{ + RemoveWay: removeWay, + PageSize: 1, + }, + client: client, + }, func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + return append([]recordedRequest(nil), requests...) + } +} + +func writeSDKSuccess(t *testing.T, w http.ResponseWriter, data any) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": true, + "data": data, + }) +} + +func writeSDKError(t *testing.T, w http.ResponseWriter, code int64, message string) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": false, + "code": code, + "message": message, + }) +} + +func writeSDKResponse(t *testing.T, w http.ResponseWriter, payload map[string]any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(payload); err != nil { + t.Fatalf("Encode response failed: %v", err) + } +} + +func assertRequestPaths(t *testing.T, requests []recordedRequest, want ...string) { + t.Helper() + got := make([]string, 0, len(requests)) + for _, req := range requests { + got = append(got, req.Path) + } + if !slices.Equal(got, want) { + t.Fatalf("unexpected request paths: got %v want %v", got, want) + } +} + +func assertFormValue(t *testing.T, form url.Values, key, want string) { + t.Helper() + if got := form.Get(key); got != want { + t.Fatalf("unexpected form value for %s: got %q want %q", key, got, want) + } +} + +func cloneValues(src url.Values) url.Values { + dst := make(url.Values, len(src)) + for key, values := range src { + dst[key] = append([]string(nil), values...) + } + return dst +} + +// --- FlexString / numeric CID tests --- + +func TestFlexStringUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"string value", `{"cid":"dir-1"}`, "dir-1"}, + {"integer value", `{"cid":3383942108160578280}`, "3383942108160578280"}, + {"large integer", `{"cid":9999999999999999999}`, "9999999999999999999"}, + {"zero", `{"cid":0}`, "0"}, + {"negative", `{"cid":-123}`, "-123"}, + {"float", `{"cid":1.5}`, "1.5"}, + {"empty string", `{"cid":""}`, ""}, + {"null", `{"cid":null}`, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if got := string(v.CID); got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestFlexStringUnmarshalInvalid(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"boolean", `{"cid":true}`}, + {"array", `{"cid":[1]}`}, + {"object", `{"cid":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err == nil { + t.Fatalf("expected error for input %s", tt.input) + } + }) + } +} + +func TestRbListRespUnmarshalCIDAsString(t *testing.T) { + raw := `{"id":"rb-1","file_name":"demo.txt","cid":"dir-1","file_size":"123"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "dir-1" { + t.Fatalf("got CID %q, want %q", string(info.CID), "dir-1") + } +} + +func TestRbListRespUnmarshalCIDAsNumber(t *testing.T) { + raw := `{"id":"rb-1","file_name":"MyFolder","cid":3383942108160578280,"file_size":"0"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "3383942108160578280" { + t.Fatalf("got CID %q, want %q", string(info.CID), "3383942108160578280") + } +} + +// --- matchRecycleBinEntry with numeric CID --- + +func TestMatchRecycleBinEntryDirMatchWithNumericCID(t *testing.T) { + // obj represents a directory with Pid (parent ID) as a large number + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("3383942108160578280"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match for directory with numeric CID, got nil") + } + if result.ID != "rb-folder-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-folder-1") + } +} + +func TestMatchRecycleBinEntryDirNoMatchWhenCIDWrong(t *testing.T) { + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("9999999999"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result != nil { + t.Fatalf("expected no match, got %+v", result) + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchWithNumericCID(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+CID, got nil") + } + if result.ID != "rb-file-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-file-1") + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchByNameOnly(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "wrong-pid", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+name, got nil") + } +} + +func TestMatchRecycleBinEntryFileNameSizeCIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "doc.pdf", Fc: "1", FS: 500} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "doc.pdf", + CID: sdk.FlexString("3383942108160578280"), + FileSize: "500", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via name+size+CID, got nil") + } +} + +func TestMatchRecycleBinEntryDirectIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + files := map[string]sdk.RbListResp_FileInfo{ + "file-1": { + ID: "rb-123", + FileName: "demo.txt", + CID: sdk.FlexString("dir-1"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected direct ID match, got nil") + } + if result.ID != "rb-123" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-123") + } +} + +func TestMatchRecycleBinEntryEmptyFiles(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + result := matchRecycleBinEntry(obj, nil) + if result != nil { + t.Fatalf("expected nil for nil files, got %+v", result) + } + result = matchRecycleBinEntry(obj, map[string]sdk.RbListResp_FileInfo{}) + if result != nil { + t.Fatalf("expected nil for empty files, got %+v", result) + } +} + +// --- Full Remove flow with numeric CID in recycle bin --- + +func TestOpen115RemoveDeleteWithNumericCID(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"folder-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "folder-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-folder-1"}) + case "/open/rb/list": + // CID returned as number (the real bug scenario) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-folder-1": map[string]any{ + "id": "rb-folder-1", + "file_name": "MyFolder", + "cid": 3383942108160578280, + "file_size": "0", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-folder-1") +} + +func TestOpen115RemoveDeleteWithStringCIDStillWorks(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + // CID returned as string (normal case) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteRetriesRecycleBinLookupUntilVisible(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 3 + recycleBinLookupRetryDelay = time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + rbListCalls := 0 + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + rbListCalls++ + if rbListCalls < 3 { + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + return + } + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + if rbListCalls != 3 { + t.Fatalf("rbListCalls = %d, want 3", rbListCalls) + } + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/list", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[5].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteStopsRetryWhenContextCancelled(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 5 + recycleBinLookupRetryDelay = 50 * time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + writeSDKError(t, w, 404, "not found") + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(ctx, obj) + if err == nil { + t.Fatalf("expected Remove to fail due to context cancellation") + } + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("expected context deadline exceeded, got: %v", err) + } +} diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index ed908e2e62..0479ac1fdb 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -11,6 +11,7 @@ type Addition struct { // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete" default:"trash"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` @@ -18,9 +19,10 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", - LinkCacheMode: driver.LinkCacheUA, + Name: "115 Open", + DefaultRoot: "0", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2c4..6af4403cf1 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "io" + "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" @@ -14,8 +15,19 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" ) +// isTokenExpiredError 检测是否为OSS凭证过期错误 +func isTokenExpiredError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "SecurityTokenExpired") || + strings.Contains(errStr, "InvalidAccessKeyId") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -71,11 +83,16 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err + // 创建OSS客户端的辅助函数 + createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { + ossClient, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return nil, err + } + return ossClient.Bucket(initResp.Bucket) } - bucket, err := ossClient.Bucket(initResp.Bucket) + + bucket, err := createBucket(tokenResp) if err != nil { return err } @@ -120,7 +137,24 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second)) + retry.Delay(time.Second), + retry.OnRetry(func(n uint, err error) { + // 如果是凭证过期错误，在重试前刷新凭证并重建bucket + if isTokenExpiredError(err) { + log.Warnf("115 OSS token expired, refreshing token...") + if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { + tokenResp = newToken + if newBucket, bucketErr := createBucket(tokenResp); bucketErr == nil { + bucket = newBucket + log.Infof("115 OSS token refreshed successfully") + } else { + log.Errorf("Failed to create new bucket with refreshed token: %v", bucketErr) + } + } else { + log.Errorf("Failed to refresh 115 OSS token: %v", refreshErr) + } + } + })) ss.FreeSectionReader(rd) if err != nil { return err diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index 6f4c7fd046..3e002ff811 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -2,8 +2,16 @@ package _115_open import ( "context" + "encoding/base32" + "encoding/hex" + "errors" "fmt" + "net/url" + "strconv" + "strings" + "time" + sdk "github.com/OpenListTeam/115-sdk-go" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -12,11 +20,34 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + log "github.com/sirupsen/logrus" ) type Open115 struct { } +type offlineTaskClient interface { + OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) + DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error +} + +type offlineTaskDetailClient interface { + OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) +} + +type offlineTaskLimiter interface { + WaitLimit(ctx context.Context) error +} + +func waitOfflineTaskLimit(ctx context.Context, client offlineTaskClient) error { + limiter, ok := client.(offlineTaskLimiter) + if !ok { + return nil + } + return limiter.WaitLimit(ctx) +} + func (o *Open115) Name() string { return "115 Open" } @@ -66,15 +97,473 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { if err != nil { return "", err } + log.Infof("[115_open] AddURL start: temp_dir=%q actual_path=%q parent_id=%q parent_name=%q url=%q", args.TempDir, actualPath, parentDir.GetID(), parentDir.GetName(), args.Url) + logOfflineURLDetails("[115_open] AddURL input", args.Url) - hashs, err := driver115Open.OfflineDownload(args.Ctx, []string{args.Url}, parentDir) - if err != nil || len(hashs) < 1 { - return "", fmt.Errorf("failed to add offline download task: %w", err) + hashs, err := addOfflineDownloadTask(args.Ctx, driver115Open, args.Url, parentDir) + if err != nil { + return "", err + } + + if len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: no task hash returned") } return hashs[0], nil } +func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, error) { + parentID, parentName := "", "" + if parentDir != nil { + parentID = parentDir.GetID() + parentName = parentDir.GetName() + } + log.Infof("[115_open] addOfflineDownloadTask: parent_id=%q parent_name=%q url=%q", parentID, parentName, url) + logOfflineURLDetails("[115_open] addOfflineDownloadTask target", url) + if err := preCleanDuplicateOfflineTasks(ctx, client, url); err != nil { + return nil, err + } + hashs, addItems, rawResp, err := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] addOfflineDownloadTask first attempt result: hashes=%v err=%v add_items=%d", hashs, err, len(addItems)) + if err == nil { + return hashs, nil + } + if !isDuplicateOfflineTaskError(err) { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] duplicate offline task detected, trying cleanup before retry") + if rawResp != "" { + log.Infof("[115_open] duplicate add response: %s", rawResp) + } + for _, item := range addItems { + log.Infof("[115_open] duplicate add item: state=%v code=%d info_hash=%q url=%q", item.State, item.Code, item.InfoHash, item.URL) + logOfflineURLDetails("[115_open] duplicate add item url", item.URL) + if item.InfoHash == "" { + log.Infof("[115_open] skipping add-response duplicate item: empty info_hash") + continue + } + if item.URL != "" && !offlineTaskURLMatches(item.URL, url) { + log.Infof("[115_open] skipping add-response duplicate item: url mismatch") + continue + } + log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) + } + log.Infof("[115_open] delete duplicate task from add response success: info_hash=%s", item.InfoHash) + waitForOfflineTaskRemoval(ctx, client, item.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after add-response delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after add-response delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) + log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + log.Infof("[115_open] delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after matched delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after matched delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + log.Warnf("[115_open] duplicate offline task detected but no matching task found in offline list") + return nil, fmt.Errorf("failed to add offline download task: %w", err) +} + +func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) + return nil + } + log.Infof("[115_open] pre-add offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + deleted := 0 + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] pre-add duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] pre-add duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + deleted++ + log.Infof("[115_open] pre-add delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + } + if deleted == 0 { + log.Infof("[115_open] pre-add duplicate scan found no matches") + } + return nil +} + +func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, nil, "", err + } + if detailClient, ok := client.(offlineTaskDetailClient); ok { + return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) + } + hashs, err := client.OfflineDownload(ctx, []string{url}, parentDir) + return hashs, nil, "", err +} + +func isDuplicateOfflineTaskError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + return strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") +} + +func offlineTaskURLMatches(taskURL string, rawURL string) bool { + taskVariants := normalizedOfflineTaskURLVariants(taskURL) + rawVariants := normalizedOfflineTaskURLVariants(rawURL) + for candidate := range taskVariants { + if _, ok := rawVariants[candidate]; ok { + return true + } + } + return false +} + +func offlineTaskMatches(task sdk.OfflineTask, rawURL string) bool { + matched, _ := offlineTaskMatchReason(task, rawURL) + return matched +} + +func offlineTaskMatchReason(task sdk.OfflineTask, rawURL string) (bool, string) { + if offlineTaskURLMatches(task.URL, rawURL) { + return true, "url variants matched" + } + if httpURLMatches(task.URL, rawURL) { + return true, "http url host+path matched" + } + rawMagnet := parseMagnetBTIH(rawURL) + if rawMagnet != "" { + taskHash := normalizeInfoHash(task.InfoHash) + if taskHash != "" && taskHash == rawMagnet { + return true, "task info_hash matched raw magnet" + } + taskURLHash := parseMagnetBTIH(task.URL) + if taskURLHash != "" && taskURLHash == rawMagnet { + return true, "task url magnet hash matched" + } + return false, fmt.Sprintf("task magnet hash mismatch: task_info_hash=%q task_url_hash=%q raw_hash=%q", taskHash, taskURLHash, rawMagnet) + } + taskED2K, rawED2K := parseED2KLink(task.URL), parseED2KLink(rawURL) + if taskED2K != nil && rawED2K != nil { + if taskED2K.Hash == rawED2K.Hash { + if taskED2K.Size == rawED2K.Size { + return true, "task url ed2k hash matched" + } + return true, "task url ed2k hash matched despite size mismatch" + } + return false, fmt.Sprintf("task url ed2k mismatch: task=%s raw=%s", taskED2K.String(), rawED2K.String()) + } + if rawED2K == nil { + return false, "raw url is not ed2k and url variants did not match" + } + if normalizeOfflineTaskURL(task.InfoHash) == rawED2K.Hash { + if task.Size == rawED2K.Size { + return true, "task info_hash matched raw ed2k" + } + return true, "task info_hash matched raw ed2k despite size mismatch" + } + taskName := normalizeOfflineTaskURL(task.Name) + if taskName == normalizeOfflineTaskURL(rawED2K.Name) && task.Size == rawED2K.Size { + return true, "task name and size matched raw ed2k" + } + return false, fmt.Sprintf("task name/hash/size mismatch: task_name=%q raw_name=%q task_info_hash=%q raw_hash=%q task_size=%d raw_size=%d", taskName, normalizeOfflineTaskURL(rawED2K.Name), normalizeOfflineTaskURL(task.InfoHash), rawED2K.Hash, task.Size, rawED2K.Size) +} + +func normalizedOfflineTaskURLVariants(raw string) map[string]struct{} { + variants := map[string]struct{}{} + queue := []string{raw} + for len(queue) > 0 { + current := normalizeOfflineTaskURL(queue[0]) + queue = queue[1:] + if current == "" { + continue + } + if _, ok := variants[current]; ok { + continue + } + variants[current] = struct{}{} + if decoded, err := url.QueryUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + if decoded, err := url.PathUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + } + return variants +} + +func httpURLMatches(taskURL, rawURL string) bool { + taskNormalized := normalizeHTTPURL(taskURL) + rawNormalized := normalizeHTTPURL(rawURL) + if taskNormalized == "" || rawNormalized == "" { + return false + } + return taskNormalized == rawNormalized +} + +func normalizeHTTPURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed == nil { + return "" + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + host := strings.ToLower(parsed.Host) + if host == "" { + return "" + } + path := strings.ToLower(parsed.Path) + path = strings.TrimSuffix(path, "/") + if path == "" { + path = "/" + } + return fmt.Sprintf("%s://%s%s", scheme, host, path) +} + +func normalizeOfflineTaskURL(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimSuffix(normalized, "/") + return strings.ToLower(normalized) +} + +type ed2kLink struct { + Name string + Size int64 + Hash string +} + +func parseED2KLink(raw string) *ed2kLink { + normalized := strings.TrimSpace(raw) + if !strings.HasPrefix(strings.ToLower(normalized), "ed2k://|file|") { + return nil + } + parts := strings.Split(normalized, "|") + if len(parts) < 6 { + return nil + } + name, err := url.PathUnescape(parts[2]) + if err != nil { + name = parts[2] + } + size, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil + } + return &ed2kLink{ + Name: normalizeOfflineTaskURL(name), + Size: size, + Hash: normalizeOfflineTaskURL(parts[4]), + } +} + +func parseMagnetBTIH(raw string) string { + if raw == "" { + return "" + } + lower := strings.ToLower(raw) + idx := strings.Index(lower, "btih:") + if idx == -1 { + return "" + } + candidate := raw[idx+len("btih:"):] + if candidate == "" { + return "" + } + for i, ch := range candidate { + if ch == '&' || ch == '#' || ch == '/' { + candidate = candidate[:i] + break + } + } + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + if decoded, err := url.QueryUnescape(candidate); err == nil { + candidate = decoded + } + return normalizeInfoHash(candidate) +} + +func normalizeInfoHash(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimPrefix(strings.ToLower(normalized), "urn:btih:") + if normalized == "" { + return "" + } + if len(normalized) == 40 && isHexString(normalized) { + return normalized + } + if len(normalized) == 32 && isBase32String(normalized) { + decoded, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(normalized)) + if err == nil && len(decoded) == 20 { + return hex.EncodeToString(decoded) + } + } + return normalized +} + +func isHexString(value string) bool { + for _, ch := range value { + if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') { + continue + } + return false + } + return true +} + +func isBase32String(value string) bool { + for _, ch := range value { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '2' && ch <= '7') { + continue + } + return false + } + return true +} + +func (e *ed2kLink) Equal(other *ed2kLink) bool { + if e == nil || other == nil { + return false + } + return e.Name == other.Name && e.Size == other.Size && e.Hash == other.Hash +} + +func (e *ed2kLink) String() string { + if e == nil { + return "" + } + return fmt.Sprintf("name=%q size=%d hash=%q", e.Name, e.Size, e.Hash) +} + +func logOfflineURLDetails(prefix string, raw string) { + if raw == "" { + log.Infof("%s details: raw is empty", prefix) + return + } + variants := normalizedOfflineTaskURLVariants(raw) + log.Infof("%s details: raw=%q normalized_variants=%v", prefix, raw, mapKeys(variants)) + if parsedMagnet := parseMagnetBTIH(raw); parsedMagnet != "" { + log.Infof("%s details: parsed_magnet_hash=%s", prefix, parsedMagnet) + } + if parsed := parseED2KLink(raw); parsed != nil { + log.Infof("%s details: parsed_ed2k=%s", prefix, parsed.String()) + } +} + +func mapKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} + +func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { + const maxChecks = 3 + for attempt := 1; attempt <= maxChecks; attempt++ { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + log.Warnf("[115_open] post-delete wait limit failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + taskList, err := client.OfflineList(ctx) + if err != nil { + log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + stillExists := false + taskStatus := -999 + taskName := "" + for _, task := range taskList.Tasks { + if normalizeOfflineTaskURL(task.InfoHash) != normalizeOfflineTaskURL(infoHash) { + continue + } + stillExists = true + taskStatus = task.Status + taskName = task.Name + break + } + log.Infof("[115_open] post-delete check: info_hash=%s attempt=%d exists=%v status=%d name=%q task_count=%d", infoHash, attempt, stillExists, taskStatus, taskName, len(taskList.Tasks)) + if !stillExists { + return + } + if attempt < maxChecks { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } +} + func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { @@ -122,13 +611,15 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = errors.New(t.GetStatus()) } return s, nil } } - s.Err = fmt.Errorf("the task has been deleted") - return nil, nil + // 任务不在列表中，可能已完成或被删除 + s.Progress = 100 + s.Completed = true + return s, nil } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go new file mode 100644 index 0000000000..2e8b947786 --- /dev/null +++ b/internal/offline_download/115_open/client_test.go @@ -0,0 +1,616 @@ +package _115_open + +import ( + "context" + "fmt" + "strings" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type mockOfflineTaskClient struct { + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error + waitLimitFunc func(ctx context.Context) error + waitLimitCalls int +} + +func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return m.offlineDownloadFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if m.offlineDownloadWithDetailsFunc == nil { + hashes, err := m.OfflineDownload(ctx, uris, dstDir) + return hashes, nil, "", err + } + return m.offlineDownloadWithDetailsFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return m.offlineListFunc(ctx) +} + +func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +} + +func (m *mockOfflineTaskClient) WaitLimit(ctx context.Context) error { + m.waitLimitCalls++ + if m.waitLimitFunc != nil { + return m.waitLimitFunc(ctx) + } + return nil +} + +func TestIsDuplicateOfflineTaskError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "code 10008", err: fmt.Errorf("code: 10008"), want: true}, + {name: "chinese duplicate", err: fmt.Errorf("任务重复"), want: true}, + {name: "already exists", err: fmt.Errorf("任务已存在"), want: true}, + {name: "english duplicate", err: fmt.Errorf("duplicate task"), want: true}, + {name: "other", err: fmt.Errorf("network timeout"), want: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isDuplicateOfflineTaskError(tc.err); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskURLMatches(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + taskURL string + rawURL string + want bool + }{ + { + name: "exact match", + taskURL: "ed2k://|file|test.avi|123|ABC|/", + rawURL: "ed2k://|file|test.avi|123|ABC|/", + want: true, + }, + { + name: "percent encoded file name", + taskURL: "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + rawURL: "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + want: true, + }, + { + name: "case and trailing slash normalized", + taskURL: "ED2K://|FILE|TEST.AVI|123|ABC|", + rawURL: "ed2k://|file|test.avi|123|abc|/", + want: true, + }, + { + name: "different link", + taskURL: "ed2k://|file|a.avi|123|ABC|/", + rawURL: "ed2k://|file|b.avi|123|ABC|/", + want: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := offlineTaskURLMatches(tc.taskURL, tc.rawURL); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskMatches(t *testing.T) { + t.Parallel() + + t.Run("match ed2k by parsed fields when task url differs", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + InfoHash: "server-task-hash", + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected task to match by ed2k parsed fields") + } + }) + + t.Run("do not match different ed2k size", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1, + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if offlineTaskMatches(task, rawURL) { + t.Fatal("expected task not to match") + } + }) + + t.Run("magnet still matches by url", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: rawURL, + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by url") + } + }) + + t.Run("match magnet by btih despite noisy tracker", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test&tr=%3C!DOCTYPE%20html%3E", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by btih") + } + }) + + t.Run("match http by host and path", func(t *testing.T) { + t.Parallel() + + rawURL := "https://example.com/files/test.mp4" + task := sdk.OfflineTask{ + URL: "https://EXAMPLE.com/files/test.mp4?token=abc", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected http task to match by host and path") + } + }) +} + +func TestAddOfflineDownloadTask(t *testing.T) { + t.Parallel() + + const ( + testURL = "https://example.com/test.torrent" + firstHash = "hash-1" + staleHash = "hash-stale" + deleteError = "delete failed" + ) + + t.Run("success on first try", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("wait limit applied for add flow", func(t *testing.T) { + t.Parallel() + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.waitLimitCalls < 2 { + t.Fatalf("want wait limit calls >= 2, got %d", client.waitLimitCalls) + } + }) + + t.Run("delete duplicate and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate magnet and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + magnetURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: magnetURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, magnetURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with decoded ed2k url", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + decodedURL := "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: decodedURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with empty task url but matching name and size", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: staleHash, + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate directly from add response info hash", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadWithDetailsFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + callCount++ + if callCount == 1 { + return nil, []sdk.AddOfflineTaskURIsResp{ + {InfoHash: staleHash, URL: testURL}, + }, `{"state":false,"code":10008,"message":"任务已存在","data":[{"info_hash":"hash-stale","url":"` + testURL + `"}]}`, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil, "", nil + }, + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("unexpected fallback OfflineDownload call") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 1 { + t.Fatalf("want 1 delete attempt, got %d", deleteCount) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("duplicate delete failure", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("duplicate task") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return fmt.Errorf(deleteError) + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), deleteError) { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("non duplicate error is returned", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("network timeout") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "network timeout") { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) +} + +func TestOpen115BasicMethods(t *testing.T) { + t.Parallel() + + o := &Open115{} + + if o.Name() != "115 Open" { + t.Fatalf("unexpected name: %s", o.Name()) + } + if o.Items() != nil { + t.Fatal("Items should return nil") + } + msg, err := o.Init() + if err != nil { + t.Fatalf("unexpected init error: %v", err) + } + if msg != "ok" { + t.Fatalf("unexpected init message: %s", msg) + } +} From 1c8d52fa4ba1d3299b1ee3596bfa9f5735c89ec9 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:37 +0800 Subject: [PATCH 050/107] feat(offline_download): multi-page task retrieval and task limit wait mechanism - Optimize task list retrieval with multi-page support and status hints - Add task limit wait mechanism to optimize offline download task processing --- internal/offline_download/tool/download.go | 22 ++--- .../offline_download/tool/download_test.go | 90 +++++++++++++++++++ 2 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 internal/offline_download/tool/download_test.go diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index b7f7a1a9df..bc5fe044a1 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -32,6 +32,8 @@ type DownloadTask struct { callStatusRetried int } +var completedOfflineTaskCleanupDelay = time.Second + func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) @@ -98,16 +100,7 @@ outer: if t.tool.Name() == "ThunderX" { return nil } - if t.tool.Name() == "115 Cloud" { - // hack for 115 - <-time.After(time.Second * 1) - err := t.tool.Remove(t) - if err != nil { - log.Errorln(err.Error()) - } - return nil - } - if t.tool.Name() == "115 Open" { + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { return nil } if t.tool.Name() == "123 Open" { @@ -148,7 +141,7 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - if t.callStatusRetried > 5 { + if t.callStatusRetried > 10 { return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) } return false, nil @@ -164,6 +157,13 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { + // For 115, remove offline task record before transfer so it gets cleaned up even if transfer fails + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { + <-time.After(completedOfflineTaskCleanupDelay) + if removeErr := t.tool.Remove(t); removeErr != nil { + log.Errorln(removeErr.Error()) + } + } err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } diff --git a/internal/offline_download/tool/download_test.go b/internal/offline_download/tool/download_test.go new file mode 100644 index 0000000000..5303d35aac --- /dev/null +++ b/internal/offline_download/tool/download_test.go @@ -0,0 +1,90 @@ +package tool + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + task2 "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +type mockTool struct { + name string + addURLFunc func(args *AddUrlArgs) (string, error) + removeFunc func(task *DownloadTask) error + statusFunc func(task *DownloadTask) (*Status, error) + runFunc func(task *DownloadTask) error +} + +func (m *mockTool) Name() string { return m.name } + +func (m *mockTool) Items() []model.SettingItem { return nil } + +func (m *mockTool) Init() (string, error) { return "ok", nil } + +func (m *mockTool) IsReady() bool { return true } + +func (m *mockTool) AddURL(args *AddUrlArgs) (string, error) { + return m.addURLFunc(args) +} + +func (m *mockTool) Remove(task *DownloadTask) error { + return m.removeFunc(task) +} + +func (m *mockTool) Status(task *DownloadTask) (*Status, error) { + return m.statusFunc(task) +} + +func (m *mockTool) Run(task *DownloadTask) error { + return m.runFunc(task) +} + +func TestDownloadTaskRun_RemovesCompleted115OpenRecord(t *testing.T) { + previousDelay := completedOfflineTaskCleanupDelay + completedOfflineTaskCleanupDelay = 0 + defer func() { + completedOfflineTaskCleanupDelay = previousDelay + }() + + removeCount := 0 + tool := &mockTool{ + name: "115 Open", + addURLFunc: func(args *AddUrlArgs) (string, error) { + return "gid-1", nil + }, + removeFunc: func(task *DownloadTask) error { + removeCount++ + if task.GID != "gid-1" { + t.Fatalf("unexpected gid: %s", task.GID) + } + return nil + }, + statusFunc: func(task *DownloadTask) (*Status, error) { + return &Status{ + Completed: true, + Status: "completed", + }, nil + }, + runFunc: func(task *DownloadTask) error { + return errs.NotSupport + }, + } + + task := &DownloadTask{ + TaskExtension: task2.TaskExtension{}, + Url: "https://example.com/test.torrent", + DstDirPath: "/115", + TempDir: "/115", + tool: tool, + } + task.SetCtx(context.Background()) + + if err := task.Run(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if removeCount != 1 { + t.Fatalf("want 1 cleanup remove, got %d", removeCount) + } +} From 10e6888154147b8eb453b1ddb15c3f444d133e68 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:44 +0800 Subject: [PATCH 051/107] feat(drivers): baidu streaming upload, quark rate-limit/retry, 123pan etag fix - Implement streaming upload for Baidu Netdisk - Add rate limiting and retry logic for quark_open, optimize upload handling - Fix quark shard size adjustment for oversized errors - Fix file copy failure to 123pan due to incorrect etag - Fix aliyundrive upload hash calculation --- drivers/123_open/driver.go | 48 ++++- drivers/aliyundrive_open/upload.go | 41 ++-- drivers/baidu_netdisk/driver.go | 214 +++----------------- drivers/baidu_netdisk/meta.go | 4 +- drivers/baidu_netdisk/upload.go | 311 +++++++++++++++++++++++++++++ drivers/baidu_netdisk/util.go | 19 +- drivers/quark_open/driver.go | 262 ++++++++++++++++++++---- drivers/quark_open/meta.go | 4 +- drivers/quark_open/util.go | 94 ++++++++- 9 files changed, 738 insertions(+), 259 deletions(-) create mode 100644 drivers/baidu_netdisk/upload.go diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 78ff272b9b..9adb1aed08 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -175,7 +175,7 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { @@ -197,14 +197,49 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(etag) >= utils.MD5.Width { + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return nil, err + } + if createResp.Data.Reuse && createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + // 秒传失败，继续后续流程 + } + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(etag) < utils.MD5.Width { + etag, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return nil, err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 秒传失败或没有 hash，缓存整个文件并计算 MD5 _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } } + + // 2. 创建上传任务（或再次尝试秒传） createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -223,13 +258,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index a4a6c1de1f..5f02c75f57 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -163,21 +163,29 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) + + // 检查是否是可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") - // read 1024 bytes to calculate pre hash - reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) - if err != nil { - return nil, err - } - hash, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return nil, err + // 优先使用预计算的 pre_hash + preHash := stream.GetHash().GetHash(utils.PRE_HASH) + if len(preHash) != utils.PRE_HASH.Width { + // 没有预计算的 pre_hash，使用 RangeRead 计算 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) + if err != nil { + return nil, err + } + preHash, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } } createData["size"] = stream.GetSize() - createData["pre_hash"] = hash + createData["pre_hash"] = preHash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { @@ -191,9 +199,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { - _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 StreamHashFile（RangeRead），不缓存 + hash, err = streamPkg.StreamHashFile(stream, utils.SHA1, 100, &up) + if err != nil { + return nil, err + } + } else { + // 不可重复读取的流，缓存并计算 + _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) + if err != nil { + return nil, err + } } } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index fe77aca38b..474dd2b987 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,30 +1,18 @@ package baidu_netdisk import ( - "bytes" "context" - "crypto/md5" - "encoding/hex" "errors" - "io" - "mime/multipart" - "net/http" "net/url" - "os" stdpath "path" "strconv" - "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" - "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -37,6 +25,7 @@ type BaiduNetdisk struct { } var ErrUploadIDExpired = errors.New("uploadid expired") +var ErrUploadURLExpired = errors.New("upload url expired or unavailable") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -199,80 +188,26 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - var ( - cache = stream.GetFile() - tmpF *os.File - err error - ) - if cache == nil { - tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - defer func() { - _ = tmpF.Close() - _ = os.Remove(tmpF.Name()) - }() - cache = tmpF - } - streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } - lastBlockSize := streamSize % sliceSize - if lastBlockSize == 0 { - lastBlockSize = sliceSize - } - - // cal md5 for first 256k data - const SliceSize int64 = 256 * utils.KB - blockList := make([]string, 0, count) - byteSize := sliceSize - fileMd5H := md5.New() - sliceMd5H := md5.New() - sliceMd5H2 := md5.New() - slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) - writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} - if tmpF != nil { - writers = append(writers, tmpF) - } - written := int64(0) - for i := 1; i <= count; i++ { - if utils.IsCanceled(ctx) { - return nil, ctx.Err() - } - if i == count { - byteSize = lastBlockSize - } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) - written += n - if err != nil && err != io.EOF { - return nil, err - } - blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) - sliceMd5H.Reset() - } - if tmpF != nil { - if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) - } - _, err = tmpF.Seek(0, io.SeekStart) - if err != nil { - return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") - } - } - contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) - sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) - blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 尝试读取已保存进度 + // step.1 流式计算MD5哈希值（使用 RangeRead，不会消耗流） + contentMd5, sliceMd5, blockList, err := d.calculateHashesStream(ctx, stream, sliceSize, &up) + if err != nil { + return nil, err + } + + blockListStr, _ := utils.Json.MarshalToString(blockList) + + // step.2 尝试读取已保存进度或执行预上传 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 @@ -288,6 +223,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { if precreateResp.UploadURL != "" { return @@ -295,58 +231,20 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } - // step.2 上传分片 + // step.3 流式上传分片 + // 创建 StreamSectionReader 用于上传 + ss, err := streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } + uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() - // 并发上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(UPLOAD_RETRY_COUNT), - retry.Delay(UPLOAD_RETRY_WAIT_TIME), - retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), - retry.DelayType(retry.BackOffDelay), - retry.RetryIf(func(err error) bool { - return !errors.Is(err, ErrUploadIDExpired) - }), - retry.LastErrorOnly(true)) - - totalParts := len(precreateResp.BlockList) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break - } - if partseq < 0 { - continue - } - i, partseq := i, partseq - offset, size := int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - size = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), - } - section := io.NewSectionReader(cache, offset, size) - err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) - if err != nil { - return err - } - precreateResp.BlockList[i] = -1 - progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) - up(progress) - return nil - }) - } - err = threadG.Wait() + // 流式并发上传 + err = d.uploadChunksStream(ctx, ss, stream, precreateResp, path, sliceSize, count, up) if err == nil { break uploadLoop } @@ -372,13 +270,19 @@ uploadLoop: precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + // 尝试重新创建 StreamSectionReader（如果流支持重新读取） + ss, err = streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } continue uploadLoop } return nil, err } defer up(100) - // step.3 创建文件 + // step.4 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { @@ -427,68 +331,6 @@ func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize in return &precreateResp, nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { - b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - mw := multipart.NewWriter(b) - _, err := mw.CreateFormFile("file", fileName) - if err != nil { - return err - } - headSize := b.Len() - err = mw.Close() - if err != nil { - return err - } - head := bytes.NewReader(b.Bytes()[:headSize]) - tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) - if err != nil { - return err - } - query := req.URL.Query() - for k, v := range params { - query.Set(k, v) - } - req.URL.RawQuery = query.Encode() - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(b.Len()) + file.Size() - - client := net.NewHttpClient() - if d.UploadSliceTimeout > 0 { - client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) - } else { - client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b.Reset() - _, err = b.ReadFrom(resp.Body) - if err != nil { - return err - } - body := b.Bytes() - respStr := string(body) - log.Debugln(respStr) - lower := strings.ToLower(respStr) - // 合并 uploadid 过期检测逻辑 - if strings.Contains(lower, "uploadid") && - (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { - return ErrUploadIDExpired - } - - errCode := utils.Json.Get(body, "error_code").ToInt() - errNo := utils.Json.Get(body, "errno").ToInt() - if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) - } - return nil -} - func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 3f3bed0229..499fcd8a87 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -31,8 +31,8 @@ type Addition struct { const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) - DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 - UPLOAD_RETRY_COUNT = 3 + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 180 // 上传分片请求默认超时时间（增加到3分钟以应对慢速网络） + UPLOAD_RETRY_COUNT = 5 // 增加重试次数以提高成功率 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go new file mode 100644 index 0000000000..c160c3a9e6 --- /dev/null +++ b/drivers/baidu_netdisk/upload.go @@ -0,0 +1,311 @@ +package baidu_netdisk + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" +) + +// calculateHashesStream 流式计算文件的MD5哈希值 +// 返回：文件MD5、前256KB的MD5、每个分片的MD5列表 +// 注意：此函数使用 RangeRead 读取数据，不会消耗流 +func (d *BaiduNetdisk) calculateHashesStream( + ctx context.Context, + stream model.FileStreamer, + sliceSize int64, + up *driver.UpdateProgress, +) (contentMd5 string, sliceMd5 string, blockList []string, err error) { + streamSize := stream.GetSize() + count := 1 + if streamSize > sliceSize { + count = int((streamSize + sliceSize - 1) / sliceSize) + } + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 前256KB的MD5 + const SliceSize int64 = 256 * utils.KB + blockList = make([]string, 0, count) + fileMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceWritten := int64(0) + + // 使用固定大小的缓冲区进行流式哈希计算 + // 这样可以利用 readFullWithRangeRead 的链接刷新逻辑 + const chunkSize = 10 * 1024 * 1024 // 10MB per chunk + buf := make([]byte, chunkSize) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return "", "", nil, ctx.Err() + } + + offset := int64(i) * sliceSize + length := sliceSize + if i == count-1 { + length = lastBlockSize + } + + // 计算分片MD5 + sliceMd5Calc := md5.New() + + // 分块读取并计算哈希 + var sliceOffset int64 = 0 + for sliceOffset < length { + readSize := chunkSize + if length-sliceOffset < int64(chunkSize) { + readSize = int(length - sliceOffset) + } + + // 使用 readFullWithRangeRead 读取数据，自动处理链接刷新 + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset+sliceOffset) + if err != nil { + return "", "", nil, err + } + + // 同时写入多个哈希计算器 + fileMd5H.Write(buf[:n]) + sliceMd5Calc.Write(buf[:n]) + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + if int64(n) > remaining { + sliceMd5H2.Write(buf[:remaining]) + sliceWritten += remaining + } else { + sliceMd5H2.Write(buf[:n]) + sliceWritten += int64(n) + } + } + + sliceOffset += int64(n) + } + + blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) + + // 更新进度（哈希计算占总进度的一小部分） + if up != nil { + progress := float64(i+1) * 10 / float64(count) + (*up)(progress) + } + } + + return hex.EncodeToString(fileMd5H.Sum(nil)), + hex.EncodeToString(sliceMd5H2.Sum(nil)), + blockList, nil +} + +// uploadChunksStream 流式上传所有分片 +func (d *BaiduNetdisk) uploadChunksStream( + ctx context.Context, + ss streamPkg.StreamSectionReaderIF, + stream model.FileStreamer, + precreateResp *PrecreateResp, + path string, + sliceSize int64, + count int, + up driver.UpdateProgress, +) error { + streamSize := stream.GetSize() + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 使用 OrderedGroup 保证 Before 阶段有序 + thread := min(d.uploadThread, len(precreateResp.BlockList)) + threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.OnRetry(func(n uint, err error) { + // 重试前检测是否需要刷新上传 URL + if errors.Is(err, ErrUploadURLExpired) { + log.Infof("[baidu_netdisk] refreshing upload URL due to error: %v", err) + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + + i, partseq := i, partseq + offset := int64(partseq) * sliceSize + size := sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + var reader io.ReadSeeker + + threadG.GoWithLifecycle(errgroup.Lifecycle{ + Before: func(ctx context.Context) error { + var err error + reader, err = ss.GetSectionReader(offset, size) + return err + }, + Do: func(ctx context.Context) error { + reader.Seek(0, io.SeekStart) + err := d.uploadSliceStream(ctx, precreateResp.UploadURL, path, + precreateResp.Uploadid, partseq, stream.GetName(), reader, size) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度从10%开始（前10%是哈希计算） + progress := 10 + float64(threadG.Success()+1)*90/float64(totalParts+1) + up(progress) + return nil + }, + After: func(err error) { + ss.FreeSectionReader(reader) + }, + }) + } + + return threadG.Wait() +} + +// uploadSliceStream 上传单个分片（接受io.ReadSeeker） +func (d *BaiduNetdisk) uploadSliceStream( + ctx context.Context, + uploadUrl string, + path string, + uploadid string, + partseq int, + fileName string, + reader io.ReadSeeker, + size int64, +) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": uploadid, + "partseq": strconv.Itoa(partseq), + } + + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() + if err != nil { + return err + } + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + size + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + // 检测超时或网络错误，标记需要刷新上传 URL + if isUploadURLError(err) { + log.Warnf("[baidu_netdisk] upload slice failed with network error: %v, will refresh upload URL", err) + return errors.Join(err, ErrUploadURLExpired) + } + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + if errCode != 0 || errNo != 0 { + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) + } + return nil +} + +// isUploadURLError 判断是否为需要刷新上传 URL 的错误 +// 包括：超时、连接被拒绝、连接重置、DNS 解析失败等网络错误 +func isUploadURLError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + // 超时错误 + if strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "deadline exceeded") { + return true + } + // 连接错误 + if strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "connection reset") || + strings.Contains(errStr, "no such host") || + strings.Contains(errStr, "network is unreachable") { + return true + } + // EOF 错误（连接被服务器关闭） + if strings.Contains(errStr, "eof") || + strings.Contains(errStr, "broken pipe") { + return true + } + return false +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 7916f9b508..a66204fa5f 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -207,7 +207,24 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) - res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) + + // Retry HEAD request with longer timeout to avoid client-side errors + // Create a client with longer timeout (base.NoRedirectClient doesn't have timeout set) + client := base.NoRedirectClient.SetTimeout(60 * time.Second) + var res *resty.Response + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + res, err = client.R(). + SetHeader("User-Agent", "pan.baidu.com"). + Head(u) + if err == nil { + break + } + if i < maxRetries-1 { + log.Warnf("HEAD request failed (attempt %d/%d): %v, retrying...", i+1, maxRetries, err) + time.Sleep(time.Duration(i+1) * 2 * time.Second) // Exponential backoff: 2s, 4s, 6s, 8s + } + } if err != nil { return nil, err } diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index f0b8baf097..26a4288ccf 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -8,6 +8,7 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -18,15 +19,23 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type QuarkOpen struct { model.Storage Addition - config driver.Config - conf Conf + config driver.Config + conf Conf + limiter *rate.Limiter } +// 速率限制常量：夸克开放平台限流，保守设置 +const ( + quarkRateLimit = 2.0 // 每秒2个请求，避免限流 +) + func (d *QuarkOpen) Config() driver.Config { return d.config } @@ -36,6 +45,9 @@ func (d *QuarkOpen) GetAddition() driver.Additional { } func (d *QuarkOpen) Init(ctx context.Context) error { + // 初始化速率限制器 + d.limiter = rate.NewLimiter(rate.Limit(quarkRateLimit), 1) + var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) @@ -52,11 +64,22 @@ func (d *QuarkOpen) Init(ctx context.Context) error { return err } +// waitLimit 等待速率限制 +func (d *QuarkOpen) waitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -67,6 +90,9 @@ func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } data := base.Json{ "fid": file.GetID(), } @@ -143,35 +169,116 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - var ( - md5 hash.Hash - sha1 hash.Hash - ) - writers := []io.Writer{} - if len(md5Str) != utils.MD5.Width { - md5 = utils.MD5.NewFunc() - writers = append(writers, md5) - } - if len(sha1Str) != utils.SHA1.Width { - sha1 = utils.SHA1.NewFunc() - writers = append(writers, sha1) + if err := d.waitLimit(ctx); err != nil { + return err } + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - if len(writers) > 0 { - _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) - if err != nil { - return err - } - if md5 != nil { - md5Str = hex.EncodeToString(md5.Sum(nil)) - } - if sha1 != nil { - sha1Str = hex.EncodeToString(sha1.Sum(nil)) + // 检查是否需要计算hash + needMD5 := len(md5Str) != utils.MD5.Width + needSHA1 := len(sha1Str) != utils.SHA1.Width + + if needMD5 || needSHA1 { + // 检查是否为可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 一次性计算所有hash，避免重复读取 + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + // 使用 RangeRead 分块读取文件，同时计算多个hash + multiWriter := io.MultiWriter(writers...) + size := stream.GetSize() + chunkSize := int64(10 * utils.MB) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := min(chunkSize, size-offset) + + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset) + if err != nil { + return fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + multiWriter.Write(buf[:n]) + offset += int64(n) + + // 更新进度（hash计算占用40%的进度） + up(40 * float64(offset) / float64(size)) + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } + } else { + // 不可重复读取的流（如网络流），需要缓存并计算hash + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) + if err != nil { + return err + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } } - // pre - pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + // pre - 带有 proof fail 重试逻辑 + var pre UpPreResp + var err error + err = retry.Do(func() error { + var preErr error + pre, preErr = d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + if preErr != nil { + // 检查是否为 proof fail 错误 + if strings.Contains(preErr.Error(), "proof") || strings.Contains(preErr.Error(), "43010") { + log.Warnf("[quark_open] Proof verification failed, retrying: %v", preErr) + return preErr // 返回错误触发重试 + } + // 检查是否为限流错误 + if strings.Contains(preErr.Error(), "限流") || strings.Contains(preErr.Error(), "rate") { + log.Warnf("[quark_open] Rate limited, waiting before retry: %v", preErr) + time.Sleep(2 * time.Second) // 额外等待 + return preErr + } + } + return preErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) if err != nil { return err } @@ -181,16 +288,70 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } - // get part info - partInfo := d._getPartInfo(stream, pre.Data.PartSize) - // get upload url info - upUrlInfo, err := d.upUrl(ctx, pre, partInfo) - if err != nil { + // 空文件特殊处理：跳过分片上传，直接调用 upFinish + // 由于夸克 API 对空文件处理不稳定，尝试完成上传，失败则直接成功返回 + if stream.GetSize() == 0 { + log.Infof("[quark_open] Empty file detected, attempting direct finish (task_id: %s)", pre.Data.TaskID) + err = d.upFinish(ctx, pre, []base.Json{}, []string{}) + if err != nil { + // 空文件 upFinish 失败，可能是 API 不支持，直接视为成功 + log.Warnf("[quark_open] Empty file upFinish failed: %v, treating as success", err) + } + up(100) + return nil + } + + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 + var upUrlInfo UpUrlInfo + var partInfo []base.Json + currentPartSize := pre.Data.PartSize + const maxRetries = 5 + const maxPartSize = 1024 * utils.MB // 1GB 上限 + + for attempt := 0; attempt < maxRetries; attempt++ { + // 计算分片信息 + partInfo = d._getPartInfo(stream, currentPartSize) + + // 尝试获取上传 URL + upUrlInfo, err = d.upUrl(ctx, pre, partInfo) + if err == nil { + // 成功获取上传 URL + log.Infof("[quark_open] Successfully obtained upload URLs with part size: %d MB (%d parts)", + currentPartSize/(1024*1024), len(partInfo)) + break + } + + // 检查是否为分片超限错误 + if strings.Contains(err.Error(), "exceed") { + if attempt < maxRetries-1 { + // 还有重试机会，翻倍分片大小 + newPartSize := currentPartSize * 2 + + // 检查是否超过上限 + if newPartSize > maxPartSize { + return fmt.Errorf("part list exceeded and cannot increase part size (current: %d MB, max: %d MB). File may be too large for Quark API", + currentPartSize/(1024*1024), maxPartSize/(1024*1024)) + } + + log.Warnf("[quark_open] Part list exceeded (attempt %d/%d, %d parts). Retrying with doubled part size: %d MB -> %d MB", + attempt+1, maxRetries, len(partInfo), + currentPartSize/(1024*1024), newPartSize/(1024*1024)) + + currentPartSize = newPartSize + continue // 重试 + } else { + // 已达到最大重试次数 + return fmt.Errorf("part list exceeded after %d retries. Last attempt: part size %d MB, %d parts", + maxRetries, currentPartSize/(1024*1024), len(partInfo)) + } + } + + // 其他错误，直接返回 return err } - // part up - ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + // part up - 使用调整后的 currentPartSize + ss, err := streamPkg.NewStreamSectionReader(stream, int(currentPartSize), &up) if err != nil { return err } @@ -204,30 +365,49 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } - offset := int64(i) * pre.Data.PartSize - size := min(pre.Data.PartSize, total-offset) + offset := int64(i) * currentPartSize + size := min(currentPartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } + + // 上传重试逻辑，包含URL刷新 + var etag string err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) - if err != nil { - return err + var uploadErr error + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + + // 检查是否为URL过期错误 + if uploadErr != nil && strings.Contains(uploadErr.Error(), "expire") { + log.Warnf("[quark_open] Upload URL expired for part %d, refreshing...", i) + // 刷新上传URL + newUpUrlInfo, refreshErr := d.upUrl(ctx, pre, partInfo) + if refreshErr != nil { + return fmt.Errorf("failed to refresh upload url: %w", refreshErr) + } + upUrlInfo = newUpUrlInfo + log.Infof("[quark_open] Upload URL refreshed successfully") + + // 使用新URL重试上传 + rd.Seek(0, io.SeekStart) + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) } - etags = append(etags, etag) - return nil + + return uploadErr }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } + etags = append(etags, etag) up(95 * float64(offset+size) / float64(total)) } diff --git a/drivers/quark_open/meta.go b/drivers/quark_open/meta.go index 3527b52e9c..ee1903939d 100644 --- a/drivers/quark_open/meta.go +++ b/drivers/quark_open/meta.go @@ -13,8 +13,8 @@ type Addition struct { APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` - AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` - SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` + AppID string `json:"app_id" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` + SignKey string `json:"sign_key" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` } type Conf struct { diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 788ca0e99a..1a3058375e 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -283,8 +284,15 @@ func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 - partInfo := make([]base.Json, 0) total := stream.GetSize() + + // 确保partSize合理：最小4MB，避免分片过多 + const minPartSize int64 = 4 * utils.MB + if partSize < minPartSize { + partSize = minPartSize + } + + partInfo := make([]base.Json, 0) left := total partNumber := 1 @@ -304,6 +312,7 @@ func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []ba partNumber++ } + log.Infof("[quark_open] Upload plan: file_size=%d, part_size=%d, part_count=%d", total, partSize, len(partInfo)) return partInfo } @@ -315,11 +324,17 @@ func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Js } var resp UpUrlResp + log.Infof("[quark_open] Requesting upload URLs for %d parts (task_id: %s)", len(partInfo), pre.Data.TaskID) + _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { + // 如果是分片超限错误，记录详细信息 + if strings.Contains(err.Error(), "part list exceed") { + log.Errorf("[quark_open] Part list exceeded limit! Requested %d parts. Please check Quark API documentation for actual limit.", len(partInfo)) + } return upUrlInfo, err } @@ -340,13 +355,43 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") + // ✅ 关键修复：使用更长的超时时间（10分钟） + // 慢速网络下大文件分片上传可能需要很长时间 + client := &http.Client{ + Timeout: 10 * time.Minute, + Transport: base.HttpClient.Transport, + } + // 发送请求 - resp, err := base.HttpClient.Do(req) + resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() + // 检查是否为URL过期错误（403, 410等状态码） + if resp.StatusCode == 403 || resp.StatusCode == 410 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("upload url expired (status: %d): %s", resp.StatusCode, string(body)) + } + + // ✅ 关键修复：409 PartAlreadyExist 不是错误！ + // 夸克使用Sequential模式，超时重试时如果分片已存在，说明第一次其实成功了 + if resp.StatusCode == 409 { + body, _ := io.ReadAll(resp.Body) + // 从响应体中提取已存在分片的ETag + if strings.Contains(string(body), "PartAlreadyExist") { + // 尝试从XML响应中提取ETag + if etag := extractEtagFromXML(string(body)); etag != "" { + log.Infof("[quark_open] Part %d already exists (409), using existing ETag: %s", partNumber+1, etag) + return etag, nil + } + // 如果无法提取ETag，返回错误 + log.Warnf("[quark_open] Part %d already exists but cannot extract ETag from response: %s", partNumber+1, string(body)) + return "", fmt.Errorf("part already exists but ETag not found in response") + } + } + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) @@ -355,6 +400,23 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber return resp.Header.Get("Etag"), nil } +// extractEtagFromXML 从OSS的XML错误响应中提取ETag +// 示例: "2F796AC486BB2891E3237D8BFDE020B5" +func extractEtagFromXML(xmlBody string) string { + start := strings.Index(xmlBody, "") + if start == -1 { + return "" + } + start += len("") + end := strings.Index(xmlBody[start:], "") + if end == -1 { + return "" + } + etag := xmlBody[start : start+end] + // 移除引号 + return strings.Trim(etag, "\"") +} + func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) @@ -417,25 +479,36 @@ func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey stri } func (d *QuarkOpen) refreshToken() error { - refresh, access, err := d._refreshToken() + refresh, access, appID, signKey, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, appID, signKey, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access + + // 如果在线API返回了AppID和SignKey，保存它们（不为空时才更新） + if appID != "" && appID != d.AppID { + d.AppID = appID + log.Infof("[quark_open] AppID updated from online API: %s", appID) + } + if signKey != "" && signKey != d.SignKey { + d.SignKey = signKey + log.Infof("[quark_open] SignKey updated from online API") + } + op.MustSaveDriverStorage(d) return nil } -func (d *QuarkOpen) _refreshToken() (string, string, error) { +func (d *QuarkOpen) _refreshToken() (string, string, string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp @@ -448,19 +521,20 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { }). Get(u) if err != nil { - return "", "", err + return "", "", "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { - return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + return "", "", "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } - return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") + return "", "", "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } - return resp.RefreshToken, resp.AccessToken, nil + // 返回所有字段，包括AppID和SignKey + return resp.RefreshToken, resp.AccessToken, resp.AppID, resp.SignKey, nil } // TODO 本地刷新逻辑 - return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") + return "", "", "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie From be3a2b338178bf2a066bfd35a0309f1a9bf10cfb Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:52 +0800 Subject: [PATCH 052/107] fix(core): copy_move depth, alias storage retrieval, sftp symlink, 500 panic - Fix pre-create subdirectory depth from 2 to 1 to avoid deep recursion - Fix srcBasePath bug and remove redundant sleep in preCreateDirectoryTree - Add unit tests for preCreateDirTreeFn - Update storage retrieval method in alias listRoot function - Fix sftp symlink path resolution - Fix 500 panic and NaN issues in openlist driver --- drivers/alias/util.go | 2 +- drivers/openlist/driver.go | 106 +++++++++++- drivers/sftp/types.go | 4 +- internal/fs/copy_move.go | 97 +++++++++++ internal/fs/copy_move_test.go | 317 ++++++++++++++++++++++++++++++++++ 5 files changed, 517 insertions(+), 9 deletions(-) create mode 100644 internal/fs/copy_move_test.go diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8e5eb8a843..b378543940 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -40,7 +40,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if !withDetails || len(v) != 1 { continue } - remoteDriver, err := op.GetStorageByMountPath(v[0]) + remoteDriver, err := fs.GetStorage(v[0], &fs.GetStoragesArgs{}) if err != nil { continue } diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 79fc511854..b37d72a068 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -14,6 +14,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" @@ -195,6 +197,92 @@ func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + // 预计算 hash（如果不存在），使用 RangeRead 不消耗 Reader + // 这样远端驱动不需要再计算，避免 HTTP body 被重复读取 + md5Hash := s.GetHash().GetHash(utils.MD5) + sha1Hash := s.GetHash().GetHash(utils.SHA1) + sha256Hash := s.GetHash().GetHash(utils.SHA256) + sha1_128kHash := s.GetHash().GetHash(utils.SHA1_128K) + preHash := s.GetHash().GetHash(utils.PRE_HASH) + + // 计算所有缺失的 hash，确保最大兼容性 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(s, utils.MD5, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate MD5: %v", err) + md5Hash = "" + } + } + if len(sha1Hash) != utils.SHA1.Width { + var err error + sha1Hash, err = stream.StreamHashFile(s, utils.SHA1, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1: %v", err) + sha1Hash = "" + } + } + if len(sha256Hash) != utils.SHA256.Width { + var err error + sha256Hash, err = stream.StreamHashFile(s, utils.SHA256, 34, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA256: %v", err) + sha256Hash = "" + } + } + + // 计算特殊 hash（用于秒传验证） + // SHA1_128K: 前128KB的SHA1，115网盘使用 + if len(sha1_128kHash) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * 1024 // 128KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + sha1_128kHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1_128K: %v", err) + sha1_128kHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for SHA1_128K: %v", err) + } + } + + // PRE_HASH: 前1024字节的SHA1，阿里云盘使用 + if len(preHash) != utils.PRE_HASH.Width { + const PreHashSize int64 = 1024 // 1KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + preHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate PRE_HASH: %v", err) + preHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for PRE_HASH: %v", err) + } + } + + // 诊断日志：检查流的状态 + if ss, ok := s.(*stream.SeekableStream); ok { + if ss.Reader != nil { + log.Warnf("[openlist] WARNING: SeekableStream.Reader is not nil for file %s, stream may have been consumed!", s.GetName()) + } + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, @@ -206,14 +294,20 @@ func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStream req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { - req.Header.Set("X-File-Md5", md5) + if len(md5Hash) > 0 { + req.Header.Set("X-File-Md5", md5Hash) + } + if len(sha1Hash) > 0 { + req.Header.Set("X-File-Sha1", sha1Hash) + } + if len(sha256Hash) > 0 { + req.Header.Set("X-File-Sha256", sha256Hash) } - if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { - req.Header.Set("X-File-Sha1", sha1) + if len(sha1_128kHash) > 0 { + req.Header.Set("X-File-Sha1-128k", sha1_128kHash) } - if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { - req.Header.Set("X-File-Sha256", sha256) + if len(preHash) > 0 { + req.Header.Set("X-File-Pre-Hash", preHash) } req.ContentLength = s.GetSize() diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 00a32f001a..a57076e08e 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -48,8 +48,8 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), - Path: target, + Path: path, // Use symlink's own path, not target path } - log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) + log.Debugf("[sftp] obj: %+v, is symlink: %v, target: %s", obj, symlink, target) return obj, nil } diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be83..5ae92524e3 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -192,6 +193,20 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + // Pre-create the destination directory first + t.Status = "ensuring destination directory exists" + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + log.Warnf("[copy_move] failed to ensure destination dir [%s]: %v, will continue", dstActualPath, err) + // Continue anyway - the directory might exist but Get failed due to cache issues + } + + // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues + // Balances between reducing API calls and maintaining fault tolerance + t.Status = "pre-creating subdirectories" + if err := t.preCreateDirectoryTree(objs, t.SrcActualPath, dstActualPath, 1); err != nil { + log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) + // Continue anyway - individual directories will be created on-demand + } existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -263,6 +278,88 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } +// preCreateDirectoryTree is a thin method wrapper that resolves the storage-bound +// makeDir / listSrc functions and delegates to the pure preCreateDirTreeFn helper. +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { + makeDir := func(ctx context.Context, path string) error { + return op.MakeDir(ctx, t.DstStorage, path) + } + listSrc := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.SrcStorage, path, model.ListArgs{}) + } + return preCreateDirTreeFn(t.Ctx(), objs, srcBasePath, dstBasePath, maxDepth, makeDir, listSrc) +} + +// preCreateDirTreeFn recursively scans source directory tree and pre-creates +// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. +// +// - maxDepth=0 – only create dirs in the current objs list (no recursion) +// - maxDepth=1 – also recurse one level deeper, etc. +// - srcBasePath – current source directory path; passed explicitly through all +// recursion levels so that subdirSrcPath is always correct (do NOT use +// t.SrcActualPath, which is fixed at the top-level path). +// +// makeDir and listSrc are injected to enable testing without a real storage driver. +func preCreateDirTreeFn( + ctx context.Context, + objs []model.Obj, + srcBasePath, dstBasePath string, + maxDepth int, + makeDir func(context.Context, string) error, + listSrc func(context.Context, string) ([]model.Obj, error), +) error { + // First pass: create immediate subdirectories + var subdirs []model.Obj + for _, obj := range objs { + // Check for cancellation + if err := ctx.Err(); err != nil { + return err + } + + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + if err := makeDir(ctx, subdirPath); err != nil { + log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) + // Continue with other directories + } + subdirs = append(subdirs, obj) + // No explicit sleep here: drivers that have QPS limits (e.g. 115, BaiduNetDisk) + // implement WaitLimit via a token-bucket rate.Limiter and call it inside their + // MakeDir, so op.MakeDir already blocks at the correct per-driver rate. + } + } + + // Stop recursion if max depth reached + if maxDepth <= 0 { + return nil + } + + // Second pass: recursively scan and create nested subdirectories + for _, subdir := range subdirs { + if err := ctx.Err(); err != nil { + return err + } + + // Build paths relative to srcBasePath (NOT t.SrcActualPath) so that + // deeper recursion levels resolve to the correct source paths. + subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) + subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) + + subObjs, err := listSrc(ctx, subdirSrcPath) + if err != nil { + log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) + continue // Skip this subdirectory, will handle when processing + } + + // Recursively create subdirectories with decreased depth + if err := preCreateDirTreeFn(ctx, subObjs, subdirSrcPath, subdirDstPath, maxDepth-1, makeDir, listSrc); err != nil { + return err + } + } + + return nil +} + var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 0000000000..5c9f4b5399 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,317 @@ +package fs + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ---------- helpers ---------- + +func dirObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: true} +} + +func fileObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: false} +} + +// callRecorder records every path passed to makeDir and listSrc. +type callRecorder struct { + mu sync.Mutex + mkdirs []string + lists []string + // listReturns maps srcPath → objects to return (nil = empty) + listReturns map[string][]model.Obj + // mkdirErr maps dstPath → error to return + mkdirErr map[string]error +} + +func newRecorder() *callRecorder { + return &callRecorder{ + listReturns: make(map[string][]model.Obj), + mkdirErr: make(map[string]error), + } +} + +func (r *callRecorder) makeDir(_ context.Context, path string) error { + r.mu.Lock() + r.mkdirs = append(r.mkdirs, path) + err := r.mkdirErr[path] + r.mu.Unlock() + return err +} + +func (r *callRecorder) listSrc(_ context.Context, path string) ([]model.Obj, error) { + r.mu.Lock() + r.lists = append(r.lists, path) + objs := r.listReturns[path] + r.mu.Unlock() + return objs, nil +} + +func (r *callRecorder) hasMkdir(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.mkdirs { + if p == path { + return true + } + } + return false +} + +func (r *callRecorder) hasList(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.lists { + if p == path { + return true + } + } + return false +} + +// ---------- tests ---------- + +// TestPreCreateDirTreeFn_EmptyObjs: no objects → no calls at all. +func TestPreCreateDirTreeFn_EmptyObjs(t *testing.T) { + rec := newRecorder() + err := preCreateDirTreeFn(context.Background(), nil, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if len(rec.lists) != 0 { + t.Errorf("expected 0 List calls, got %d: %v", len(rec.lists), rec.lists) + } +} + +// TestPreCreateDirTreeFn_OnlyFiles: file objects only → zero MakeDir calls. +func TestPreCreateDirTreeFn_OnlyFiles(t *testing.T) { + objs := []model.Obj{fileObj("a.txt"), fileObj("b.txt")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d", len(rec.mkdirs)) + } +} + +// TestPreCreateDirTreeFn_FlatDirs_MaxDepth0: dirs present, maxDepth=0 → MakeDir +// called for each dir with correct dstPath, NO listSrc calls. +func TestPreCreateDirTreeFn_FlatDirs_MaxDepth0(t *testing.T) { + objs := []model.Obj{dirObj("subA"), fileObj("file.txt"), dirObj("subB")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst/parent", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + if rec.hasMkdir("/dst/parent/file.txt") { + t.Error("MakeDir must NOT be called for a file") + } + if len(rec.lists) != 0 { + t.Errorf("maxDepth=0 must not trigger any List calls, got: %v", rec.lists) + } +} + +// TestPreCreateDirTreeFn_Recursion_CorrectSrcPath is the regression test for the +// srcBasePath bug: with maxDepth=1 the recursive List must use the SUBDIR src path, +// not the original top-level srcBasePath. +func TestPreCreateDirTreeFn_Recursion_CorrectSrcPath(t *testing.T) { + // /src/parent contains [subA(dir), subB(dir)] + // /src/parent/subA contains [subA1(dir)] + // /src/parent/subB contains [] + topObjs := []model.Obj{dirObj("subA"), dirObj("subB")} + rec := newRecorder() + rec.listReturns["/src/parent/subA"] = []model.Obj{dirObj("subA1")} + rec.listReturns["/src/parent/subB"] = []model.Obj{} + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src/parent", "/dst/parent", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // ── first level dirs must be created + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + + // ── listSrc must use subdirSrcPath (NOT the whole /src/parent again) + if !rec.hasList("/src/parent/subA") { + t.Error("listSrc must be called with /src/parent/subA, got:", rec.lists) + } + if !rec.hasList("/src/parent/subB") { + t.Error("listSrc must be called with /src/parent/subB, got:", rec.lists) + } + // The original bug would have called listSrc("/src/parent/subA") as + // stdpath.Join(t.SrcActualPath, "subA") where t.SrcActualPath=="/src/parent", + // but in a deeper recursive call (e.g. maxDepth=2) it would have used + // the top-level path incorrectly; verify the nested mkdir used the right dst. + if !rec.hasMkdir("/dst/parent/subA/subA1") { + t.Error("expected MakeDir(/dst/parent/subA/subA1), got mkdirs:", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion: with maxDepth=1 recursion +// goes exactly one level. The nested list returns another dir, but since maxDepth +// reaches 0 that deeper dir must NOT be listed further. +func TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion(t *testing.T) { + topObjs := []model.Obj{dirObj("sub")} + rec := newRecorder() + // sub contains deeper, deeper contains deepest + rec.listReturns["/src/sub"] = []model.Obj{dirObj("deeper")} + rec.listReturns["/src/sub/deeper"] = []model.Obj{dirObj("deepest")} // should NOT be listed + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/sub") { + t.Error("expected /dst/sub to be created") + } + if !rec.hasMkdir("/dst/sub/deeper") { + t.Error("expected /dst/sub/deeper to be created (within maxDepth=1)") + } + // deepest must NOT be created (would require maxDepth=2) + if rec.hasMkdir("/dst/sub/deeper/deepest") { + t.Error("/dst/sub/deeper/deepest must NOT be created at maxDepth=1") + } + // /src/sub/deeper must NOT be listed (we've hit maxDepth=0 at that point) + if rec.hasList("/src/sub/deeper") { + t.Error("/src/sub/deeper must NOT be listed when maxDepth reaches 0") + } +} + +// TestPreCreateDirTreeFn_ContextCancelled: context cancelled before processing → +// returns ctx.Err, makes zero or partial calls. +func TestPreCreateDirTreeFn_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("no MakeDir should be called after cancellation, got: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_ContextCancelledDuringRecursion: context is cancelled +// during the second-pass recursion loop. +func TestPreCreateDirTreeFn_ContextCancelledDuringRecursion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Two dirs; cancel after the first List in recursion + callCount := 0 + listSrc := func(c context.Context, path string) ([]model.Obj, error) { + callCount++ + cancel() // cancel on first list call + return nil, nil + } + objs := []model.Obj{dirObj("sub1"), dirObj("sub2")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled after cancellation during recursion, got: %v", err) + } + if callCount > 1 { + t.Errorf("listSrc should have been called at most once before ctx.Err fired, got %d", callCount) + } +} + +// TestPreCreateDirTreeFn_MakeDirErrorNonFatal: a MakeDir failure on one dir must +// not stop processing of subsequent dirs. +func TestPreCreateDirTreeFn_MakeDirErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB"), dirObj("subC")} + rec := newRecorder() + rec.mkdirErr["/dst/subA"] = errors.New("quota exceeded") + + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("error should not propagate from MakeDir failure: %v", err) + } + // All three must have been attempted despite the error on subA + for _, p := range []string{"/dst/subA", "/dst/subB", "/dst/subC"} { + if !rec.hasMkdir(p) { + t.Errorf("expected MakeDir(%s) to be called", p) + } + } +} + +// TestPreCreateDirTreeFn_ListErrorNonFatal: a List error for one subdir during +// recursion skips that subdir but continues with the rest. +func TestPreCreateDirTreeFn_ListErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB")} + listCallCount := 0 + listSrc := func(_ context.Context, path string) ([]model.Obj, error) { + listCallCount++ + if path == "/src/subA" { + return nil, errors.New("I/O error") + } + return []model.Obj{dirObj("nested")}, nil + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, listSrc); err != nil { + t.Fatalf("List error must not be fatal: %v", err) + } + // subB's nested dir should still be processed despite subA's List failure + if !rec.hasMkdir("/dst/subB/nested") { + t.Error("expected /dst/subB/nested to be created despite subA list error, mkdirs:", rec.mkdirs) + } + if listCallCount != 2 { + t.Errorf("both subdirs must be attempted for listing, got %d calls", listCallCount) + } +} + +// TestPreCreateDirTreeFn_MixedObjs: mixed files and dirs; only dirs are processed. +func TestPreCreateDirTreeFn_MixedObjs(t *testing.T) { + objs := []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 2 { + t.Errorf("expected exactly 2 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if !rec.hasMkdir("/dst/assets") || !rec.hasMkdir("/dst/pkg") { + t.Errorf("unexpected mkdirs: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_Timeout: context with a very short deadline cancels execution. +func TestPreCreateDirTreeFn_Timeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + time.Sleep(5 * time.Millisecond) // ensure deadline has passed + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected DeadlineExceeded, got: %v", err) + } +} From 1fe5a8dd082bfd3bbd1c5096e5390cf4f8ee9043 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:32:02 +0800 Subject: [PATCH 053/107] feat(frontend): dynamic frontend fetching, CI upgrades, and build infrastructure - Add dynamic frontend pull from GitHub rolling release at startup - Implement frontend caching mechanism with ETag-based validation - Track rolling release by tag commit hash - Unify configurable frontend repo and release channel across build and runtime - Upgrade GitHub Actions to Node24-compatible versions (checkout v5, setup-go v6, etc.) - Update Docker workflow to support frontend version matrix builds - Optimize CI build speed with smart caching --- .github/workflows/beta_release.yml | 1 + .github/workflows/build.yml | 6 +- .github/workflows/issue_pr_comment.yml | 4 +- .github/workflows/release_docker.yml | 8 +- .github/workflows/test_docker.yml | 144 ++++--- .gitignore | 5 +- build.sh | 47 ++- internal/bootstrap/run.go | 2 + internal/conf/config.go | 4 +- internal/conf/var.go | 1 + internal/frontend/fetcher.go | 501 +++++++++++++++++++++++++ internal/frontend/fetcher_test.go | 480 +++++++++++++++++++++++ internal/frontend/watcher.go | 122 ++++++ public/dist/README.md | 1 - server/handles/fsread.go | 6 +- server/static/static.go | 88 ++++- 16 files changed, 1331 insertions(+), 89 deletions(-) create mode 100644 internal/frontend/fetcher.go create mode 100644 internal/frontend/fetcher_test.go create mode 100644 internal/frontend/watcher.go delete mode 100644 public/dist/README.md diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 7827f1dab1..496fd0e0b6 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -140,6 +140,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} env: GOFLAGS: ${{ matrix.goflags }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58cc3ef6f8..e59c5364e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: strategy: @@ -55,7 +58,8 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} output: openlist$ext - name: Verify musl binary is static diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index 1b51e23b9e..bc29e6bf6b 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -16,7 +16,7 @@ jobs: if: github.event_name == 'issues' steps: - name: Check issue for unchecked tasks and reply - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | let comment = ""; @@ -81,7 +81,7 @@ jobs: if: github.event_name == 'pull_request' steps: - name: Check PR title for required prefix and comment - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const title = context.payload.pull_request.title || ""; diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 1badaf45a9..4292dacfd2 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -51,7 +51,7 @@ jobs: - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -91,7 +91,7 @@ jobs: - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -147,7 +147,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' @@ -231,7 +231,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 5448adc56d..f1947f4d1b 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,150 +1,196 @@ name: Beta Release (Docker) - on: workflow_dispatch: + inputs: + frontend_repo: + description: 'Frontend repo, e.g. Ironboxplus/OpenList-Frontend' + required: false + default: 'OpenListTeam/OpenList-Frontend' + type: string + frontend_channel: + description: 'Frontend release channel to build' + required: false + default: rolling + type: choice + options: + - rolling + - latest + - both push: branches: - main pull_request: branches: - - main + - copy + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + IMAGE_NAME: openlist REGISTRY: ghcr.io - ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} - IMAGE_TAGS_BETA: | - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + ARTIFACT_NAME_PREFIX: 'binaries_docker_release' + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' + # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) + # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest + strategy: + matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - name: Setup Go + uses: actions/setup-go@v6 with: go-version: '1.25.0' + cache: true + cache-dependency-path: go.sum + + - name: Get Frontend Cache Version + id: frontend-cache + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: ${{ matrix.frontend_channel }} + run: | + frontend_repo="${{ env.FRONTEND_REPO }}" + web_version="$WEB_VERSION" + github_auth_args=() + + if [ -n "$GH_TOKEN" ]; then + github_auth_args=(-H "Authorization: Bearer $GH_TOKEN") + fi + + if [ "$web_version" = "latest" ]; then + frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') + else + frontend_version="$web_version" + fi + + echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" + echo "version=$frontend_version" >> "$GITHUB_OUTPUT" + echo "Frontend repo: $frontend_repo" + echo "Frontend cache version: $frontend_version" + + - name: Cache Frontend + id: cache-frontend + uses: actions/cache@v5 + with: + path: public/dist + key: frontend-${{ steps.frontend-cache.outputs.repo }}-${{ steps.frontend-cache.outputs.version }} + restore-keys: | + frontend-${{ steps.frontend-cache.outputs.repo }}- - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash build.sh prepare docker-multiplatform - - name: Build go binary (beta) + - name: Build go binary run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} + WEB_VERSION: ${{ matrix.frontend_channel }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} overwrite: true - path: | - build/ - !build/*.tgz - !build/musl-libs/** + path: build/linux/amd64/openlist release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} + # 构建所有变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - tag_favor: "" + image_tag_suffix: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - tag_favor: "suffix=-ffmpeg,onlatest=true" + image_tag_suffix: "-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - tag_favor: "suffix=-aria2,onlatest=true" + image_tag_suffix: "-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - tag_favor: "suffix=-aio,onlatest=true" + image_tag_suffix: "-aio" steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} - path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} + path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} - tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + tags: | + type=raw,value=front-${{ matrix.frontend_channel }}${{ matrix.image_tag_suffix }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' && matrix.image == 'latest' }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }},mode=max diff --git a/.gitignore b/.gitignore index 1d71f0d608..d421551100 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .DS_Store output/ /dist/ - +.omx # Binaries for programs and plugins *.exe *.exe~ @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +nul diff --git a/build.sh b/build.sh index b6baca1fe7..d37ddad730 100644 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ if [[ "$*" == *"lite"* ]]; then useLite=true fi +skipFrontendFetch="${SKIP_FRONTEND_FETCH:-false}" + if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" @@ -31,6 +33,10 @@ else webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi +if [ -n "$WEB_VERSION" ]; then + webVersion="$WEB_VERSION" +fi + echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then @@ -46,6 +52,7 @@ ldflags="\ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$version' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ +-X 'github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=$frontendRepo' \ " # Keep sqlite driver tag selection centralized to avoid target drift. @@ -97,6 +104,11 @@ AssertStaticBinary() { } FetchWebRolling() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -110,6 +122,11 @@ FetchWebRolling() { } FetchWebRelease() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') @@ -236,8 +253,8 @@ BuildDockerMultiplatform() { docker_lflags="$(GetMuslStaticLdflags)" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -257,15 +274,17 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" - done + # ARM docker variants stay disabled on this branch to keep the workflow x64-only. + # If they are re-enabled later, they should follow the same static-link pattern. + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" + # done } BuildRelease() { @@ -655,7 +674,11 @@ if [ "$buildType" = "dev" ]; then fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then - FetchWebRolling + if [ "$WEB_VERSION" = "latest" ]; then + FetchWebRelease + else + FetchWebRolling + fi else FetchWebRelease fi diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba657..ff02509baa 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -273,6 +274,7 @@ func Start() { func Shutdown(timeout time.Duration) { utils.Log.Println("Shutdown server...") + frontend.StopWatcher() fs.ArchiveContentUploadTaskManager.RemoveAll() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/internal/conf/config.go b/internal/conf/config.go index 0f66761168..223fd9606c 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -122,6 +122,7 @@ type Config struct { TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` + FrontendRepo string `json:"frontend_repo" env:"FRONTEND_REPO"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` AutoMemoryLimit int `json:"auto_memory_limit" env:"AUTO_MEMORY_LIMIT"` @@ -168,7 +169,8 @@ func DefaultConfig(dataDir string) *Config { Host: "http://localhost:7700", Index: "openlist", }, - BleveDir: indexDir, + BleveDir: indexDir, + FrontendRepo: FrontendRepoDefault, Log: LogConfig{ Enable: true, Name: logPath, diff --git a/internal/conf/var.go b/internal/conf/var.go index 6b25bcfb5b..b11f3086ed 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,6 +12,7 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" + FrontendRepoDefault string = "OpenListTeam/OpenList-Frontend" ) var ( diff --git a/internal/frontend/fetcher.go b/internal/frontend/fetcher.go new file mode 100644 index 0000000000..363d5dfcae --- /dev/null +++ b/internal/frontend/fetcher.go @@ -0,0 +1,501 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + defaultFrontendRepo = "OpenListTeam/OpenList-Frontend" + versionFile = ".frontend_version" + distDirName = "dist" +) + +// FetchResult contains the result of a fetch operation +type FetchResult struct { + Version string + Downloaded bool + DistPath string +} + +// GetDistPath returns the path where dynamically fetched frontend dist is stored +func GetDistPath() string { + return filepath.Join(flags.DataDir, "frontend_dist") +} + +// GetVersionFilePath returns the path to the version tracking file +func GetVersionFilePath() string { + return filepath.Join(GetDistPath(), versionFile) +} + +// HasValidDist checks if the dynamic dist directory exists and has an index.html +func HasValidDist() bool { + distPath := GetDistPath() + _, err := os.Stat(filepath.Join(distPath, distDirName, "index.html")) + return err == nil +} + +// ReadCurrentVersion reads the currently cached version from disk +func ReadCurrentVersion() string { + data, err := os.ReadFile(GetVersionFilePath()) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// writeVersion writes the version string to the version tracking file +func writeVersion(version string) error { + return os.WriteFile(GetVersionFilePath(), []byte(version), 0644) +} + +// FetchFromRolling downloads the frontend dist from the GitHub rolling release +func FetchFromRolling(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "rolling", "") +} + +// FetchFromLatest downloads the frontend dist from the GitHub latest release +func FetchFromLatest(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "", "") +} + +// githubRelease represents a GitHub release for JSON parsing +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []struct { + BrowserDownloadURL string `json:"browser_download_url"` + Name string `json:"name"` + } `json:"assets"` + PublishedAt string `json:"published_at"` +} + +type githubRef struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +type githubAnnotatedTag struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +func getFrontendRepo() string { + if conf.Conf != nil && strings.TrimSpace(conf.Conf.FrontendRepo) != "" { + return strings.TrimSpace(conf.Conf.FrontendRepo) + } + return defaultFrontendRepo +} + +func shortHash(sha string) string { + const shortLen = 12 + if len(sha) > shortLen { + return sha[:shortLen] + } + return sha +} + +func versionIdentifier(tag, commitSHA, fallback string) string { + if strings.TrimSpace(commitSHA) != "" { + return fmt.Sprintf("%s@%s", tag, shortHash(commitSHA)) + } + if strings.TrimSpace(fallback) != "" { + return fallback + } + return tag +} + +func resolveTagCommitSHA(ctx context.Context, client *http.Client, baseURL, tag string) (string, error) { + if strings.TrimSpace(tag) == "" { + return "", fmt.Errorf("empty tag") + } + + apiBase := "https://api.github.com" + if baseURL != "" { + apiBase = strings.TrimRight(baseURL, "/") + } + + repo := getFrontendRepo() + refURL := fmt.Sprintf("%s/repos/%s/git/ref/tags/%s", apiBase, repo, url.PathEscape(tag)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, refURL, nil) + if err != nil { + return "", fmt.Errorf("create ref request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetch tag ref: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("tag ref API returned %d: %s", resp.StatusCode, string(body)) + } + + var ref githubRef + if err := json.NewDecoder(resp.Body).Decode(&ref); err != nil { + return "", fmt.Errorf("decode ref JSON: %w", err) + } + + switch ref.Object.Type { + case "commit": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty commit sha in ref response") + } + return ref.Object.SHA, nil + case "tag": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty tag sha in ref response") + } + tagObjURL := fmt.Sprintf("%s/repos/%s/git/tags/%s", apiBase, repo, ref.Object.SHA) + tagReq, err := http.NewRequestWithContext(ctx, http.MethodGet, tagObjURL, nil) + if err != nil { + return "", fmt.Errorf("create tag object request: %w", err) + } + tagReq.Header.Set("Accept", "application/vnd.github.v3+json") + tagReq.Header.Set("User-Agent", "OpenList") + + tagResp, err := client.Do(tagReq) + if err != nil { + return "", fmt.Errorf("fetch tag object: %w", err) + } + defer tagResp.Body.Close() + + if tagResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(tagResp.Body) + return "", fmt.Errorf("tag object API returned %d: %s", tagResp.StatusCode, string(body)) + } + + var tagObj githubAnnotatedTag + if err := json.NewDecoder(tagResp.Body).Decode(&tagObj); err != nil { + return "", fmt.Errorf("decode tag object JSON: %w", err) + } + if tagObj.Object.SHA == "" { + return "", fmt.Errorf("empty object sha in tag object response") + } + return tagObj.Object.SHA, nil + default: + if ref.Object.SHA == "" { + return "", fmt.Errorf("unsupported ref object type %q with empty sha", ref.Object.Type) + } + return ref.Object.SHA, nil + } +} + +// fetchFromTag downloads frontend dist from a GitHub release tag. +// If baseURL is non-empty, it replaces api.github.com (used for testing). +func fetchFromTag(ctx context.Context, tag string, baseURL string) (*FetchResult, error) { + repo := getFrontendRepo() + var apiURL string + if baseURL != "" { + if tag == "" { + apiURL = fmt.Sprintf("%s/repos/%s/releases/latest", baseURL, repo) + } else { + apiURL = fmt.Sprintf("%s/repos/%s/releases/tags/%s", baseURL, repo, tag) + } + } else { + if tag == "" { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo) + } else { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/tags/%s", repo, tag) + } + } + + client := newHTTPClient() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch release info: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("github API returned %d: %s", resp.StatusCode, string(body)) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("decode release JSON: %w", err) + } + + // Find the dist tarball URL (non-lite) + var tarURL string + for _, asset := range release.Assets { + if strings.Contains(asset.Name, "openlist-frontend-dist") && + !strings.Contains(asset.Name, "lite") && + strings.HasSuffix(asset.Name, ".tar.gz") { + tarURL = asset.BrowserDownloadURL + break + } + } + if tarURL == "" { + return nil, fmt.Errorf("no frontend dist tarball found in release %s", release.TagName) + } + + commitSHA, err := resolveTagCommitSHA(ctx, client, baseURL, release.TagName) + if err != nil { + utils.Log.Warnf("[frontend] failed to resolve tag %s hash: %v", release.TagName, err) + } + + resolvedVersion := versionIdentifier(release.TagName, commitSHA, tarURL) + + // Use tag+commit-hash as the primary version identifier. + // For rolling releases the tag itself is static, but its target commit moves. + // If hash resolve fails, fallback to tarball URL so updates can still be detected. + currentVersion := ReadCurrentVersion() + if currentVersion == resolvedVersion && HasValidDist() { + utils.Log.Infof("[frontend] version %s already cached, skipping download", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: false, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil + } + + utils.Log.Infof("[frontend] downloading version %s from %s", resolvedVersion, tarURL) + if err := downloadAndExtract(ctx, client, tarURL); err != nil { + return nil, fmt.Errorf("download and extract: %w", err) + } + + if err := writeVersion(resolvedVersion); err != nil { + utils.Log.Warnf("[frontend] failed to write version file: %v", err) + } + + utils.Log.Infof("[frontend] successfully fetched version %s", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: true, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil +} + +func downloadAndExtract(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create download request: %w", err) + } + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download tarball: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download returned status %d", resp.StatusCode) + } + + destDir := GetDistPath() + tmpDir := filepath.Join(destDir, ".tmp-"+fmt.Sprintf("%d", time.Now().UnixNano())) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := extractTarGz(resp.Body, tmpDir); err != nil { + return fmt.Errorf("extract tar.gz: %w", err) + } + + // Determine the source directory: + // If the tarball contains a "dist" subdirectory, use it; + // otherwise, the files are at the root and we use tmpDir directly. + srcDir := tmpDir + if _, err := os.Stat(filepath.Join(tmpDir, distDirName)); err == nil { + srcDir = filepath.Join(tmpDir, distDirName) + } + + // Atomic swap: rename source to final + finalDir := filepath.Join(destDir, distDirName) + oldDir := filepath.Join(destDir, distDirName+".old") + // Remove previous backup if exists + os.RemoveAll(oldDir) + // Move current dist out of the way if it exists + os.Rename(finalDir, oldDir) + // Move new dist into place + if err := os.Rename(srcDir, finalDir); err != nil { + // Rollback + os.RemoveAll(finalDir) + os.Rename(oldDir, finalDir) + return fmt.Errorf("rename new dist: %w", err) + } + os.RemoveAll(oldDir) + + return nil +} + +func extractTarGz(r io.Reader, dest string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("tar next: %w", err) + } + + // Normalize: strip leading ./ so "./dist" becomes "dist" + name := strings.TrimPrefix(hdr.Name, "./") + if name == "" || name == "." { + continue // skip bare directory entry + } + + target := filepath.Join(dest, name) + + // Security: prevent path traversal + if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("path traversal detected: %s", hdr.Name) + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + +// newHTTPClient creates an HTTP client that respects proxy configuration +func newHTTPClient() *http.Client { + transport := &http.Transport{} + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { + if proxyURL := mustParseURL(conf.Conf.ProxyAddress); proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{ + Transport: transport, + Timeout: 5 * time.Minute, + } +} + +func mustParseURL(raw string) *url.URL { + u, err := url.Parse(raw) + if err != nil { + utils.Log.Warnf("[frontend] invalid proxy URL %q: %v", raw, err) + return nil + } + return u +} + +// EnsureDist ensures a valid frontend dist is available, fetching if necessary. +// It first tries the dynamic dist, then falls back to fetching from GitHub. +// Returns the path to the dist directory, or empty string if no dist is available. +func EnsureDist(ctx context.Context) string { + // If user explicitly configured dist_dir, use that + if conf.Conf != nil && conf.Conf.DistDir != "" { + if _, err := os.Stat(filepath.Join(conf.Conf.DistDir, "index.html")); err == nil { + return conf.Conf.DistDir + } + } + + // Check if dynamic dist already exists + if HasValidDist() { + return filepath.Join(GetDistPath(), distDirName) + } + + // If auto-fetch is enabled (and WebVersion is rolling/beta/dev), try fetching + if shouldAutoFetch() { + utils.Log.Infof("[frontend] no local dist found, fetching from rolling release...") + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] failed to fetch from rolling: %v", err) + // Fall through to return empty (embedded dist will be used as fallback) + return "" + } + return result.DistPath + } + + return "" +} + +func shouldAutoFetch() bool { + v := conf.WebVersion + return v == "" || v == "rolling" || v == "beta" || v == "dev" +} + +// Ensure the directory exists for the frontend dist +func init() { + _ = os.MkdirAll(GetDistPath(), 0755) +} + +// Ensure that the sync.Once pattern is used for the fetcher +var ( + fetchMu sync.Mutex + fetchDone bool + fetchResult string +) + +// EnsureDistOnce is a thread-safe version of EnsureDist that only fetches once per process. +// On failure, it does not lock the state so subsequent calls can retry. +func EnsureDistOnce(ctx context.Context) string { + fetchMu.Lock() + defer fetchMu.Unlock() + if fetchDone { + return fetchResult + } + result := EnsureDist(ctx) + if result != "" { + fetchResult = result + fetchDone = true + } + return result +} + +// ResetFetchState resets the fetch state (used for testing or re-fetch) +func ResetFetchState() { + fetchMu.Lock() + defer fetchMu.Unlock() + fetchDone = false + fetchResult = "" +} diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go new file mode 100644 index 0000000000..14f7a33913 --- /dev/null +++ b/internal/frontend/fetcher_test.go @@ -0,0 +1,480 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// createTestTarGz creates a tar.gz containing files with given names and contents +func createTestTarGz(t *testing.T, files map[string]string) []byte { + t.Helper() + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return + } + if _, err := tw.Write([]byte(content)); err != nil { + return + } + } + }() + data, err := io.ReadAll(pr) + if err != nil { + t.Fatalf("read tar.gz: %v", err) + } + return data +} + +func TestExtractTarGz(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "dist/index.html": "hello", + "dist/assets/app.js": "console.log('app')", + "dist/assets/style.css": "body {}", + "dist/images/logo.svg": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz: %v", err) + } + + for name, expectedContent := range files { + path := filepath.Join(tmpDir, name) + data, err := os.ReadFile(path) + if err != nil { + t.Errorf("read %s: %v", name, err) + continue + } + if string(data) != expectedContent { + t.Errorf("content of %s: got %q, want %q", name, string(data), expectedContent) + } + } +} + +func TestExtractTarGzDotSlash(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "./dist/index.html": "dot-slash", + "./": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz with ./ prefix: %v", err) + } + + data, err := os.ReadFile(filepath.Join(tmpDir, "dist", "index.html")) + if err != nil { + t.Fatalf("read dist/index.html: %v", err) + } + if string(data) != "dot-slash" { + t.Errorf("got %q, want dot-slash content", string(data)) + } +} + +func TestExtractTarGzPathTraversal(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "../../../etc/passwd": "root:x:0:0", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err == nil { + t.Fatal("expected error for path traversal, got nil") + } + if !strings.Contains(err.Error(), "path traversal") { + t.Errorf("expected path traversal error, got: %v", err) + } +} + +func TestHasValidDist(t *testing.T) { + if HasValidDist() { + t.Log("HasValidDist returned true (may have existing dist from previous runs)") + } +} + +func TestWriteAndReadVersion(t *testing.T) { + _ = os.MkdirAll(GetDistPath(), 0755) + versionPath := GetVersionFilePath() + + origData, origErr := os.ReadFile(versionPath) + defer func() { + if origErr == nil { + _ = os.WriteFile(versionPath, origData, 0644) + } else { + _ = os.Remove(versionPath) + } + }() + + testVersion := "v1.0.0-test" + if err := writeVersion(testVersion); err != nil { + t.Fatalf("writeVersion: %v", err) + } + + got := ReadCurrentVersion() + if got != testVersion { + t.Errorf("ReadCurrentVersion: got %q, want %q", got, testVersion) + } +} + +func TestShouldAutoFetch(t *testing.T) { + origVersion := conf.WebVersion + defer func() { conf.WebVersion = origVersion }() + + tests := []struct { + version string + want bool + }{ + {"", true}, + {"rolling", true}, + {"beta", true}, + {"dev", true}, + {"v3.0.0", false}, + {"latest", false}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + conf.WebVersion = tt.version + if got := shouldAutoFetch(); got != tt.want { + t.Errorf("shouldAutoFetch(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestFetchFromRollingIntegration(t *testing.T) { + files := map[string]string{ + "./dist/index.html": "integration", + "./dist/assets/test.js": "console.log('test')", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-test", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/frontend.tar.gz" + }] + }`, ts.URL))) + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling-test": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "0123456789abcdef0123456789abcdef01234567" + } + }`)) + case "/download/frontend.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + ctx := context.Background() + result, err := fetchFromTag(ctx, "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + + if result.Version != "rolling-test@0123456789ab" { + t.Errorf("version: got %q, want %q", result.Version, "rolling-test@0123456789ab") + } + if !result.Downloaded { + t.Error("expected Downloaded=true") + } + + idx, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(idx) != "integration" { + t.Errorf("index.html: got %q", string(idx)) + } + + ver := ReadCurrentVersion() + expectedVer := "rolling-test@0123456789ab" + if ver != expectedVer { + t.Errorf("version file: got %q, want %q", ver, expectedVer) + } +} + +func TestLegacyConfigJSONGetsDefaultFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"site_url":"https://example.com"}`), cfg); err != nil { + t.Fatalf("unmarshal legacy config: %v", err) + } + if cfg.FrontendRepo != defaultFrontendRepo { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, defaultFrontendRepo) + } +} + +func TestDefaultConfigUsesBuiltFrontendRepoDefault(t *testing.T) { + orig := conf.FrontendRepoDefault + conf.FrontendRepoDefault = "Ironboxplus/OpenList-Frontend" + defer func() { conf.FrontendRepoDefault = orig }() + + cfg := conf.DefaultConfig("data") + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, "Ironboxplus/OpenList-Frontend") + } +} + +func TestExistingConfigJSONKeepsFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"frontend_repo":"Ironboxplus/OpenList-Frontend"}`), cfg); err != nil { + t.Fatalf("unmarshal config with frontend_repo: %v", err) + } + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q", cfg.FrontendRepo) + } +} + +func TestFetchFromTagUsesConfiguredFrontendRepo(t *testing.T) { + origConf := conf.Conf + if origConf == nil { + conf.Conf = &conf.Config{} + } else { + confCopy := *origConf + conf.Conf = &confCopy + } + defer func() { conf.Conf = origConf }() + conf.Conf.FrontendRepo = "Ironboxplus/OpenList-Frontend" + + files := map[string]string{ + "./dist/index.html": "custom-repo", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/Ironboxplus/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-custom", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/custom.tar.gz" + }] + }`, ts.URL))) + case "/repos/Ironboxplus/OpenList-Frontend/git/ref/tags/rolling-custom": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "fedcba9876543210fedcba9876543210fedcba98" + } + }`)) + case "/download/custom.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tarData) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0o755) + _ = os.RemoveAll(filepath.Join(GetDistPath(), distDirName)) + _ = os.Remove(GetVersionFilePath()) + + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag(custom repo): %v", err) + } + if result.Version != "rolling-custom@fedcba987654" { + t.Fatalf("version: got %q, want %q", result.Version, "rolling-custom@fedcba987654") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "custom-repo" { + t.Fatalf("index.html: got %q", string(data)) + } +} + +func TestResolveTagCommitSHA_AnnotatedTag(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "tag", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }`)) + case "/repos/OpenListTeam/OpenList-Frontend/git/tags/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + sha, err := resolveTagCommitSHA(context.Background(), ts.Client(), ts.URL, "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA: %v", err) + } + if sha != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("sha: got %q, want %q", sha, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + } +} + +func TestVersionIdentifierFallback(t *testing.T) { + tests := []struct { + name string + tag string + sha string + fallback string + want string + }{ + {name: "hash preferred", tag: "rolling", sha: "0123456789abcdef", fallback: "fallback", want: "rolling@0123456789ab"}, + {name: "fallback url", tag: "rolling", sha: "", fallback: "http://example.com/dist.tar.gz", want: "http://example.com/dist.tar.gz"}, + {name: "tag only", tag: "rolling", sha: "", fallback: "", want: "rolling"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := versionIdentifier(tt.tag, tt.sha, tt.fallback) + if got != tt.want { + t.Fatalf("versionIdentifier() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestResolveTagCommitSHA_RealGitHubWithProxy10808(t *testing.T) { + if os.Getenv("OPENLIST_REAL_GITHUB_TEST") != "1" { + t.Skip("set OPENLIST_REAL_GITHUB_TEST=1 to run real GitHub integration test (proxy 127.0.0.1:10808 recommended)") + } + + if os.Getenv("HTTP_PROXY") == "" && os.Getenv("http_proxy") == "" { + _ = os.Setenv("HTTP_PROXY", "http://127.0.0.1:10808") + } + if os.Getenv("HTTPS_PROXY") == "" && os.Getenv("https_proxy") == "" { + _ = os.Setenv("HTTPS_PROXY", "http://127.0.0.1:10808") + } + + client := newHTTPClient() + sha, err := resolveTagCommitSHA(context.Background(), client, "", "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA(real): %v", err) + } + + matched, _ := regexp.MatchString("^[0-9a-f]{40}$", sha) + if !matched { + t.Fatalf("sha format invalid: %q", sha) + } + + // Optional sanity: ensure API can fetch release JSON in real scenario + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling", nil) + if err != nil { + t.Fatalf("create release request: %v", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("fetch release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("release API status=%d body=%s", resp.StatusCode, string(body)) + } + + var release map[string]any + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + t.Fatalf("decode release: %v", err) + } + if release["tag_name"] == nil { + t.Fatalf("release tag_name missing") + } +} + +func TestEnsureDistOnceFailureDoesNotLock(t *testing.T) { + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0755) + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + origVersion := conf.WebVersion + conf.WebVersion = "v3.0.0" // shouldAutoFetch returns false + defer func() { conf.WebVersion = origVersion }() + + ctx := context.Background() + + result := EnsureDistOnce(ctx) + if result != "" { + t.Errorf("expected empty result, got %q", result) + } + + // Second call should also work (not locked by previous failure) + result2 := EnsureDistOnce(ctx) + if result2 != "" { + t.Errorf("expected empty result on retry, got %q", result2) + } +} diff --git a/internal/frontend/watcher.go b/internal/frontend/watcher.go new file mode 100644 index 0000000000..1a0102fd14 --- /dev/null +++ b/internal/frontend/watcher.go @@ -0,0 +1,122 @@ +package frontend + +import ( + "context" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const defaultCheckInterval = 30 * time.Minute + +// Watcher periodically checks for new frontend versions and fetches them. +type Watcher struct { + interval time.Duration + stopCh chan struct{} + stopped bool + mu sync.Mutex + onUpdated func() +} + +// NewWatcher creates a new frontend watcher. +// onUpdated is called when a new version is fetched (used to reload static files). +func NewWatcher(onUpdated func()) *Watcher { + return &Watcher{ + interval: defaultCheckInterval, + stopCh: make(chan struct{}), + onUpdated: onUpdated, + } +} + +// SetInterval changes the check interval. Must be called before Start. +func (w *Watcher) SetInterval(d time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + w.interval = d +} + +// Start begins the periodic check loop in a background goroutine. +func (w *Watcher) Start() { + w.mu.Lock() + interval := w.interval + w.mu.Unlock() + + go func() { + utils.Log.Infof("[frontend] watcher started, checking every %s", interval) + // Check immediately on start, then periodically + w.check() + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-w.stopCh: + utils.Log.Infof("[frontend] watcher stopped") + return + case <-ticker.C: + w.check() + } + } + }() +} + +// Stop signals the watcher to stop. +func (w *Watcher) Stop() { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return + } + w.stopped = true + close(w.stopCh) +} + +func (w *Watcher) check() { + if !shouldAutoFetch() { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] watcher check failed: %v", err) + return + } + + if result.Downloaded { + utils.Log.Infof("[frontend] watcher fetched new version: %s", result.Version) + if w.onUpdated != nil { + w.onUpdated() + } + } +} + +// globalWatcher is the singleton watcher instance +var ( + globalWatcher *Watcher + watcherMu sync.Mutex +) + +// StartWatcher starts the global frontend watcher. +func StartWatcher(onUpdated func()) { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + return + } + globalWatcher = NewWatcher(onUpdated) + globalWatcher.Start() +} + +// StopWatcher stops the global frontend watcher. +func StopWatcher() { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + globalWatcher.Stop() + globalWatcher = nil + } +} diff --git a/public/dist/README.md b/public/dist/README.md deleted file mode 100644 index d8709fb571..0000000000 --- a/public/dist/README.md +++ /dev/null @@ -1 +0,0 @@ -## Put dist of frontend here. \ No newline at end of file diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 856f46e543..8fd731af7b 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -230,6 +230,10 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + hashInfo := obj.GetHash().Export() + if hashInfo == nil { + hashInfo = make(map[*utils.HashType]string) + } resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), @@ -237,7 +241,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), + HashInfo: hashInfo, Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), diff --git a/server/static/static.go b/server/static/static.go index 29f97ff746..3850b6d11f 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -1,6 +1,7 @@ package static import ( + "context" "encoding/json" "errors" "fmt" @@ -8,10 +9,14 @@ import ( "io/fs" "net/http" "os" + "path/filepath" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/public" @@ -32,21 +37,60 @@ type Manifest struct { Icons []ManifestIcon `json:"icons"` } -var static fs.FS +// reloadableFS wraps fs.FS with thread-safe swapping. +// This allows gin StaticFS routes (which capture the fs.FS at registration time) +// to serve updated files after a watcher-triggered reload. +type reloadableFS struct { + mu sync.RWMutex + current fs.FS +} + +func (r *reloadableFS) Open(name string) (fs.File, error) { + r.mu.RLock() + current := r.current + r.mu.RUnlock() + return current.Open(name) +} + +func (r *reloadableFS) swap(f fs.FS) { + r.mu.Lock() + r.current = f + r.mu.Unlock() +} + +var staticFS = &reloadableFS{} func initStatic() { utils.Log.Debug("Initializing static file system...") - if conf.Conf.DistDir == "" { - dist, err := fs.Sub(public.Public, "dist") - if err != nil { - utils.Log.Fatalf("failed to read dist dir: %v", err) - } - static = dist - utils.Log.Debug("Using embedded dist directory") + // 1. User explicitly configured dist_dir + if conf.Conf.DistDir != "" { + staticFS.swap(os.DirFS(conf.Conf.DistDir)) + utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + return + } + // 2. Try dynamic dist (fetched from rolling release) + if frontend.HasValidDist() { + distPath := filepath.Join(frontend.GetDistPath(), "dist") + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using dynamically fetched dist: %s", distPath) + return + } + // 3. Try auto-fetching from rolling (short timeout to avoid blocking startup) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + distPath := frontend.EnsureDistOnce(ctx) + cancel() + if distPath != "" { + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using auto-fetched dist: %s", distPath) return } - static = os.DirFS(conf.Conf.DistDir) - utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + // 4. Final fallback to embedded dist + dist, err := fs.Sub(public.Public, "dist") + if err != nil { + utils.Log.Fatalf("failed to read dist dir: %v", err) + } + staticFS.swap(dist) + utils.Log.Debug("Using embedded dist directory") } func replaceStrings(content string, replacements map[string]string) string { @@ -74,7 +118,7 @@ func initIndex(siteConfig SiteConfig) { utils.Log.Info("Successfully fetched index.html from CDN") } else { utils.Log.Debug("Reading index.html from static files system...") - indexFile, err := static.Open("index.html") + indexFile, err := staticFS.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") @@ -131,13 +175,21 @@ func UpdateIndex() { utils.Log.Debug("Index.html update completed") } +// ReloadStatic reloads the static files from disk (called by the watcher after an update) +func ReloadStatic() { + utils.Log.Info("[static] reloading static files after frontend update...") + siteConfig := getSiteConfig() + initStatic() + initIndex(siteConfig) +} + func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() - + // Get site title from settings siteTitle := setting.GetStr(conf.SiteTitle) - + // Get logo from settings, use the first line (light theme logo) logoSetting := setting.GetStr(conf.Logo) logoUrl := strings.Split(logoSetting, "\n")[0] @@ -167,7 +219,7 @@ func ManifestJSON(c *gin.Context) { c.Header("Content-Type", "application/json") c.Header("Cache-Control", "public, max-age=3600") // cache for 1 hour - + if err := json.NewEncoder(c.Writer).Encode(manifest); err != nil { utils.Log.Errorf("Failed to encode manifest.json: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to generate manifest"}) @@ -180,8 +232,12 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { siteConfig := getSiteConfig() initStatic() initIndex(siteConfig) + + // Start the frontend watcher for periodic updates + frontend.StartWatcher(ReloadStatic) + folders := []string{"assets", "images", "streamer", "static"} - + if conf.Conf.Cdn == "" { utils.Log.Debug("Setting up static file serving...") r.Use(func(c *gin.Context) { @@ -192,7 +248,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for _, folder := range folders { - sub, err := fs.Sub(static, folder) + sub, err := fs.Sub(staticFS, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } From fe89c1fbec264cbc62f002f799818c1f5f844fe5 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:32:12 +0800 Subject: [PATCH 054/107] chore: add project docs and update dependencies (115-sdk-go fork) - Add CLAUDE.md for project guidance and development instructions - Add compatibility report - Use Ironboxplus/115-sdk-go v0.2.5 fork in go.mod replace directive --- CLAUDE.md | 346 ++++++++++++++++++++++++++++++++++++++++ COMPATIBILITY_REPORT.md | 204 +++++++++++++++++++++++ go.mod | 2 +- go.sum | 6 +- 4 files changed, 553 insertions(+), 5 deletions(-) create mode 100644 CLAUDE.md create mode 100644 COMPATIBILITY_REPORT.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..c3ffdded09 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,346 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Core Development Principles + +1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal +2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file +3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload + +## Build and Development Commands + +```bash +# Development +go run main.go # Run backend server (default port 5244) +air # Hot reload during development (uses .air.toml) +./build.sh dev # Build development version with frontend +./build.sh release # Build release version + +# Testing +go test ./... # Run all tests + +# Docker +docker-compose up # Run with docker-compose +docker build -f Dockerfile . # Build docker image +``` + +**Build Script Details** (`build.sh`): +- Fetches frontend from OpenListTeam/OpenList-Frontend releases +- Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` +- Supports `dev`, `beta`, and release builds +- Downloads prebuilt frontend distribution automatically + +**Go Version**: Requires Go 1.23.4+ + +## Architecture Overview + +### Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` - JSON field name + - `type:"select"` - Input type (select, string, text, bool, number) + - `required:"true"` - Required field + - `options:"a,b,c"` - Dropdown options + - `default:"value"` - Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { + // Initialize client, login, etc. +} + +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // Return list of files/folders +} + +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Return download URL or RangeReader +} +``` + +### Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +### Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +### Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +### Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF**: Core interface for range-based reading + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader**: Wraps RangeReader with automatic link refresh + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel + - Assembles final stream + +**Stream Types and Reader Management**: + +⚠️ **CRITICAL**: SeekableStream.Reader must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** - use lazy initialization via `generateReader()` + +**Common Pitfall - Early Reader Creation**: +```go +// ❌ WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// ✅ CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +**Hash Calculation for Uploads**: +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + +**Link Refresh Pattern**: +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + // Get fresh link from storage driver + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +**Proxy Function** (`server/common/proxy.go`): + +Handles multiple scenarios: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +### Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` - Load config, environment variables +2. `Log()` - Initialize logging +3. `InitDB()` - Connect to database +4. `data.InitData()` - Initialize default data +5. `LoadStorages()` - Load and initialize all storage drivers +6. `InitTaskManager()` - Start background tasks +7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` - Feature not implemented +- `errs.ObjectNotFound` - File/folder not found +- `errs.NotFolder` - Path is not a directory +- `errs.StorageNotInit` - Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { + // Refresh link +} +``` + +### Saving Driver State + +When updating tokens or credentials: +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +Use `rate.Limiter` for API rate limits: +```go +type YourDriver struct { + limiter *rate.Limiter +} + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} + +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +Always respect context cancellation in long operations: +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +## Important Conventions + +**Naming**: +- Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (e.g., `internal/op`) +- Interfaces: PascalCase with suffix (e.g., `Reader`, `Writer`) + +**Driver Configuration Fields**: +- Use `driver.RootPath` or `driver.RootID` for root folder +- Add `omitempty` to optional JSON fields +- Use descriptive help text in struct tags + +**Retries and Timeouts**: +- Use `github.com/avast/retry-go` for retry logic +- Set reasonable timeouts on HTTP clients (default 30s in `base.RestyClient`) +- For unstable APIs, implement exponential backoff + +**Logging**: +- Use `logrus` via `log` package +- Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` +- Include driver name in logs: `log.Infof("[driver_name] message")` + +## Project Context + +OpenList is a community-driven fork of AList, focused on: +- Long-term governance and trust +- Support for 70+ cloud storage providers +- Web UI for file management +- Multi-protocol support (HTTP, WebDAV, FTP, SFTP, S3) +- Offline downloads (Aria2, Transmission) +- Full-text search +- Archive extraction + +**License**: AGPL-3.0 diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md new file mode 100644 index 0000000000..0b8be3b526 --- /dev/null +++ b/COMPATIBILITY_REPORT.md @@ -0,0 +1,204 @@ +# Rebase兼容性分析报告 + +## 提交概览 +共引入 **21个commits**，主要涉及以下模块： + +### 核心功能改动 + +#### 1. **链接刷新机制** (`internal/stream/util.go`) +**Commits**: +- `4c33ffa4` feat(link): add link refresh capability for expired download links +- `f38fe180` fix(stream): 修复链接过期检测逻辑，避免将上下文取消视为链接过期 +- `7cf362c6` fix(stream): 更新过期链接检查逻辑，支持所有4xx客户端错误 +- `03fbaf1c` refactor(stream): 移除过时的链接刷新逻辑，添加自愈读取器以处理0字节读取 + +**核心代码**: +```go +// 新增常量 +MAX_LINK_REFRESH_COUNT = 50 // 链接最大刷新次数 +MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead重试次数（从3提升到5） + +// 新增函数 +IsLinkExpiredError(err error) bool // 判断是否为链接过期错误 + +// 新增结构 +RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // 防止无限循环 +} + +selfHealingReadCloser struct { + // 检测0字节读取，自动刷新链接 +} +``` + +**功能说明**: +1. **链接过期检测**: 识别多种云盘的过期错误（expired, token expired, access denied, 4xx状态码等） +2. **自动刷新**: 检测到过期时自动调用Refresher获取新链接，最多刷新50次 +3. **自愈机制**: 处理某些云盘返回200但内容为空的情况（0字节读取检测） +4. **并发安全**: 使用sync.Mutex保护共享状态 +5. **Context隔离**: 刷新时使用WithoutCancel避免用户取消操作影响刷新 + +**潜在风险**: +- ✅ Context.WithoutCancel需要Go 1.21+ +- ✅ 并发场景下的锁竞争 +- ✅ refreshCount可能在某些场景下不递增导致无限循环 + +--- + +#### 2. **目录预创建优化** (`internal/fs/copy_move.go`) +**Commit**: `ce0da112` fix(copy_move): 将预创建子目录的深度从2级调整为1级 + +**核心代码**: +```go +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { + // 第一轮：创建直接子目录 + for _, obj := range objs { + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + op.MakeDir(t.Ctx(), t.DstStorage, subdirPath) + subdirs = append(subdirs, obj) + } + } + + // 停止递归条件 + if maxDepth <= 0 { + return nil + } + + // 第二轮：递归创建嵌套目录 + for _, subdir := range subdirs { + subObjs := op.List(...) + preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1) + } +} +``` + +**功能说明**: +1. **深度控制**: 默认maxDepth=1，只预创建2级目录（当前+子级） +2. **防止深度递归**: 避免在大型项目中递归过深导致栈溢出或性能问题 +3. **错误容忍**: MakeDir失败时继续处理其他目录 +4. **Context感知**: 每次循环检查ctx.Err()支持取消操作 + +**潜在风险**: +- ✅ op.MakeDir和op.List调用需要存储初始化 +- ✅ 大量目录时的性能问题 +- ✅ Context取消时的资源清理 + +--- + +#### 3. **网络优化** (`drivers/`, `internal/net/`) +**Commits**: +- `b9dafa65` feat(network): 增加对慢速网络的支持，调整超时和重试机制 +- `bce47884` fix(driver): 增加夸克分片大小调整逻辑，支持重试机制 +- `0b8471f6` feat(quark_open): 添加速率限制和重试逻辑 + +**功能说明**: +1. 提升RangeRead重试次数: 3 → 5 +2. 调整网络超时参数 +3. 添加分片上传重试逻辑 + +--- + +#### 4. **驱动修复** +**Commits**: +- `da2812c0` fix(google_drive): 更新Put方法以支持可重复读取流和不可重复读取流的MD5校验 +- `5a6bad90` feat(google_drive): 添加文件夹创建的锁机制和重试逻辑 +- `a54b2388` feat(google_drive): 添加处理重复文件名的功能 +- `9ef22ec9` fix(driver): fix file copy failure to 123pan due to incorrect etag +- `0ead87ef` fix(alias): update storage retrieval method in listRoot function +- `311f6246` fix: 修复500 panic和NaN问题 + +--- + +## 兼容性评估 + +### ✅ 编译兼容性 +- 构建成功，无语法错误 +- 依赖版本无冲突 + +### ✅ API兼容性 +- 新增函数不破坏现有接口 +- RefreshableRangeReader实现model.RangeReaderIF接口 +- 向后兼容旧代码 + +### ⚠️ 运行时兼容性 +**需要验证的场景**: +1. **并发安全**: RefreshableRangeReader的并发读取 +2. **资源泄漏**: Context取消时goroutine是否正确退出 +3. **边界条件**: + - refreshCount达到50次的行为 + - 0字节读取检测的准确性 + - maxDepth=0时的目录创建 +4. **错误处理**: + - nil Refresher时的处理 + - 链接刷新失败时的回退机制 +5. **性能**: + - 大文件下载时的刷新开销 + - 深层目录结构的预创建性能 + +--- + +## 测试需求 + +### 必须测试的场景 + +#### Stream包测试 +1. **IsLinkExpiredError准确性** + - 各种云盘的过期错误格式 + - Context取消不应判断为过期 + - HTTP 4xx/5xx的区分 + +2. **RefreshableRangeReader可靠性** + - 正常读取流程 + - 自动刷新触发和成功 + - 达到最大刷新次数 + - 并发读取安全性 + - Context取消的正确处理 + +3. **selfHealingReadCloser** + - 0字节读取检测 + - 刷新重试机制 + - 资源正确关闭 + +#### FS包测试 +1. **preCreateDirectoryTree** + - 深度控制正确性(0, 1, 2级) + - 大量目录的性能 + - Context取消的响应 + - 错误容忍性 + +--- + +## 风险等级: **中等** + +**原因**: +- ✅ 新功能设计合理，有明确的边界和错误处理 +- ⚠️ 并发场景需要充分测试 +- ⚠️ 链接刷新逻辑复杂，需要验证各种边界情况 +- ⚠️ 依赖op包的函数需要正确的初始化 + +--- + +## 推送建议: **通过测试后可推送** + +**前置条件**: +1. 完成全面的单元测试（见下方测试代码） +2. 验证并发安全性 +3. 确认Context取消不会导致资源泄漏 +4. 性能测试通过（大文件、深层目录） + +**建议测试命令**: +```bash +# 单元测试 +go test ./internal/stream ./internal/fs -v -count=1 -race + +# 压力测试 +go test ./internal/stream -run Stress -v -count=10 + +# 完整测试套件 +go test ./... -short -count=1 +``` diff --git a/go.mod b/go.mod index 9a254b99cd..10f893b0d4 100644 --- a/go.mod +++ b/go.mod @@ -312,6 +312,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.5 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index dcdfc512a3..9aa270844a 100644 --- a/go.sum +++ b/go.sum @@ -21,16 +21,14 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/Ironboxplus/115-sdk-go v0.2.5 h1:8giRpk9TwDT/5oe6F1H6h5+OwMMkM/vSYJqZomsmp20= +github.com/Ironboxplus/115-sdk-go v0.2.5/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= -github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= -github.com/OpenListTeam/115-sdk-go v0.2.4 h1:dVoEz3Pm996n/ZCuRckQvz36w938uNHjPrS7E/SVpBM= -github.com/OpenListTeam/115-sdk-go v0.2.4/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= From 0dce454e3566bcb611ea353f83e651823e5baab8 Mon Sep 17 00:00:00 2001 From: cyk Date: Fri, 8 May 2026 22:13:29 +0800 Subject: [PATCH 055/107] fix: bounded auth retry, mkdirLocks cleanup, EOF handling, tar size limit, dist swap lock, Go 1.26 vet - Google Drive Put: replace infinite 401 recursion with maxPutAuthRetries=2 - Google Drive MakeDir: delete mkdirLocks entry after unlock to prevent memory leak - selfHealingReadCloser: only reconnect on io.ErrUnexpectedEOF, not normal io.EOF - RefreshableRangeReader: document concurrency contract for innerReader replacement - Frontend extractTarGz: add maxExtractFileSize (500MB) and io.LimitReader - Frontend dist swap: add distSwapMu to protect rename window - Fix Go 1.26 non-constant format string vet errors across drivers --- drivers/123/util.go | 2 +- drivers/189/util.go | 2 +- drivers/189pc/utils.go | 4 +- drivers/chaoxing/driver.go | 4 +- drivers/google_drive/driver.go | 15 ++++- drivers/google_drive/driver_test.go | 71 +++++++++++++++++++++ drivers/google_drive/util.go | 4 +- drivers/google_photo/util.go | 2 +- drivers/lanzou/util.go | 2 +- internal/frontend/fetcher.go | 18 ++++-- internal/frontend/fetcher_test.go | 34 ++++++++++ internal/offline_download/115/client.go | 2 +- internal/offline_download/pikpak/pikpak.go | 2 +- internal/stream/util.go | 9 ++- internal/stream/util_test.go | 73 ++++++++++++++++++++++ 15 files changed, 222 insertions(+), 22 deletions(-) create mode 100644 drivers/google_drive/driver_test.go diff --git a/drivers/123/util.go b/drivers/123/util.go index 5cce49b65b..d53a3d9db8 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -181,7 +181,7 @@ func (d *Pan123) login() error { return err } if utils.Json.Get(res.Body(), "code").ToInt() != 200 { - err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString()) + err = fmt.Errorf("%s", utils.Json.Get(res.Body(), "message").ToString()) } else { d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString() } diff --git a/drivers/189/util.go b/drivers/189/util.go index a98b0d0161..5fad92c1dd 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -238,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } - log.Debugf(res.String()) + log.Debugf("%s", res.String()) return errors.New(res.String()) } diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index e863060afe..86cf834e4e 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -354,7 +354,7 @@ func (y *Cloud189PC) loginByPassword() (err error) { return &erron } if tokenInfo.ResCode != 0 { - err = fmt.Errorf(tokenInfo.ResMessage) + err = fmt.Errorf("%s", tokenInfo.ResMessage) return err } y.Addition.AccessToken = tokenInfo.AccessToken @@ -414,7 +414,7 @@ func (y *Cloud189PC) loginByQRCode() error { return err } if tokenInfo.ResCode != 0 { - return fmt.Errorf(tokenInfo.ResMessage) + return fmt.Errorf("%s", tokenInfo.ResMessage) } y.Addition.AccessToken = tokenInfo.AccessToken y.Addition.RefreshToken = tokenInfo.RefreshToken diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index dfd25d1957..54345ff125 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -55,13 +55,13 @@ func (d *ChaoXing) refreshCookie() error { func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } }) return nil diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 1e2e476ce5..0e1c04f46c 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -85,7 +85,10 @@ func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) lock := lockVal.(*sync.Mutex) lock.Lock() - defer lock.Unlock() + defer func() { + lock.Unlock() + mkdirLocks.Delete(lockKey) + }() // Check if folder already exists with retry to handle API eventual consistency escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") @@ -184,7 +187,13 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } +const maxPutAuthRetries = 2 + func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + return d.putWithRetry(ctx, dstDir, file, up, 0) +} + +func (d *GoogleDrive) putWithRetry(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, authRetries int) error { // 1. 准备MD5（用于完整性校验） md5Hash := file.GetHash().GetHash(utils.MD5) @@ -255,12 +264,12 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.File return err } if e.Error.Code != 0 { - if e.Error.Code == 401 { + if e.Error.Code == 401 && authRetries < maxPutAuthRetries { err = d.refreshToken() if err != nil { return err } - return d.Put(ctx, dstDir, file, up) + return d.putWithRetry(ctx, dstDir, file, up, authRetries+1) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } diff --git a/drivers/google_drive/driver_test.go b/drivers/google_drive/driver_test.go new file mode 100644 index 0000000000..b98bdcf146 --- /dev/null +++ b/drivers/google_drive/driver_test.go @@ -0,0 +1,71 @@ +package google_drive + +import ( + "sync" + "testing" +) + +func TestMaxPutAuthRetriesIsBounded(t *testing.T) { + if maxPutAuthRetries < 1 { + t.Fatalf("maxPutAuthRetries=%d, must be >= 1", maxPutAuthRetries) + } + if maxPutAuthRetries > 5 { + t.Fatalf("maxPutAuthRetries=%d, must be <= 5 to prevent excessive retries", maxPutAuthRetries) + } +} + +func TestMkdirLocksCleanedUpAfterUse(t *testing.T) { + // Reset state + mkdirLocks = sync.Map{} + + key1 := "parent-1/folder-a" + key2 := "parent-2/folder-b" + + // Simulate two MakeDir calls storing locks + mkdirLocks.LoadOrStore(key1, &sync.Mutex{}) + mkdirLocks.LoadOrStore(key2, &sync.Mutex{}) + + // Both should exist + if _, ok := mkdirLocks.Load(key1); !ok { + t.Fatal("key1 should exist") + } + if _, ok := mkdirLocks.Load(key2); !ok { + t.Fatal("key2 should exist") + } + + // After MakeDir completes, locks should be cleaned up + mkdirLocks.Delete(key1) + mkdirLocks.Delete(key2) + + if _, ok := mkdirLocks.Load(key1); ok { + t.Fatal("key1 should be deleted after cleanup") + } + if _, ok := mkdirLocks.Load(key2); ok { + t.Fatal("key2 should be deleted after cleanup") + } +} + +func TestMkdirLocksNoCrossContamination(t *testing.T) { + mkdirLocks = sync.Map{} + + key := "parent/shared-folder" + lockVal, _ := mkdirLocks.LoadOrStore(key, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + + // Simulate concurrent access: lock should be shared for same key + lockVal2, loaded := mkdirLocks.LoadOrStore(key, &sync.Mutex{}) + if !loaded { + t.Fatal("second LoadOrStore should return existing entry") + } + lock2 := lockVal2.(*sync.Mutex) + if lock != lock2 { + t.Fatal("same key should return same mutex instance") + } + + // Different key should get different lock + lockVal3, _ := mkdirLocks.LoadOrStore("other-parent/other-folder", &sync.Mutex{}) + lock3 := lockVal3.(*sync.Mutex) + if lock == lock3 { + t.Fatal("different keys should have different mutex instances") + } +} diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 1fc68c3359..53695deee0 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 3a9b66ab2e..81d05b4929 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error { return err } if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 9a15d428e5..37844500ea 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -91,7 +91,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, if info == "" { info = utils.Json.Get(data, "info").ToString() } - return data, fmt.Errorf(info) + return data, fmt.Errorf("%s", info) } } diff --git a/internal/frontend/fetcher.go b/internal/frontend/fetcher.go index 363d5dfcae..b087615786 100644 --- a/internal/frontend/fetcher.go +++ b/internal/frontend/fetcher.go @@ -24,6 +24,7 @@ const ( defaultFrontendRepo = "OpenListTeam/OpenList-Frontend" versionFile = ".frontend_version" distDirName = "dist" + maxExtractFileSize = 500 * 1024 * 1024 // 500MB per file ) // FetchResult contains the result of a fetch operation @@ -333,20 +334,20 @@ func downloadAndExtract(ctx context.Context, client *http.Client, url string) er srcDir = filepath.Join(tmpDir, distDirName) } - // Atomic swap: rename source to final + // Atomic swap: hold lock to minimize the window where dist is absent finalDir := filepath.Join(destDir, distDirName) oldDir := filepath.Join(destDir, distDirName+".old") - // Remove previous backup if exists + + distSwapMu.Lock() os.RemoveAll(oldDir) - // Move current dist out of the way if it exists os.Rename(finalDir, oldDir) - // Move new dist into place if err := os.Rename(srcDir, finalDir); err != nil { - // Rollback os.RemoveAll(finalDir) os.Rename(oldDir, finalDir) + distSwapMu.Unlock() return fmt.Errorf("rename new dist: %w", err) } + distSwapMu.Unlock() os.RemoveAll(oldDir) return nil @@ -388,6 +389,9 @@ func extractTarGz(r io.Reader, dest string) error { return err } case tar.TypeReg: + if hdr.Size > maxExtractFileSize { + return fmt.Errorf("file too large: %s (%d bytes)", hdr.Name, hdr.Size) + } if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err } @@ -395,7 +399,7 @@ func extractTarGz(r io.Reader, dest string) error { if err != nil { return err } - if _, err := io.Copy(f, tr); err != nil { + if _, err := io.Copy(f, io.LimitReader(tr, maxExtractFileSize)); err != nil { f.Close() return err } @@ -469,6 +473,8 @@ func init() { _ = os.MkdirAll(GetDistPath(), 0755) } +var distSwapMu sync.Mutex + // Ensure that the sync.Once pattern is used for the fetcher var ( fetchMu sync.Mutex diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go index 14f7a33913..bae99f47fb 100644 --- a/internal/frontend/fetcher_test.go +++ b/internal/frontend/fetcher_test.go @@ -115,6 +115,40 @@ func TestExtractTarGzPathTraversal(t *testing.T) { } } +func TestExtractTarGzRejectsOversizedFile(t *testing.T) { + tmpDir := t.TempDir() + // Create a tar.gz with a file whose header claims a size exceeding the limit + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + hdr := &tar.Header{ + Name: "dist/huge.bin", + Mode: 0644, + Size: maxExtractFileSize + 1, + } + _ = tw.WriteHeader(hdr) + // Write just enough to pass; the size check should reject before reading + buf := make([]byte, 1024) + for written := int64(0); written < hdr.Size; written += int64(len(buf)) { + n := min(int64(len(buf)), hdr.Size-written) + _, _ = tw.Write(buf[:n]) + } + }() + data, _ := io.ReadAll(pr) + + err := extractTarGz(strings.NewReader(string(data)), tmpDir) + if err == nil { + t.Fatal("expected error for oversized file, got nil") + } + if !strings.Contains(err.Error(), "too large") { + t.Errorf("expected 'too large' error, got: %v", err) + } +} + func TestHasValidDist(t *testing.T) { if HasValidDist() { t.Log("HasValidDist returned true (may have existing dist from previous runs)") diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index d31971d7cb..c026ee7eee 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -125,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index 8a6693091f..e223937e7c 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -127,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { - s.Err = fmt.Errorf(t.Message) + s.Err = fmt.Errorf("%s", t.Message) } return s, nil } diff --git a/internal/stream/util.go b/internal/stream/util.go index a10772d54e..06a754e86a 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -110,6 +110,13 @@ func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { return reader, nil } +// RangeRead obtains a reader reference under lock, then issues the range +// request outside the lock. The local copy of `reader` remains valid even +// if a concurrent refresh nils r.innerReader and replaces r.link, because +// doRefreshLocked only clears the pointer — it does not close or invalidate +// the old reader object. Each reader is an independent RangeReaderFunc +// (closure over a URL), so stale readers simply use the old URL; if it has +// expired, selfHealingReadCloser handles the retry transparently. func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { r.mu.Lock() reader, err := r.getInnerReader() @@ -240,7 +247,7 @@ func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n in return true } - if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + if errors.Is(err, io.ErrUnexpectedEOF) { log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) return true } diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go index 6dc4c8a09c..b739802717 100644 --- a/internal/stream/util_test.go +++ b/internal/stream/util_test.go @@ -116,6 +116,79 @@ func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t * } } +// TestSelfHealingReadCloser_NormalEOFDoesNotTriggerReconnect verifies that a +// legitimate io.EOF (all data delivered) does NOT trigger a link refresh. +func TestSelfHealingReadCloser_NormalEOFDoesNotTriggerReconnect(t *testing.T) { + data := []byte("hello world") + refreshes := 0 + + inner := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: inner} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: inner}, nil, nil + } + + rrr := NewRefreshableRangeReader(link, int64(len(data))) + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead error: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("got %q, want %q", got, data) + } + if refreshes != 0 { + t.Fatalf("refreshes = %d, want 0 (normal EOF should not trigger refresh)", refreshes) + } +} + +// TestSelfHealingReadCloser_UnexpectedEOFTriggersReconnect verifies that +// io.ErrUnexpectedEOF (stream interrupted) DOES trigger reconnect. +func TestSelfHealingReadCloser_UnexpectedEOFTriggersReconnect(t *testing.T) { + data := []byte("0123456789") + refreshes := 0 + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 4, io.ErrUnexpectedEOF), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + rrr := NewRefreshableRangeReader(link, int64(len(data))) + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead error: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("got %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } +} + type flakyReadCloser struct { data []byte failAfter int From 2d888967308fabfa19779e6f797032b104c50560 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 9 May 2026 11:58:37 +0800 Subject: [PATCH 056/107] fix(115_open): handle OSS upload timeout and PartAlreadyExist retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NewOSSUploadHttpClient: ResponseHeaderTimeout 15s → 5min for uploads - Handle PartAlreadyExist (409) in retry by recovering part via ListUploadedParts instead of failing, fixes timeout-then-retry conflict with oss.Sequential() --- drivers/115_open/upload.go | 25 ++++++++++++++++++++++++- drivers/115_open/upload_test.go | 29 +++++++++++++++++++++++++++++ internal/net/oss.go | 31 +++++++++++++++++++++++++++++-- internal/net/oss_test.go | 15 +++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 drivers/115_open/upload_test.go diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 6af4403cf1..7ee755e747 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/base64" + "fmt" "io" "strings" "time" @@ -28,6 +29,14 @@ func isTokenExpiredError(err error) bool { strings.Contains(errStr, "InvalidAccessKeyId") } +// isPartAlreadyExistError 检测是否为分片已存在错误（超时后重试时 OSS 返回 409） +func isPartAlreadyExistError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "PartAlreadyExist") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -129,6 +138,21 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, rd.Seek(0, io.SeekStart) part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) if err != nil { + if isPartAlreadyExistError(err) { + log.Infof("115 OSS part %d already exists, retrieving from ListUploadedParts", i) + lpr, listErr := bucket.ListUploadedParts(imur) + if listErr != nil { + return fmt.Errorf("part %d already exists but ListUploadedParts failed: %w", i, listErr) + } + for _, p := range lpr.UploadedParts { + if p.PartNumber == int(i) { + parts[i-1] = oss.UploadPart{PartNumber: p.PartNumber, ETag: p.ETag} + log.Infof("115 OSS part %d recovered: ETag=%s", i, p.ETag) + return nil + } + } + return fmt.Errorf("part %d reported as existing but not found in ListUploadedParts", i) + } return err } parts[i-1] = part @@ -139,7 +163,6 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), retry.OnRetry(func(n uint, err error) { - // 如果是凭证过期错误，在重试前刷新凭证并重建bucket if isTokenExpiredError(err) { log.Warnf("115 OSS token expired, refreshing token...") if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { diff --git a/drivers/115_open/upload_test.go b/drivers/115_open/upload_test.go new file mode 100644 index 0000000000..5ad2ff1a06 --- /dev/null +++ b/drivers/115_open/upload_test.go @@ -0,0 +1,29 @@ +package _115_open + +import "testing" + +func TestIsPartAlreadyExistError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"generic error", errStr("some random error"), false}, + {"timeout", errStr("net/http: timeout awaiting response headers"), false}, + {"part already exist", errStr(`oss: service returned error: StatusCode=409, ErrorCode=PartAlreadyExist, ErrorMessage="For sequential multipart upload, you can't overwrite uploaded parts."`), true}, + {"partial match", errStr("PartAlreadyExist"), true}, + {"case sensitive", errStr("partalreadyexist"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isPartAlreadyExistError(tt.err); got != tt.want { + t.Errorf("isPartAlreadyExistError() = %v, want %v", got, tt.want) + } + }) + } +} + +type errStr string + +func (e errStr) Error() string { return string(e) } diff --git a/internal/net/oss.go b/internal/net/oss.go index a897161f14..b28e06bc4a 100644 --- a/internal/net/oss.go +++ b/internal/net/oss.go @@ -1,9 +1,36 @@ package net -import "github.com/aliyun/aliyun-oss-go-sdk/oss" +import ( + "crypto/tls" + stdnet "net" + "net/http" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) func NewOSSClient(endpoint, accessKeyID, accessKeySecret string, options ...oss.ClientOption) (*oss.Client, error) { - clientOptions := []oss.ClientOption{oss.HTTPClient(NewHttpClient())} + clientOptions := []oss.ClientOption{oss.HTTPClient(NewOSSUploadHttpClient())} clientOptions = append(clientOptions, options...) return oss.New(endpoint, accessKeyID, accessKeySecret, clientOptions...) } + +func NewOSSUploadHttpClient() *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + DialContext: (&stdnet.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ResponseHeaderTimeout: 5 * time.Minute, + } + + SetProxyIfConfigured(transport) + + return &http.Client{ + Timeout: time.Hour * 48, + Transport: transport, + } +} diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go index 9001cd39d3..13a4f86fe7 100644 --- a/internal/net/oss_test.go +++ b/internal/net/oss_test.go @@ -8,6 +8,21 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" ) +func TestNewOSSUploadHttpClientHasLongerTimeout(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + defer func() { conf.Conf = oldConf }() + + client := NewOSSUploadHttpClient() + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", client.Transport) + } + if transport.ResponseHeaderTimeout < 120_000_000_000 { // 2 minutes minimum + t.Fatalf("ResponseHeaderTimeout=%v, want >= 2m for upload", transport.ResponseHeaderTimeout) + } +} + func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { oldConf := conf.Conf conf.Conf = conf.DefaultConfig("data") From 7e8351f7d8423de13922a92c48afab385545b4be Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 9 May 2026 12:22:21 +0800 Subject: [PATCH 057/107] docs: add OVERVIEW.md project index and JOURNAL.md development log --- JOURNAL.md | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++++ OVERVIEW.md | 79 +++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 JOURNAL.md create mode 100644 OVERVIEW.md diff --git a/JOURNAL.md b/JOURNAL.md new file mode 100644 index 0000000000..91becbf1cb --- /dev/null +++ b/JOURNAL.md @@ -0,0 +1,131 @@ +# Development Journal + +Chronological log of all changes in this fork, from earliest to latest. + +--- + +## 2026-04-25 — Initial Feature Batch (rebased onto up/main) + +### `17be63fb` feat(stream): link refresh, self-healing reader, seekable prefetch, and upload hash rework +- Added `RefreshableRangeReader`: wraps `RangeReader` with automatic link refresh on expiry (max 50 attempts) +- Added `selfHealingReadCloser`: detects 0-byte reads, connection resets, and `io.ErrUnexpectedEOF`; reconnects from current offset transparently +- Added `IsLinkExpiredError()`: checks error strings + HTTP 4xx status codes for link expiry +- Added 2-window async prefetch in `directSectionReader`: while uploading chunk N, prefetch chunk N+1 via goroutine +- Reworked `StreamHashFile` to use `RangeRead` for `SeekableStream` (never consumes `Reader`) +- Added `ReadFullWithRangeRead` with retry (max 5 attempts, 1-5s backoff) +- Files: `internal/stream/util.go`, `internal/stream/stream.go` + +### `1db9136f` feat(google_drive): duplicate filename handling, folder lock, retry, and MD5 checksum +- Added `mkdirLocks` (`sync.Map`) to prevent concurrent creation of duplicate folders +- Added existence check with retry before folder creation +- Added 500ms consistency delay after folder creation +- Added MD5 hash computation for upload integrity (via `StreamHashFile`) +- Added `chunkUpload` with retry and `RangeRead`-based streaming +- Files: `drivers/google_drive/driver.go`, `drivers/google_drive/util.go` + +### `f9bc1567` feat(115_open): permanent delete, proxy_range, offline task fixes, and error handling +- Added `RemoveWay` config option: "trash" (default) or "delete" (permanent) +- Implemented `removePermanently`: deletes from recycle bin after trash +- Added `findRecycleBinEntry` with paginated recycle bin search +- Added `matchRecycleBinEntry` with multi-strategy matching (ID → SHA1 → name+size) +- Added `findRecycleBinEntryWithRetry` (4 attempts, 300ms backoff) for eventual consistency +- Added `FlexString` CID handling (numeric/string JSON interop) +- Added `proxy_range` option exposure +- Files: `drivers/115_open/driver.go`, `drivers/115_open/meta.go`, `drivers/115_open/upload.go`, `drivers/115_open/driver_test.go` + +### `f1493fc9` feat(offline_download): multi-page task retrieval and task limit wait mechanism +- 115 `OfflineList` now paginates through all pages (was only page 1) +- Added task limit wait mechanism in offline download client +- Moved 115 offline task cleanup from `Run()` to `Update()` so it runs even if transfer fails +- Files: `internal/offline_download/115_open/client.go`, `internal/offline_download/tool/download.go` + +### `35130094` feat(drivers): baidu streaming upload, quark rate-limit/retry, 123pan etag fix +- **Baidu Netdisk**: extracted upload logic to `upload.go`, full streaming upload support +- **Quark Open**: added rate limiter, retry with chunk size adjustment on 413 error +- **123 Open**: fixed copy failure due to incorrect etag, added SHA1 rapid upload +- Normalized `StreamHashFile` progress weight to 100 across all drivers +- Files: `drivers/baidu_netdisk/upload.go`, `drivers/quark_open/driver.go`, `drivers/123_open/driver.go` + +### `7787ddbc` fix(core): copy_move depth, alias storage retrieval, sftp symlink, 500 panic +- Fixed `preCreateDirectoryTree` depth from 2 to 1 to avoid deep recursion +- Fixed alias storage retrieval method in `listRoot` +- Fixed SFTP symlink handling +- Fixed 500 panic and NaN issues +- Files: `internal/fs/copy_move.go`, `drivers/alias/util.go`, `drivers/sftp/types.go` + +### `4e735df0` feat(frontend): dynamic frontend fetching, CI upgrades, and build infrastructure +- Added `internal/frontend/fetcher.go`: auto-download frontend dist from GitHub releases +- Added `internal/frontend/watcher.go`: periodic check (30min) for new versions +- Added `FrontendRepoDefault` ldflags variable for build-time frontend repo injection +- CI: action version upgrades, frontend caching, version matrix builds +- `build.sh`: frontend repo configurable via `FRONTEND_REPO` env var +- Files: `internal/frontend/`, `.github/workflows/`, `build.sh`, `internal/conf/`, `server/static/` + +### `0bf12c5c` chore: add project docs and update dependencies (115-sdk-go fork) +- Added `CLAUDE.md` with comprehensive project guidance +- Added `COMPATIBILITY_REPORT.md` for 115-sdk-go fork analysis +- Updated 115-sdk-go dependency to `v0.2.5` (FlexString CID support) +- Files: `CLAUDE.md`, `COMPATIBILITY_REPORT.md`, `go.mod`, `go.sum` + +--- + +## 2026-05-08 — Rebase onto upstream + Code Review Fixes + +### Rebase onto `up/main` @ `c7c0cfae` +- `feat/dynamic-frontend` cleanly rebased (8 commits, 0 conflicts) +- Upstream additions included: path validation, SplitSeq perf, ObjectAlreadyExists check, qBittorrent login fix, custom share IDs, Getter interfaces (webdav/s3/115_open), about page logo fix + +### `0369bc0a` fix: bounded auth retry, mkdirLocks cleanup, EOF handling, tar size limit, dist swap lock, Go 1.26 vet +Code review identified 16 issues (2 CRITICAL, 5 HIGH). Fixed 6: +- **Google Drive Put 401 recursion** (CRITICAL): replaced infinite recursive `d.Put()` call with `putWithRetry()` + `maxPutAuthRetries=2` +- **mkdirLocks memory leak** (HIGH): added `defer mkdirLocks.Delete(lockKey)` after unlock +- **selfHealingReadCloser EOF** (HIGH): removed `io.EOF` from reconnect trigger, kept only `io.ErrUnexpectedEOF` +- **Frontend tar extraction** (HIGH): added `maxExtractFileSize=500MB` + `io.LimitReader` + `hdr.Size` check +- **Frontend dist swap TOCTOU** (HIGH): added `distSwapMu` mutex around rename window +- **RefreshableRangeReader concurrency** (CRITICAL): documented that local `reader` copy is safe after `innerReader` replacement +- **Go 1.26 vet**: fixed `fmt.Errorf` non-constant format strings across 8 drivers/packages +- Added tests: `drivers/google_drive/driver_test.go`, stream EOF tests, frontend oversized file test +- Files: 14 files changed, +222/-22 + +--- + +## 2026-05-09 — OSS Upload Fix + Hash Prefetch + +### `214881b3` fix(115_open): handle OSS upload timeout and PartAlreadyExist retry +- **Root cause**: `ResponseHeaderTimeout=15s` in shared `NewHttpClient()` was too short for uploading 20MB OSS parts. Timeout caused part to be uploaded but unconfirmed; retry hit `PartAlreadyExist` (409). +- **Fix 1**: Created `NewOSSUploadHttpClient()` with `ResponseHeaderTimeout=5min` dedicated to OSS uploads +- **Fix 2**: In `multpartUpload` retry, detect `PartAlreadyExist` → call `ListUploadedParts` to recover the part's ETag → treat as success +- Added `isPartAlreadyExistError()` helper +- Tests: `upload_test.go` (PartAlreadyExist detection), `oss_test.go` (upload client timeout) +- Files: `drivers/115_open/upload.go`, `internal/net/oss.go` + +### `94591821` perf(stream): add double-buffer prefetch to StreamHashFile for SeekableStream +- **Before**: hash calculation read 10MB chunks sequentially (network idle during hash computation) +- **After**: while hashing chunk N, goroutine prefetches chunk N+1 via `ReadFullWithRangeRead` +- Extracted `streamHashSeekableWithPrefetch()` with double-buffering pattern +- Hash values identical — no change to upload flow or rapid-upload logic +- `FileStream` path unchanged (sequential read) +- Test: `TestStreamHashFile_SeekablePrefetchProducesSameHash` +- Files: `internal/stream/util.go`, `internal/stream/util_test.go` + +--- + +## Architecture Notes + +### Upload Data Flow (Current) +``` +Pass 1: Hash Calculation (with prefetch) + StreamHashFile → ReadFullWithRangeRead + chunk N: hash computation (CPU) + chunk N+1: async prefetch (network I/O) ← overlapped + +Pass 2: Multipart Upload (with prefetch) + directSectionReader.GetSectionReader + chunk N: upload to cloud (network I/O) + chunk N+1: async prefetch (network I/O) ← overlapped +``` + +### Proxy Architecture +- `conf.Conf.ProxyAddress` → global HTTP proxy for all server-side traffic +- Per-storage `WebProxy` / `DownProxyURL` → browser download only, NOT copy tasks +- OSS uploads use dedicated `NewOSSUploadHttpClient()` with longer timeouts diff --git a/OVERVIEW.md b/OVERVIEW.md new file mode 100644 index 0000000000..01c71fb63d --- /dev/null +++ b/OVERVIEW.md @@ -0,0 +1,79 @@ +# OpenList Fork — Project Overview + +This is a fork of [OpenListTeam/OpenList](https://github.com/OpenListTeam/OpenList) (upstream), maintained at [Ironboxplus/OpenList](https://github.com/Ironboxplus/OpenList). + +## Branch Structure + +| Branch | Purpose | +|--------|---------| +| `feat/dynamic-frontend` | **Active development branch** (default). Rebased on `up/main`. | +| `copy` | Legacy branch with unsquashed commits. Superseded by `feat/dynamic-frontend`. | +| `main` | Synced from upstream via rebase workflow. | + +## Documentation Index + +| File | Description | +|------|-------------| +| [CLAUDE.md](CLAUDE.md) | AI coding guidance: architecture, driver system, stream/upload internals, conventions | +| [COMPATIBILITY_REPORT.md](COMPATIBILITY_REPORT.md) | 115-sdk-go fork compatibility analysis | +| [OVERVIEW.md](OVERVIEW.md) | This file — project index and high-level map | +| [JOURNAL.md](JOURNAL.md) | Chronological development log with all changes | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Upstream contribution guidelines | +| [README.md](README.md) | Upstream project README | +| [SECURITY.md](SECURITY.md) | Security policy | + +## Key Subsystems Modified (vs Upstream) + +### 1. Full-Streaming Upload with Async Prefetch +- **Files**: `internal/stream/util.go`, `internal/stream/stream.go` +- Two-pass flow: hash calculation (with double-buffer prefetch) → multipart upload (with 2-window async prefetch) +- `SeekableStream` uses `RangeRead` exclusively, never consumes the `Reader` +- `selfHealingReadCloser`: transparent link refresh + reconnect-from-offset on stream interruption +- `RefreshableRangeReader`: auto-refresh expired download links (up to 50 retries) + +### 2. Dynamic Frontend Fetcher +- **Files**: `internal/frontend/fetcher.go`, `internal/frontend/watcher.go` +- Auto-downloads frontend dist from GitHub releases on startup (when `WebVersion` is rolling/beta/dev) +- `Watcher` polls every 30 min for new versions, hot-swaps dist directory +- Configurable `FrontendRepo` (default: `OpenListTeam/OpenList-Frontend`, overridable per deployment) +- `FrontendRepoDefault` injectable via ldflags at build time + +### 3. Driver Enhancements + +| Driver | Changes | +|--------|---------| +| 115 Open | Permanent delete with recycle-bin retry, FlexString CID, OSS upload timeout fix, PartAlreadyExist recovery, proxy_range, offline task multi-page | +| Google Drive | Duplicate filename handling, per-folder MakeDir lock, bounded 401 retry, MD5 checksum, mkdirLocks cleanup | +| Baidu Netdisk | Full streaming upload (extracted from monolithic driver) | +| Quark Open | Rate limiting, retry with chunk size adjustment | +| 123 Open | SHA1 rapid upload, etag fix, StreamHashFile progress normalization | +| Aliyundrive Open | StreamHashFile progress normalization | + +### 4. CI/CD & Build +- **Files**: `.github/workflows/`, `build.sh` +- Frontend version matrix (rolling + latest) for Docker builds +- `FrontendRepoDefault` x-flag in CI +- Action version upgrades (checkout v6, setup-go v6, cache v5) +- Frontend caching in CI to avoid redundant downloads +- Static linking verification for musl builds + +### 5. Offline Download +- **Files**: `internal/offline_download/115_open/`, `internal/offline_download/tool/` +- Multi-page task retrieval for 115 +- Task limit wait mechanism +- Cleanup moved to `Update()` to ensure it runs even if transfer fails + +## Upstream Sync + +Remote `up` points to `https://github.com/OpenListTeam/OpenList.git`. + +```bash +git fetch up +git rebase up/main # on feat/dynamic-frontend +``` + +Current base: `up/main` @ `c7c0cfae` (2026-05-06) + +## Global Proxy + +All server-side HTTP traffic (driver API calls, copy/move transfers, frontend fetching) uses `conf.Conf.ProxyAddress` (global setting). Per-storage `WebProxy`/`DownProxyURL` only affects browser-facing download behavior, NOT server-side copy tasks. From 50d5a1c9db413784f5231f0235855bd4c0d30590 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 10 May 2026 15:01:50 +0800 Subject: [PATCH 058/107] fix(115_open): rate-limit every SDK call in Put method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Put() only called WaitLimit once at entry, but then fired 3-4 SDK requests (UploadInit ×3, UploadGetToken) without rate limiting. Under concurrent copy tasks this overwhelmed the 115 API, causing state:false/code:0 rejections on all requests including List/MakeDir. Now each SDK call in Put is individually rate-limited, matching the pattern used by List, MakeDir, Move, etc. --- drivers/115_open/driver.go | 18 ++-- drivers/115_open/driver_test.go | 152 ++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 5 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 36cc469ba2..ee4c09c479 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -459,11 +459,7 @@ func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *s } func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - err := d.WaitLimit(ctx) - if err != nil { - return err - } - + var err error sha1 := file.GetHash().GetHash(utils.SHA1) sha1128k := file.GetHash().GetHash(utils.SHA1_128K) @@ -472,6 +468,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre // 如果有预计算的 hash，先尝试秒传 if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + if err := d.WaitLimit(ctx); err != nil { + return err + } resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -554,6 +553,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } // 1. Init（SeekableStream 或已缓存的 FileStream） + if err := d.WaitLimit(ctx); err != nil { + return err + } resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -587,6 +589,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } + if err := d.WaitLimit(ctx); err != nil { + return err + } resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -605,6 +610,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } // 3. get upload token + if err := d.WaitLimit(ctx); err != nil { + return err + } tokenResp, err := d.client.UploadGetToken(ctx) if err != nil { return err diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 1d96646952..bdcf4e0423 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/json" + "io" "net/http" "net/http/httptest" "net/url" @@ -13,12 +14,17 @@ import ( "time" sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "golang.org/x/time/rate" ) type recordedRequest struct { Path string Form url.Values + Time time.Time } type rewriteTransport struct { @@ -179,6 +185,7 @@ func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) requests = append(requests, recordedRequest{ Path: r.URL.Path, Form: cloneValues(r.Form), + Time: time.Now(), }) mu.Unlock() responder(w, r) @@ -637,3 +644,148 @@ func TestOpen115RemoveDeleteStopsRetryWhenContextCancelled(t *testing.T) { t.Fatalf("expected context deadline exceeded, got: %v", err) } } + +// --- Put rate-limiting tests --- + +// mockFileStreamer satisfies model.FileStreamer with pre-computed hashes for testing. +type mockFileStreamer struct { + name string + size int64 + hashInfo utils.HashInfo + data []byte +} + +func (m *mockFileStreamer) Read(p []byte) (int, error) { return 0, io.EOF } +func (m *mockFileStreamer) Close() error { return nil } +func (m *mockFileStreamer) Add(_ io.Closer) {} +func (m *mockFileStreamer) AddIfCloser(_ any) {} +func (m *mockFileStreamer) GetSize() int64 { return m.size } +func (m *mockFileStreamer) GetName() string { return m.name } +func (m *mockFileStreamer) ModTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) CreateTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) IsDir() bool { return false } +func (m *mockFileStreamer) GetHash() utils.HashInfo { return m.hashInfo } +func (m *mockFileStreamer) GetID() string { return "" } +func (m *mockFileStreamer) GetPath() string { return "" } +func (m *mockFileStreamer) GetMimetype() string { return "application/octet-stream" } +func (m *mockFileStreamer) NeedStore() bool { return false } +func (m *mockFileStreamer) IsForceStreamUpload() bool { return false } +func (m *mockFileStreamer) GetExist() model.Obj { return nil } +func (m *mockFileStreamer) SetExist(_ model.Obj) {} +func (m *mockFileStreamer) GetFile() model.File { return nil } +func (m *mockFileStreamer) RangeRead(_ http_range.Range) (io.Reader, error) { + return strings.NewReader(string(m.data)), nil +} +func (m *mockFileStreamer) CacheFullAndWriter(_ *model.UpdateProgress, _ io.Writer) (model.File, error) { + return nil, nil +} + +func newTestOpen115WithRateLimit(t *testing.T, limitRate float64, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + driver, requests := newTestOpen115(t, "trash", responder) + driver.limiter = rate.NewLimiter(rate.Limit(limitRate), 1) + return driver, requests +} + +func TestPutRateLimitsEverySDKCall(t *testing.T) { + // Rate limit: 10 req/s → each WaitLimit blocks ~100ms + const limitRate = 10.0 + + driver, requests := newTestOpen115WithRateLimit(t, limitRate, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/upload/init": + // First call: status=1 (not rapid), second call: status=2 (rapid success) + if r.FormValue("sign_key") != "" { + writeSDKSuccess(t, w, map[string]any{"status": 2}) + } else { + writeSDKSuccess(t, w, map[string]any{ + "status": 7, + "sign_key": "test-key", + "sign_check": "0-10", + }) + } + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + stream := &mockFileStreamer{ + name: "test.txt", + size: 100, + hashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + utils.SHA1_128K: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }), + data: make([]byte, 100), + } + dstDir := &model.Object{ID: "0", Name: "root", IsFolder: true} + up := func(float64) {} + + start := time.Now() + err := driver.Put(context.Background(), dstDir, stream, up) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + + reqs := requests() + // Expect: pre-hash UploadInit + main UploadInit + sign-check UploadInit = 3 calls + if len(reqs) < 3 { + t.Fatalf("expected at least 3 requests, got %d", len(reqs)) + } + assertRequestPaths(t, reqs, "/open/upload/init", "/open/upload/init", "/open/upload/init") + + // With 2 SDK calls each preceded by WaitLimit(10/s), minimum elapsed is ~100ms. + // Without WaitLimit, both calls fire instantly (<10ms). + minExpected := time.Duration(float64(time.Second) / limitRate * float64(len(reqs)-1)) + tolerance := minExpected * 7 / 10 // 70% to account for timing jitter + if elapsed < tolerance { + t.Fatalf("Put completed too fast (%v), expected at least %v — WaitLimit likely missing before some SDK calls", elapsed, tolerance) + } + + // Also verify individual request spacing + for i := 1; i < len(reqs); i++ { + gap := reqs[i].Time.Sub(reqs[i-1].Time) + gapMin := time.Duration(float64(time.Second) / limitRate * 0.7) + if gap < gapMin { + t.Fatalf("gap between request %d and %d is %v, expected at least %v — WaitLimit missing", i-1, i, gap, gapMin) + } + } +} + +func TestPutRateLimitsPreHashPath(t *testing.T) { + const limitRate = 10.0 + + driver, requests := newTestOpen115WithRateLimit(t, limitRate, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/upload/init": + // Rapid upload success on first try + writeSDKSuccess(t, w, map[string]any{"status": 2}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + stream := &mockFileStreamer{ + name: "test.txt", + size: 100, + hashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + utils.SHA1_128K: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }), + data: make([]byte, 100), + } + dstDir := &model.Object{ID: "0", Name: "root", IsFolder: true} + + err := driver.Put(context.Background(), dstDir, stream, func(float64) {}) + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + + reqs := requests() + if len(reqs) != 1 { + t.Fatalf("expected 1 request, got %d: %v", len(reqs), reqs) + } + assertRequestPaths(t, reqs, "/open/upload/init") +} From eb540734cf1b792e7a58a4bd4e2a97854682fe3a Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 10 May 2026 19:55:52 +0800 Subject: [PATCH 059/107] fix(deps): bump 115-sdk-go to v0.2.6 for concurrent refresh fix The SDK's authRequest had no concurrency protection on token refresh. Multiple goroutines hitting 401 simultaneously would all call RefreshToken independently, causing "refresh frequently" errors and corrupting the stored tokens. v0.2.6 adds mutex with double-check pattern to ensure only one goroutine refreshes at a time. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10f893b0d4..6a2495e6af 100644 --- a/go.mod +++ b/go.mod @@ -312,6 +312,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.5 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.6 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 9aa270844a..fa018c70ae 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/Ironboxplus/115-sdk-go v0.2.5 h1:8giRpk9TwDT/5oe6F1H6h5+OwMMkM/vSYJqZomsmp20= -github.com/Ironboxplus/115-sdk-go v0.2.5/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.6 h1:xsb7MYOf4IVszL4XqHKTAvpgZAmdr+jMMw3XOC955zQ= +github.com/Ironboxplus/115-sdk-go v0.2.6/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From e5f6a33dc1e6b8a5ac83ee712c66ba3c6a1797d2 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 11 May 2026 12:43:32 +0800 Subject: [PATCH 060/107] fix(115_open): handle empty data response for non-existent paths 115 API returns {state:true, data:[]} for non-existent paths instead of an error code. SDK v0.2.8 adds ErrDataEmpty sentinel; driver.Get() maps it to ObjectNotFound so MakeDir can create directories normally. --- JOURNAL.md | 88 +++++++++++++++++++++++++++++++++ drivers/115_open/driver.go | 2 +- drivers/115_open/driver_test.go | 37 ++++++++++++++ go.mod | 2 +- go.sum | 4 +- 5 files changed, 129 insertions(+), 4 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 91becbf1cb..ff58ac2369 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -110,6 +110,94 @@ Code review identified 16 issues (2 CRITICAL, 5 HIGH). Fixed 6: --- +## 2026-05-10 — 115 Rate Limiting + Concurrent Token Refresh Fix + +### 问题现象 + +复制任务（`/scnet/` → `/storage/my_115/`）全部失败，错误 `code: 0, message:` 和 `code: 40100000, message: 参数错误！`。目录确实存在，单任务正常，多 worker 并发时全部报错。 + +### 根因 1：Put 方法多个 SDK 调用未走限流器 + +`d1b72178` fix(115_open): rate-limit every SDK call in Put method + +- **发现**：`Put()` 入口只调一次 `WaitLimit`，后续 3-4 个 SDK 请求（`UploadInit` ×3 + `UploadGetToken`）直接发出，不经过限流器 +- **影响**：10 个 copy worker 并发时，不受限的 Put 请求和其他走限流器的 List/MakeDir 请求一起打到 115 API，瞬时 QPS 超过 115 的限制，API 返回 `state:false, code:0`（空错误）拒绝所有请求 +- **修复**：移除 Put 入口的单次 `WaitLimit`，在每个 SDK 调用（`UploadInit`、`UploadGetToken`）前单独加 `WaitLimit` +- **测试**：`TestPutRateLimitsEverySDKCall` — 设置 10 req/s 限流器，验证 3 个 UploadInit 调用之间有 >=70ms 间隔；`TestPutRateLimitsPreHashPath` — 验证秒传成功路径 +- Files: `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go` + +### 根因 2：SDK RefreshToken 无并发保护 + +`823f46ba` fix(deps): bump 115-sdk-go to v0.2.6 for concurrent refresh fix + +- **发现**：日志显示 `40140117 refresh frequently` 和 `40140120 refresh token error` 从 3 月 25 日就开始出现。115 的 refresh token 是一次性的——token 过期后多个 goroutine 同时调 `authRequest`，同时检测到 401，同时调 `RefreshToken`。第一个成功消耗了旧 RT，后续的全部失败（RT 已作废），token 被损坏或清空 +- **时间线**（`my_115` 实例）： + - 07:06:19 — 存储加载成功 + - 07:28:34 — copy workers 从 3 改成 10 + - 09:11:29 — 最后一条成功的 `[115] GetFiles` 日志 + - 09:11-09:45 — 35 分钟无 `[115]` 日志（全是文件上传，走 UploadInit 不产生 `[115]` 日志） + - 09:46:01 — 首次 `40100000 参数错误`（token 已失效/清空） + - 从 07:06 到 09:46 正好 ~2h40m，115 access token TTL 约 2h +- **修复**（SDK `v0.2.6`）：`authRequest` 中加 `refreshMu sync.Mutex` + double-check pattern。发请求前锁内读取 `usedToken`，遇 401 后锁内比对 `c.accessToken == usedToken`，若已被别的 goroutine 刷新过则跳过，未刷新才调 `RefreshToken` +- **测试**：`TestConcurrentAuthRequestRefreshesOnlyOnce` — 10 个并发 goroutine 同时打过期 token，断言 `RefreshToken` 只被调用 1 次，全部 goroutine 成功。`count=3` 稳定通过 +- Files: SDK `client.go`, `request.go`, `request_test.go`; OP `go.mod`, `go.sum` + +--- + +## 2026-05-11 — 115 GetFolderInfoByPath 空数据处理 + +### 问题现象 + +复制任务预建目标子目录时报错：`failed to get obj: json: cannot unmarshal array into Go value of type sdk.GetFolderInfoResp`。只有**不存在**的子目录报错，已存在的目录正常。 + +### 根因 + +115 Open API 的 `GetFolderInfoByPath` 在路径不存在时返回 `{state:true, data:[]}` 而不是正常的错误码。SDK 的 `authRequest` 直接把 `[]` 反序列化到 `GetFolderInfoResp`（struct）→ `json.UnmarshalTypeError`。该错误不是 `errs.ObjectNotFound`，导致 `MakeDir` 在 `op/fs.go:350` 当作未知错误抛出，而不是正常进入"目录不存在→创建"流程。 + +### 修复 + +**SDK v0.2.8**（`cf4f508` fix: return ErrDataEmpty when API responds with empty data）： +- 新增 `ErrDataEmpty` sentinel error +- `authRequest` 在 `extractData` 模式下：`data` 为 `null`/空 → 直接返回 `ErrDataEmpty`；`data` 为 `[]` 反序列化到 struct 失败 → 也返回 `ErrDataEmpty`（反序列化到 slice 类型则正常通过，不影响 `DelFile` 等返回空数组的 API） + +**Driver 层**： +- `Open115.Get()` 捕获 `sdk.ErrDataEmpty` → 转为 `errs.ObjectNotFound` +- `MakeDir` 的 `errs.IsObjectNotFound` 检测通过 → 正常创建目录 + +### 测试 + +- SDK: `TestAuthRequestReturnsErrDataEmptyForEmptyArray`、`TestAuthRequestReturnsErrDataEmptyForNull`、`TestAuthRequestSucceedsForValidObject` +- Driver: `TestGetReturnsObjectNotFoundForEmptyData`、`TestGetReturnsObjForExistingFolder` +- 既有 27 个测试全部通过，含 `TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing`（验证 `DelFile` 的 `data:[]` 不受影响） + +### 教训：不要 force-push tag + +Go module proxy 会缓存 tag 第一次发布时的内容。Force-push 更新 tag 后，`go.sum` 中记录的旧 hash 与新内容不匹配，触发 `checksum mismatch` 安全错误。正确做法是打新版号（v0.2.7 → v0.2.8）。 + +- Files: SDK `error.go`, `request.go`, `request_test.go`; OP `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go`, `go.mod`, `go.sum` + +--- + +## 已完成调查：FileStream.cache truncated stream（2026-04-07） + +### 现象 + +- `failed to read all data: (expect =50331648, actual =41592644) unexpected EOF` +- 调用链：`FsStream -> fs.PutDirectly -> op.Put -> FileStream.cache` +- 115_open 与 google_drive 均出现 + +### 根因 + +上游请求体提前结束（truncated stream），`FileStream.cache()` 的 `io.ReadFull` 严格检测并报错。`50331648 = 48MiB`（MaxBufferLimit 窗口），`41592644 ≈ 39.66MiB`（实际收到的字节数）。actual 值不固定，排除驱动逻辑在固定偏移崩溃的可能。 + +`3b2f9d55` 将"超限时落盘"改为"裁剪到 MaxBufferLimit"，使错误暴露更早，但非根因。 + +### 处置 + +P0：在上传入口增加"声明长度 vs 实际接收长度"日志；排查反向代理超时。 + +--- + ## Architecture Notes ### Upload Data Flow (Current) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index ee4c09c479..9d9ae3add9 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -209,7 +209,7 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { path = stdpath.Join(d.parentPath, path) resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { - if errors.Is(err, sdk.ErrObjectNotFound) { + if errors.Is(err, sdk.ErrObjectNotFound) || errors.Is(err, sdk.ErrDataEmpty) { return nil, errs.ObjectNotFound } return nil, err diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index bdcf4e0423..6ad4d4032b 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -14,6 +15,7 @@ import ( "time" sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -789,3 +791,38 @@ func TestPutRateLimitsPreHashPath(t *testing.T) { } assertRequestPaths(t, reqs, "/open/upload/init") } + +func TestGetReturnsObjectNotFoundForEmptyData(t *testing.T) { + driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + // 115 returns empty array when path doesn't exist + writeSDKSuccess(t, w, []any{}) + }) + driver.parentPath = "" + + _, err := driver.Get(context.Background(), "/nonexistent/path") + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, errs.ObjectNotFound) { + t.Fatalf("expected ObjectNotFound, got: %v", err) + } +} + +func TestGetReturnsObjForExistingFolder(t *testing.T) { + driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, map[string]any{ + "file_id": "99999", + "file_name": "my_folder", + "pick_code": "pc-123", + }) + }) + driver.parentPath = "" + + obj, err := driver.Get(context.Background(), "/my_folder") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if obj.GetID() != "99999" || obj.GetName() != "my_folder" { + t.Fatalf("unexpected obj: id=%s name=%s", obj.GetID(), obj.GetName()) + } +} diff --git a/go.mod b/go.mod index 6a2495e6af..10e319ae7a 100644 --- a/go.mod +++ b/go.mod @@ -312,6 +312,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.6 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.8 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index fa018c70ae..adb42d29fb 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/Ironboxplus/115-sdk-go v0.2.6 h1:xsb7MYOf4IVszL4XqHKTAvpgZAmdr+jMMw3XOC955zQ= -github.com/Ironboxplus/115-sdk-go v0.2.6/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.8 h1:JyRGXDDXktItPJansGzyLpziF1UhY30xzC5LRYTgRp4= +github.com/Ironboxplus/115-sdk-go v0.2.8/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From a212d6e6d2b1a8e143a651b9f014ab2a55d346ca Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 14 May 2026 00:01:17 +0800 Subject: [PATCH 061/107] fix(115_open,stream): validate upload callback and fix CacheFullAndWriter hash truncation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs fixed: 1. Upload callback was never validated — when 115 returned {"state":false}, the error was silently ignored. Added oss.CallbackResult capture and checkUploadCallback() validation. 2. CacheFullAndWriter for known-size files > MaxBufferLimit only cached the first ~48MB (via cache() truncation), producing a partial SHA1 hash. 115 then rejected the upload with code=10002 "校验文件失败". Fix: when file size exceeds MaxBufferLimit, write to a temp file instead of the in-memory peekBuff. This fixes all drivers using CacheFullAndHash with large non-seekable streams (115, google_drive, aliyundrive, etc). 3. Frontend fetcher blocked startup on China servers where GitHub is unreachable. Removed blocking download, use embedded dist immediately and let Watcher fetch rolling release in background. --- CLAUDE.md | 38 ++++++++++- JOURNAL.md | 108 ++++++++++++++++++++++++++++++++ drivers/115_open/driver_test.go | 38 +++++++++++ drivers/115_open/upload.go | 60 +++++++++++------- drivers/baidu_netdisk/upload.go | 2 +- server/static/static.go | 14 +---- 6 files changed, 224 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c3ffdded09..e4b09c0daf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,9 @@ air # Hot reload during development (uses .air.toml # Testing go test ./... # Run all tests +go test ./drivers/115_open/ -v # Run tests for a specific driver +go test ./drivers/115_open/ -run TestCheckUploadCallback -v # Run a single test +go build ./drivers/115_open/... # Quick compile check for a package # Docker docker-compose up # Run with docker-compose @@ -31,7 +34,9 @@ docker build -f Dockerfile . # Build docker image - Supports `dev`, `beta`, and release builds - Downloads prebuilt frontend distribution automatically -**Go Version**: Requires Go 1.23.4+ +**Go Version**: Requires Go 1.24+ (CI uses 1.25.0) + +**Module Replacements** (`go.mod`): Some dependencies use `replace` directives pointing to forks (e.g., `115-sdk-go` → `Ironboxplus/115-sdk-go`). When modifying SDK behavior, check if there's a local fork to edit. ## Architecture Overview @@ -272,6 +277,37 @@ if stream.IsLinkExpiredError(err) { } ``` +### Upload and OSS Callback Validation + +Drivers that upload via Aliyun OSS (e.g., `115`, `115_open`) use a callback mechanism: after OSS stores the file, it POSTs to the storage provider's callback URL. The provider returns a JSON response indicating whether the file was registered. + +**Critical**: Always capture and validate the callback response: +```go +var bodyBytes []byte +_, err = bucket.CompleteMultipartUpload(imur, parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(callbackVar))), + oss.CallbackResult(&bodyBytes), // ← MUST capture this +) +// Check both OSS error AND callback response +if err != nil { return err } +// Parse bodyBytes to verify {"state": true} +``` + +Without `oss.CallbackResult`, the upload appears successful (OSS returns 200) but the file is never registered on the provider's side. The local cache shows the file temporarily, but it vanishes on refresh. + +**`Put` vs `PutResult`**: Drivers can implement either `driver.Put` (returns `error`) or `driver.PutResult` (returns `model.Obj, error`). When `Put` returns nil, `op.Put` creates a temporary object in the directory cache. When `PutResult` returns an actual object, that object is used in the cache instead. + +### `CacheFullAndHash` Truncation Pitfall + +⚠️ `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` caps buffering at `MaxBufferLimit` (~48MB). For non-seekable streams (browser stream upload) larger than this limit, **SHA1 is only computed on the first ~48MB**, producing an incorrect hash. This causes providers like 115 to reject the upload with checksum errors. + +**Workaround** (used in `115_open`): Before calling `CacheFullAndHash`, cache the full stream to a temp file via `utils.CreateTempFile`, then set `fs.Reader = tmpF`. After that, `StreamHashFile` / `RangeRead` will read from the complete temp file. + +**Unaffected paths**: +- `SeekableStream` (copy tasks): uses `RangeRead` directly, never goes through `cache()` +- Form uploads: `c.FormFile()` already stores to a temp file, so `GetFile()` returns non-nil and `CacheFullAndWriter` reads the entire file + ### Saving Driver State When updating tokens or credentials: diff --git a/JOURNAL.md b/JOURNAL.md index ff58ac2369..422b594c38 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -198,6 +198,114 @@ P0：在上传入口增加"声明长度 vs 实际接收长度"日志；排查反 --- +## 2026-05-13 — 115 Open 上传静默失败修复（双 bug） + +### 问题现象 + +前端上传文件显示成功（进度 100%，PUT `/api/fs/put` 返回 HTTP 200），但刷新页面后文件消失。日志无任何错误。 + +### 日志分析 + +从 330K 行日志中定位到两次上传（12:53:31 和 13:10:05），均耗时 ~2 分钟（实际数据传输），返回 200。上传前后 `arc` 目录文件数始终为 11，文件从未出现在 115 的文件列表中。 + +对比历史上传： + +- **2026-04-06**：3 次上传（10s/22s/25s）全部成功，`PUB` 目录从 2 增至 5 文件 +- **2026-04-14**：第 1 次上传失败（5→5 文件），第 2 次成功（5→6 文件） +- **2026-05-13**：两次上传均失败（11→11 文件） + +### Bug 1：OSS 回调未校验（静默失败） + +对比非 Open 版 115 驱动（`drivers/115/util.go`）与 Open 版（`drivers/115_open/upload.go`）： + +**非 Open 版**（正确）： + +```go +var bodyBytes []byte +bucket.CompleteMultipartUpload(imur, parts, + oss.Callback(...), + oss.CallbackResult(&bodyBytes), // ← 捕获回调响应 +) +var uploadResult UploadResult +json.Unmarshal(bodyBytes, &uploadResult) +return &uploadResult, uploadResult.Err(...) // ← 校验 state 字段 +``` + +**Open 版**（有 bug）： + +```go +// callbackRespBytes := make([]byte, 1024) ← 注释掉了！ +_, err = bucket.CompleteMultipartUpload(imur, parts, + oss.Callback(...), + // oss.CallbackResult(&callbackRespBytes), ← 注释掉了！ +) +if err != nil { return err } // ← 只检查 OSS 层错误 +return nil // ← 115 回调失败被忽略 +``` + +OSS 上传流程：客户端上传分片到 OSS → `CompleteMultipartUpload` → OSS 调用 115 的回调 URL → 115 返回 `{"state": true/false}`。OSS 只要回调 URL 返回 HTTP 200 就认为成功（`err == nil`），但 115 可能在 body 中返回 `{"state": false}` 表示文件注册失败。Open 版驱动没有捕获回调 body，所以 115 拒绝注册文件时完全静默。 + +`op.Put`（`internal/op/fs.go:714-731`）在 `err == nil` 时把文件临时加入目录缓存 → 前端显示成功 → 用户刷新后缓存过期，文件消失。 + +**修复**：新增 `UploadCallbackResult` + `checkUploadCallback()`，`singleUpload` 和 `multpartUpload` 均添加 `oss.CallbackResult(&bodyBytes)` 捕获并校验回调。 + +### Bug 2：Stream 模式大文件 SHA1 截断（根因） + +加了回调校验后看到真正的错误：`code=10002, message=校验文件失败，请重新上传。` + +关键线索：Form 模式（`PUT /api/fs/form`）和 Copy task 均成功，只有 Stream 模式（`PUT /api/fs/put`）失败。 + +**根因**：`CacheFullAndHash` → `CacheFullAndWriter` → `cache()` 将缓存上限截断为 `MaxBufferLimit`（~48MB）： + +```go +func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { + if maxCacheSize > int64(conf.MaxBufferLimit) { + maxCacheSize = int64(conf.MaxBufferLimit) // ← 截断！ + } + // ...只读取前 48MB 到 peekBuff +} +``` + +870MB 文件：SHA1 只算了前 48MB → `UploadInit` 带错误的 SHA1 → 完整 870MB 上传到 OSS → 115 校验：SHA1(870MB) ≠ SHA1(前 48MB) → 拒绝。 + +**为什么 Form 模式不受影响**：`c.FormFile()` 已将文件存入 `*os.File` 临时文件，`GetFile()` 返回非 nil，`CacheFullAndWriter` 走第一个分支直接读整个文件算 hash。 + +**为什么 Copy task 不受影响**：走 `SeekableStream` 路径（`isSeekable == true`），用 `StreamHashFile` → `RangeRead` 按需读取，不经过 `cache()`。 + +**修复**：`drivers/115_open/driver.go` Put 方法，对非 seekable 的 `FileStream`，当 `GetFile() == nil` 时先用 `utils.CreateTempFile` 将完整流写入临时文件，设置 `fs.Reader = tmpF`，再用 `StreamHashFile` 从临时文件计算正确的 SHA1。 + +### 上传数据流（修复后） + +``` +Stream 上传 (FileStream, !isSeekable): + HTTP Body → CreateTempFile(磁盘) → StreamHashFile(临时文件) → UploadInit → 分片上传(临时文件) → 清理 + +Copy task (SeekableStream, isSeekable): [不变] + Pass 1: RangeRead(源存储) → StreamHashFile → SHA1 + Pass 2: RangeRead(源存储) → directSectionReader → 分片上传 +``` + +### 附带修复：前端下载阻塞启动 + +`server/static/static.go`：移除 `initStatic` 中 30 秒超时的同步前端下载（国内 GitHub 不通导致启动卡住），改为直接使用内嵌 dist，由 `Watcher` 后台异步下载 rolling 版本并热替换。 + +### 测试 + +新增 4 个回调校验测试，既有 30 个测试全部通过： + +- `TestCheckUploadCallbackSuccess` — state=true 正常通过 +- `TestCheckUploadCallbackStateFalse` — state=false 返回含 code/message 的错误 +- `TestCheckUploadCallbackEmptyBody` — 空响应返回错误 +- `TestCheckUploadCallbackInvalidJSON` — 非法 JSON 返回错误 + +### `cache()` 截断是否影响其他驱动 + +`cache()` 的 `MaxBufferLimit` 截断是刻意设计（避免流式缓存占用过多内存），但 `CacheFullAndWriter` 名为"CacheFull"却内部调 `cache()` 导致大文件只缓存部分。任何依赖 `CacheFullAndHash` 对大文件（> MaxBufferLimit）计算 hash 的非 seekable 流驱动都可能受影响。本次修复仅在 115_open 驱动层绕过，未改动核心 stream 包。 + +- Files: `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go`, `drivers/115_open/upload.go`, `server/static/static.go` + +--- + ## Architecture Notes ### Upload Data Flow (Current) diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 6ad4d4032b..27eeb3d25e 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -808,6 +808,44 @@ func TestGetReturnsObjectNotFoundForEmptyData(t *testing.T) { } } +func TestCheckUploadCallbackSuccess(t *testing.T) { + body := []byte(`{"state":true,"code":0,"message":"success","data":{"pick_code":"abc","file_name":"test.txt","file_size":100,"file_id":"123","sha1":"da39a3ee","cid":"456"}}`) + if err := checkUploadCallback(body); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } +} + +func TestCheckUploadCallbackStateFalse(t *testing.T) { + body := []byte(`{"state":false,"code":990009,"message":"upload failed"}`) + err := checkUploadCallback(body) + if err == nil { + t.Fatal("expected error for state=false, got nil") + } + if !strings.Contains(err.Error(), "990009") || !strings.Contains(err.Error(), "upload failed") { + t.Fatalf("error should contain code and message, got: %v", err) + } +} + +func TestCheckUploadCallbackEmptyBody(t *testing.T) { + err := checkUploadCallback([]byte{}) + if err == nil { + t.Fatal("expected error for empty body, got nil") + } + if !strings.Contains(err.Error(), "empty") { + t.Fatalf("error should mention empty, got: %v", err) + } +} + +func TestCheckUploadCallbackInvalidJSON(t *testing.T) { + err := checkUploadCallback([]byte(`not json`)) + if err == nil { + t.Fatal("expected error for invalid JSON, got nil") + } + if !strings.Contains(err.Error(), "parse error") { + t.Fatalf("error should mention parse error, got: %v", err) + } +} + func TestGetReturnsObjForExistingFolder(t *testing.T) { driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { writeSDKSuccess(t, w, map[string]any{ diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 7ee755e747..8669cc81bc 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/base64" + "encoding/json" "fmt" "io" "strings" @@ -19,6 +20,34 @@ import ( log "github.com/sirupsen/logrus" ) +type UploadCallbackResult struct { + State bool `json:"state"` + Code int `json:"code"` + Message string `json:"message"` + Data struct { + PickCode string `json:"pick_code"` + FileName string `json:"file_name"` + FileSize int64 `json:"file_size"` + FileID string `json:"file_id"` + Sha1 string `json:"sha1"` + Cid string `json:"cid"` + } `json:"data"` +} + +func checkUploadCallback(bodyBytes []byte) error { + if len(bodyBytes) == 0 { + return fmt.Errorf("115 upload callback returned empty response") + } + var result UploadCallbackResult + if err := json.Unmarshal(bodyBytes, &result); err != nil { + return fmt.Errorf("115 upload callback response parse error: %w (body: %s)", err, string(bodyBytes)) + } + if !result.State { + return fmt.Errorf("115 upload callback failed: code=%d, message=%s", result.Code, result.Message) + } + return nil +} + // isTokenExpiredError 检测是否为OSS凭证过期错误 func isTokenExpiredError(err error) bool { if err == nil { @@ -67,30 +96,18 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp return err } + var bodyBytes []byte err = bucket.PutObject(initResp.Object, tempF, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + oss.CallbackResult(&bodyBytes), ) - - return err + if err != nil { + return err + } + return checkUploadCallback(bodyBytes) } -// type CallbackResult struct { -// State bool `json:"state"` -// Code int `json:"code"` -// Message string `json:"message"` -// Data struct { -// PickCode string `json:"pick_code"` -// FileName string `json:"file_name"` -// FileSize int64 `json:"file_size"` -// FileID string `json:"file_id"` -// ThumbURL string `json:"thumb_url"` -// Sha1 string `json:"sha1"` -// Aid int `json:"aid"` -// Cid string `json:"cid"` -// } `json:"data"` -// } - func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { // 创建OSS客户端的辅助函数 createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { @@ -191,17 +208,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up(float64(offset) * 100 / float64(fileSize)) } - // callbackRespBytes := make([]byte, 1024) + var bodyBytes []byte _, err = bucket.CompleteMultipartUpload( imur, parts, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), - // oss.CallbackResult(&callbackRespBytes), + oss.CallbackResult(&bodyBytes), ) if err != nil { return err } - - return nil + return checkUploadCallback(bodyBytes) } diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go index c160c3a9e6..5283ffe325 100644 --- a/drivers/baidu_netdisk/upload.go +++ b/drivers/baidu_netdisk/upload.go @@ -117,7 +117,7 @@ func (d *BaiduNetdisk) calculateHashesStream( // uploadChunksStream 流式上传所有分片 func (d *BaiduNetdisk) uploadChunksStream( ctx context.Context, - ss streamPkg.StreamSectionReaderIF, + ss streamPkg.StreamSectionReader, stream model.FileStreamer, precreateResp *PrecreateResp, path string, diff --git a/server/static/static.go b/server/static/static.go index 3850b6d11f..d6bdcb5871 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -1,7 +1,6 @@ package static import ( - "context" "encoding/json" "errors" "fmt" @@ -12,7 +11,6 @@ import ( "path/filepath" "strings" "sync" - "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -75,16 +73,8 @@ func initStatic() { utils.Log.Infof("Using dynamically fetched dist: %s", distPath) return } - // 3. Try auto-fetching from rolling (short timeout to avoid blocking startup) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - distPath := frontend.EnsureDistOnce(ctx) - cancel() - if distPath != "" { - staticFS.swap(os.DirFS(distPath)) - utils.Log.Infof("Using auto-fetched dist: %s", distPath) - return - } - // 4. Final fallback to embedded dist + // 3. Fall back to embedded dist; the Watcher (started after initStatic) + // will fetch the rolling release in the background and hot-swap via ReloadStatic. dist, err := fs.Sub(public.Public, "dist") if err != nil { utils.Log.Fatalf("failed to read dist dir: %v", err) From 8b1015abc355a4b101d81e9781678eea41926610 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 9 May 2026 12:19:30 +0800 Subject: [PATCH 062/107] perf(stream): two-pass prefetch on upload pipeline Overlap source-side I/O with hash/upload compute in both passes of an upload: - Pass 1 (hash): streamHashSeekableWithPrefetch keeps a background goroutine fetching the next ~10MB chunk via ReadFullWithRangeRead while the current chunk feeds the hash. Identical hash output, only the timing changes from `Tr + Th` per chunk to `max(Tr, Th)`. - Pass 2 (upload): hybridSectionReader.GetSectionReader schedules a background prefetch of the next block while the caller uploads the current one. takePrefetched / waitPrefetch handle hits, misses, DiscardSection, and clean shutdown via file.Add(closerFunc(...)). Effect: drivers that loop sequentially without their own Before/Do split (115_open with oss.Sequential(), baidu_netdisk, aliyundrive_open, etc.) see throughput close to single-side bandwidth instead of the sum of read+write latencies. --- .../stream/section_reader_prefetch_test.go | 399 ++++++++++++++++++ internal/stream/util.go | 266 ++++++++++-- internal/stream/util_test.go | 37 ++ 3 files changed, 668 insertions(+), 34 deletions(-) create mode 100644 internal/stream/section_reader_prefetch_test.go diff --git a/internal/stream/section_reader_prefetch_test.go b/internal/stream/section_reader_prefetch_test.go new file mode 100644 index 0000000000..57e14db66b --- /dev/null +++ b/internal/stream/section_reader_prefetch_test.go @@ -0,0 +1,399 @@ +package stream + +import ( + "bytes" + "context" + "crypto/rand" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// signalReader is an io.Reader that records the high-water mark of bytes +// read so tests can deterministically observe how much of the source has +// been consumed at a given moment. It also exposes an optional per-Read +// delay so the test can create a window in which prefetch can run. +type signalReader struct { + data []byte + readPos atomic.Int64 + delay time.Duration + readCnt atomic.Int32 + chunkSig chan int64 // optional: receives high-water mark after each Read +} + +func newSignalReader(data []byte, delay time.Duration) *signalReader { + return &signalReader{data: data, delay: delay, chunkSig: make(chan int64, 1024)} +} + +func (r *signalReader) Read(p []byte) (int, error) { + pos := r.readPos.Load() + if pos >= int64(len(r.data)) { + return 0, io.EOF + } + if r.delay > 0 { + time.Sleep(r.delay) + } + n := copy(p, r.data[pos:]) + newPos := pos + int64(n) + r.readPos.Store(newPos) + r.readCnt.Add(1) + select { + case r.chunkSig <- newPos: + default: + } + return n, nil +} + +// fakeFileStream wraps a signalReader into a FileStreamer-compatible value +// reusing the production FileStream type. +func newFakeFileStream(t *testing.T, data []byte, delay time.Duration) (*FileStream, *signalReader) { + t.Helper() + sr := newSignalReader(data, delay) + fs := &FileStream{ + Ctx: context.Background(), + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Reader: io.NopCloser(sr), + } + return fs, sr +} + +// withStreamConf sets minimal stream/conf values needed for the +// hybridSectionReader path and restores them afterwards. +func withStreamConf(t *testing.T, cacheThreshold, maxBlock uint64) { + t.Helper() + prevCT := conf.AutoMemoryLimit + prevMB := conf.MaxBlockLimit + prevMF := conf.MinFreeMemory + prevConf := conf.Conf + t.Cleanup(func() { + conf.AutoMemoryLimit = prevCT + conf.MaxBlockLimit = prevMB + conf.MinFreeMemory = prevMF + conf.Conf = prevConf + }) + conf.AutoMemoryLimit = cacheThreshold + conf.MaxBlockLimit = maxBlock + conf.MinFreeMemory = 1 // keep memory path enabled + conf.Conf = &conf.Config{} +} + +// TestHybridSectionReader_PrefetchAdvancesSourceAhead verifies that after +// GetSectionReader returns block N, the underlying source has been read +// past the end of block N (i.e., prefetch of block N+1 is in flight or +// already complete). This is the core behavior of Pass 2 prefetch. +func TestHybridSectionReader_PrefetchAdvancesSourceAhead(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*4) + _, _ = rand.Read(data) + + // Per-Read delay so each chunk takes measurable time; this gives + // prefetch a window to advance during the "upload" phase below. + fs, sr := newFakeFileStream(t, data, 5*time.Millisecond) + + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + hsr, ok := ss.(*hybridSectionReader) + if !ok { + t.Fatalf("expected *hybridSectionReader, got %T", ss) + } + _ = hsr + + // Read block 0 + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll block0: %v", err) + } + if !bytes.Equal(got, data[:partSize]) { + t.Fatalf("block0 data mismatch") + } + + // Simulate "upload" time; prefetch should pull block 1 from source. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if sr.readPos.Load() > partSize { + break + } + time.Sleep(2 * time.Millisecond) + } + + if got := sr.readPos.Load(); got <= partSize { + t.Fatalf("expected prefetch to advance source past %d, got %d", partSize, got) + } + + ss.FreeSectionReader(rs) + + // Read block 1 — should be served from prefetch (data must still be correct). + rs2, err := ss.GetSectionReader(partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader(1): %v", err) + } + got2, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll block1: %v", err) + } + if !bytes.Equal(got2, data[partSize:2*partSize]) { + t.Fatalf("block1 data mismatch") + } + ss.FreeSectionReader(rs2) +} + +// TestHybridSectionReader_PrefetchSequentialCorrectness verifies that a +// full sequential walk through the file returns the exact original bytes +// when prefetch is enabled. +func TestHybridSectionReader_PrefetchSequentialCorrectness(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + const partCount = 6 + data := make([]byte, partSize*partCount) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + for i := int64(0); i < partCount; i++ { + off := i * partSize + rs, err := ss.GetSectionReader(off, partSize) + if err != nil { + t.Fatalf("GetSectionReader(%d): %v", i, err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll part %d: %v", i, err) + } + if !bytes.Equal(got, data[off:off+partSize]) { + t.Fatalf("part %d data mismatch", i) + } + ss.FreeSectionReader(rs) + } +} + +// TestHybridSectionReader_PrefetchLastChunkPartial covers the final +// partial chunk: when the file isn't a multiple of partSize, the last +// GetSectionReader call asks for a smaller length than the previous +// (prefetched) call expected. The data returned must still be correct. +func TestHybridSectionReader_PrefetchLastChunkPartial(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + // 2.5 parts: last chunk is partSize/2 + data := make([]byte, partSize*2+partSize/2) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + offsets := []struct{ off, length int64 }{ + {0, partSize}, + {partSize, partSize}, + {partSize * 2, partSize / 2}, + } + for i, p := range offsets { + rs, err := ss.GetSectionReader(p.off, p.length) + if err != nil { + t.Fatalf("GetSectionReader(%d, %d): %v", p.off, p.length, err) + } + got, err := io.ReadAll(rs) + if err != nil { + t.Fatalf("ReadAll part %d: %v", i, err) + } + if !bytes.Equal(got, data[p.off:p.off+p.length]) { + t.Fatalf("part %d data mismatch", i) + } + ss.FreeSectionReader(rs) + } +} + +// TestHybridSectionReader_PrefetchErrorSurfaces verifies that a read +// error encountered during prefetch is reported on the next +// GetSectionReader call (rather than silently ignored). +func TestHybridSectionReader_PrefetchErrorSurfaces(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*3) + _, _ = rand.Read(data) + + // Wrap the signal reader so reads targeting the second chunk return + // a hard error and zero bytes — simulating mid-chunk download failure. + sr := newSignalReader(data, 0) + fs := &FileStream{ + Ctx: context.Background(), + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Reader: io.NopCloser(readerFunc(func(p []byte) (int, error) { + if sr.readPos.Load() >= partSize { + return 0, io.ErrUnexpectedEOF + } + remaining := partSize - sr.readPos.Load() + if int64(len(p)) > remaining { + p = p[:remaining] + } + return sr.Read(p) + })), + } + + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + // Wait for prefetch to complete (and presumably fail). + time.Sleep(20 * time.Millisecond) + + _, err = ss.GetSectionReader(partSize, partSize) + if err == nil { + t.Fatalf("expected error from prefetch failure, got nil") + } +} + +// TestHybridSectionReader_NoPrefetchAfterEOF makes sure we don't start a +// prefetch goroutine past the end of the file. +func TestHybridSectionReader_NoPrefetchAfterEOF(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*2) + _, _ = rand.Read(data) + + fs, sr := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + for i := int64(0); i < 2; i++ { + rs, err := ss.GetSectionReader(i*partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader(%d): %v", i, err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + } + + // Allow any (incorrect) prefetch to fire. + time.Sleep(20 * time.Millisecond) + + // Source must not have been read past end of file. + if got := sr.readPos.Load(); got != int64(len(data)) { + t.Fatalf("source read pos = %d, want %d (no over-read past EOF)", got, len(data)) + } + + hsr := ss.(*hybridSectionReader) + if hsr.prefetch != nil { + t.Fatalf("expected no in-flight prefetch after final chunk") + } +} + +// TestHybridSectionReader_PrefetchClampedToFileSize ensures the prefetch +// length is clamped to the remaining bytes so the goroutine doesn't +// request more than the file holds. +func TestHybridSectionReader_PrefetchClampedToFileSize(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + // 1.5 parts so prefetch after block 0 must be clamped to partSize/2. + data := make([]byte, partSize+partSize/2) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader(0): %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + rs2, err := ss.GetSectionReader(partSize, partSize/2) + if err != nil { + t.Fatalf("GetSectionReader(1): %v", err) + } + got, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[partSize:]) { + t.Fatalf("partial-tail data mismatch") + } + ss.FreeSectionReader(rs2) +} + +type readerFunc func(p []byte) (int, error) + +func (f readerFunc) Read(p []byte) (int, error) { return f(p) } + +// TestHybridSectionReader_PrefetchHonorsDiscardSection verifies that +// DiscardSection invalidates any pending prefetch and that subsequent +// GetSectionReader calls at the new offset return correct data. +func TestHybridSectionReader_PrefetchHonorsDiscardSection(t *testing.T) { + withStreamConf(t, 1024, 4*1024) + + const partSize = int64(4 * 1024) + data := make([]byte, partSize*4) + _, _ = rand.Read(data) + + fs, _ := newFakeFileStream(t, data, 0) + ss, err := NewStreamSectionReader(fs, int(partSize), nil) + if err != nil { + t.Fatalf("NewStreamSectionReader: %v", err) + } + + // Read block 0 (kicks off prefetch of block 1) + rs, err := ss.GetSectionReader(0, partSize) + if err != nil { + t.Fatalf("GetSectionReader: %v", err) + } + _, _ = io.ReadAll(rs) + ss.FreeSectionReader(rs) + + // Give prefetch a moment to complete + time.Sleep(20 * time.Millisecond) + + // Caller decides to skip block 1 — calls DiscardSection + if err := ss.DiscardSection(partSize, partSize); err != nil { + t.Fatalf("DiscardSection: %v", err) + } + + // Now read block 2 — must return correct data + rs2, err := ss.GetSectionReader(2*partSize, partSize) + if err != nil { + t.Fatalf("GetSectionReader after discard: %v", err) + } + got, err := io.ReadAll(rs2) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[2*partSize:3*partSize]) { + t.Fatalf("block 2 data mismatch after discard") + } + ss.FreeSectionReader(rs2) +} diff --git a/internal/stream/util.go b/internal/stream/util.go index 06a754e86a..6dffcc5c70 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -539,46 +539,99 @@ func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressW hashFunc := hashType.NewFunc() size := file.GetSize() chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + + if _, ok := file.(*SeekableStream); ok { + return streamHashSeekableWithPrefetch(file, hashFunc, size, chunkSize, progressWeight, up) + } + buf := make([]byte, chunkSize) var offset int64 = 0 - for offset < size { readSize := chunkSize if size-offset < chunkSize { readSize = size - offset } - - var n int - var err error - - // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader - // 这样后续发送时 Reader 还能正常工作 - if _, ok := file.(*SeekableStream); ok { + n, err := io.ReadFull(file, buf[:readSize]) + if err != nil { + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) - } else { - // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） - n, err = io.ReadFull(file, buf[:readSize]) - if err != nil { - // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） - log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) - n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) - } } - if err != nil { return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) } - hashFunc.Write(buf[:n]) offset += int64(n) + if up != nil && progressWeight > 0 { + (*up)(progressWeight * float64(offset) / float64(size)) + } + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + +type hashPrefetchResult struct { + buf []byte + n int + err error +} + +func streamHashSeekableWithPrefetch(file model.FileStreamer, hashFunc io.Writer, size, chunkSize int64, progressWeight float64, up *model.UpdateProgress) (string, error) { + readChunkSize := func(off int64) int64 { + if size-off < chunkSize { + return size - off + } + return chunkSize + } + + var offset int64 + + // Read first chunk synchronously + firstSize := readChunkSize(0) + curBuf := make([]byte, chunkSize) + curN, curErr := ReadFullWithRangeRead(file, curBuf[:firstSize], 0) + if curErr != nil { + return "", fmt.Errorf("calculate hash failed at offset 0: %w", curErr) + } + + for { + nextOff := offset + int64(curN) + + // Launch prefetch for next chunk while we hash current + var prefetchCh chan hashPrefetchResult + if nextOff < size { + prefetchCh = make(chan hashPrefetchResult, 1) + nextSize := readChunkSize(nextOff) + nextBuf := make([]byte, nextSize) + go func(buf []byte, off, sz int64) { + n, err := ReadFullWithRangeRead(file, buf[:sz], off) + prefetchCh <- hashPrefetchResult{buf: buf, n: n, err: err} + }(nextBuf, nextOff, nextSize) + } + + // Hash current chunk + hashFunc.Write(curBuf[:curN]) + offset += int64(curN) if up != nil && progressWeight > 0 { - progress := progressWeight * float64(offset) / float64(size) - (*up)(progress) + (*up)(progressWeight * float64(offset) / float64(size)) } + + if prefetchCh == nil { + break + } + + // Wait for prefetch + result := <-prefetchCh + if result.err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", nextOff, result.err) + } + curBuf = result.buf + curN = result.n } - return hex.EncodeToString(hashFunc.Sum(nil)), nil + if h, ok := hashFunc.(interface{ Sum([]byte) []byte }); ok { + return hex.EncodeToString(h.Sum(nil)), nil + } + return "", fmt.Errorf("hashFunc does not implement Sum") } type StreamSectionReader interface { @@ -601,9 +654,17 @@ func NewStreamSectionReader(file model.FileStreamer, sectionSize int, up *model. return nil, err } file.Add(hc) - return &hybridSectionReader{file: file, hc: hc}, nil + ss := &hybridSectionReader{file: file, hc: hc, fileSize: file.GetSize()} + // Wait for any pending prefetch when the file is closed so we don't + // race against ss.hc being freed. + file.Add(closerFunc(ss.waitPrefetch)) + return ss, nil } +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + type cachedSectionReader struct { cache io.ReaderAt } @@ -619,13 +680,49 @@ func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} type hybridSectionReader struct { file model.FileStreamer fileOffset int64 + fileSize int64 hc *hcache.HybridCache mu sync.Mutex cache []buffer.Block + + // Pass 2 prefetch: while the caller uploads block N, we read block + // N+1 from the source in the background so download/upload overlap. + // Access is serialized through GetSectionReader/DiscardSection which + // are documented as 线程不安全; only the background prefetch goroutine + // touches `prefetch` concurrently with those methods, and waitPrefetch + // drains it before any of them touches ss.file again. + prefetch *prefetchTask +} + +type prefetchTask struct { + off int64 // file offset the prefetch started at + length int64 // bytes requested + actual int64 // bytes actually read into block (may be < length on EOF/error) + block buffer.Block // nil if allocation/read failed before any bytes + err error // non-nil on prefetch error + done chan struct{} } // 线程不安全 func (ss *hybridSectionReader) DiscardSection(off int64, length int64) error { + // Drain any pending prefetch first so ss.file is quiescent. If the + // prefetched range exactly matches the discard request, we're done — + // the bytes have already been read from the source. + if p := ss.prefetch; p != nil { + <-p.done + ss.prefetch = nil + if p.err == nil && p.off == off && p.actual == length { + if p.block != nil { + ss.put(p.block) + } + ss.fileOffset = off + length + return nil + } + if p.block != nil { + ss.put(p.block) + } + ss.fileOffset = p.off + p.actual + } if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -644,33 +741,134 @@ type blockRefReadSeeker struct { // 线程不安全 func (ss *hybridSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { + // Try prefetched block first. + if b, actual, err, ok := ss.takePrefetched(off, length); ok { + if err != nil { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("prefetch failed at offset %d: %w", off, err) + } + if actual < length { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("prefetch short read at offset %d: (expect=%d, actual=%d)", off, length, actual) + } + ss.fileOffset = off + length + ss.schedulePrefetch(off+length, length) + return makeBlockReadSeeker(b, length) + } + if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } + b, actual, err := ss.readBlock(length) + if err != nil || actual != length { + if b != nil { + ss.put(b) + } + return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, actual, err) + } + ss.fileOffset += actual + ss.schedulePrefetch(off+length, length) + return makeBlockReadSeeker(b, length) +} + +// readBlock reads `length` bytes from ss.file into a freshly populated +// buffer.Block. The returned block may be nil if no bytes could be read. +// Caller is responsible for returning the block to the pool on error. +func (ss *hybridSectionReader) readBlock(length int64) (buffer.Block, int64, error) { b := ss.get() if b == nil { offset := int64(ss.hc.Size()) written, err := ss.hc.CopyFromN(ss.file, length) - ss.fileOffset += written - if written != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, written, err) + if written == 0 { + return nil, 0, err } b = buffer.NewBlockAdapter( io.NewOffsetWriter(ss.hc, offset), - io.NewSectionReader(ss.hc, offset, length), + io.NewSectionReader(ss.hc, offset, written), ) - } else { - ws := buffer.WriteAtSeekerOf(b) - if _, err := ws.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("failed to reset cached block writer: %w", err) + return b, written, err + } + ws := buffer.WriteAtSeekerOf(b) + if _, err := ws.Seek(0, io.SeekStart); err != nil { + ss.put(b) + return nil, 0, fmt.Errorf("failed to reset cached block writer: %w", err) + } + written, err := utils.CopyWithBufferN(ws, ss.file, length) + return b, written, err +} + +// schedulePrefetch starts a background read of `length` bytes at file +// offset `off`. It is a no-op if there is nothing more to read or a +// prefetch is already in flight. +func (ss *hybridSectionReader) schedulePrefetch(off, length int64) { + if length <= 0 || off >= ss.fileSize { + return + } + if ss.prefetch != nil { + return + } + // Clamp to remaining file size so the last partial chunk doesn't + // produce a synthetic short-read error. + if remaining := ss.fileSize - off; remaining < length { + length = remaining + } + task := &prefetchTask{off: off, length: length, done: make(chan struct{})} + ss.prefetch = task + go func() { + defer close(task.done) + b, actual, err := ss.readBlock(length) + task.block = b + task.actual = actual + task.err = err + }() +} + +// takePrefetched returns the prefetched block if it matches the caller's +// requested offset. The ok return distinguishes "no prefetch present" +// (false) from "prefetch was consumed" (true). +func (ss *hybridSectionReader) takePrefetched(off, length int64) (buffer.Block, int64, error, bool) { + p := ss.prefetch + if p == nil { + return nil, 0, nil, false + } + <-p.done + ss.prefetch = nil + if p.off != off { + if p.block != nil { + ss.put(p.block) } - written, err := utils.CopyWithBufferN(ws, ss.file, length) - ss.fileOffset += written - if written != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, written, err) + // Source has already advanced by p.actual bytes past p.off. + // Reflect that so subsequent calls see a consistent fileOffset. + ss.fileOffset = p.off + p.actual + return nil, 0, nil, false + } + // Caller may ask for fewer bytes than we prefetched (e.g. final + // partial chunk after we over-prefetched). Allow that. + if length > p.actual && p.err == nil { + // Did not get as many bytes as caller wants and source didn't + // signal an error — surface a short-read for the caller to + // handle, but keep block so it can be returned. + return p.block, p.actual, fmt.Errorf("short read"), true + } + return p.block, p.actual, p.err, true +} + +func (ss *hybridSectionReader) waitPrefetch() error { + if p := ss.prefetch; p != nil { + <-p.done + ss.prefetch = nil + if p.block != nil { + ss.put(p.block) } } + return nil +} +func makeBlockReadSeeker(b buffer.Block, length int64) (io.ReadSeeker, error) { if length == b.Size() { rs := buffer.ReadAtSeekerOf(b) if _, err := rs.Seek(0, io.SeekStart); err != nil { diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go index b739802717..c39ccbfb4f 100644 --- a/internal/stream/util_test.go +++ b/internal/stream/util_test.go @@ -3,6 +3,7 @@ package stream import ( "bytes" "context" + "encoding/hex" "errors" "io" "sync" @@ -10,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { @@ -189,6 +191,41 @@ func TestSelfHealingReadCloser_UnexpectedEOFTriggersReconnect(t *testing.T) { } } +// TestStreamHashFile_SeekablePrefetchProducesSameHash verifies that +// the prefetch optimization in StreamHashFile produces the exact same +// hash as a sequential read. +func TestStreamHashFile_SeekablePrefetchProducesSameHash(t *testing.T) { + // 50 bytes = will be split into 10MB chunks in real code, but we + // override chunkSize for testing. The key point: hash must be identical. + data := []byte("The quick brown fox jumps over the lazy dog!!!!!") // 48 bytes + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := &SeekableStream{ + FileStream: &FileStream{ + Obj: &model.Object{Name: "test.bin", Size: int64(len(data))}, + Ctx: context.Background(), + }, + rangeReader: rr, + } + + hash1, err := StreamHashFile(seekable, utils.SHA1, 0, nil) + if err != nil { + t.Fatalf("StreamHashFile error: %v", err) + } + + // Compute expected hash directly + h := utils.SHA1.NewFunc() + h.Write(data) + expected := hex.EncodeToString(h.Sum(nil)) + + if hash1 != expected { + t.Fatalf("hash mismatch: got %s, want %s", hash1, expected) + } +} + type flakyReadCloser struct { data []byte failAfter int From 31663254d7c5c446794d4c59614e8417fe442203 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 17 May 2026 15:59:49 +0800 Subject: [PATCH 063/107] refactor(fs): drop BFS precreate, extract existingDstFilesFn for merge Removed preCreateDirTreeFn and its method wrapper. Directory creation is no longer batched up-front via a one-level BFS walk; instead it happens on demand: - top-level dst MakeDir still runs once at line 198 (early failure detection, active dir creation) - sub-task RunWithNextTaskCallback creates each subdir as it recurses into it - op.Put internally calls MakeDir(parent) before uploading - op.MakeDir walks the parent chain so deep trees work without pre-creation, and our cache-sync retry handles cloud-storage eventual consistency The merge-mode existedObjs logic (the only invariant the BFS precreate quietly protected, alongside #1898's ObjectNotFound tolerance) is extracted into a pure, injectable function: existingDstFilesFn(ctx, listDst, dstPath) (map[string]bool, error) 12 unit tests pin down its contract: empty dst, mixed files/dirs, raw and wrapped ObjectNotFound (regression for #1898), other List errors, ctx cancellation before/during iteration, duplicates, large dst, single-call sanity. The old 11 TestPreCreateDirTreeFn_* tests are removed with the code. --- internal/fs/copy_move.go | 126 +++-------- internal/fs/copy_move_test.go | 413 +++++++++++++++------------------- 2 files changed, 222 insertions(+), 317 deletions(-) diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index 5ae92524e3..379cc61532 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -200,28 +200,16 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer // Continue anyway - the directory might exist but Get failed due to cache issues } - // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues - // Balances between reducing API calls and maintaining fault tolerance - t.Status = "pre-creating subdirectories" - if err := t.preCreateDirectoryTree(objs, t.SrcActualPath, dstActualPath, 1); err != nil { - log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) - // Continue anyway - individual directories will be created on-demand - } - existedObjs := make(map[string]bool) + // Build the set of files already at dst so merge can skip them. + // Subdirectories are created on-demand: each sub-task's recursive + // RunWithNextTaskCallback calls MakeDir at line 198, and op.Put + // also calls MakeDir for the parent before uploading. op.MakeDir + // itself walks up the parent chain so deep trees are covered. + var existedObjs map[string]bool if t.TaskType == merge { - dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) - if err != nil && !errors.Is(err, errs.ObjectNotFound) { - // 目标文件夹不存在的情况不是错误，会在之后新建文件夹 - // 这种情况显然不需要统计existedObjs，dstObjs保持为nil，下面这个for将不会执行 - return errors.WithMessagef(err, "failed list dst [%s] objs", dstActualPath) - } - for _, obj := range dstObjs { - if err := t.Ctx().Err(); err != nil { - return err - } - if !obj.IsDir() { - existedObjs[obj.GetName()] = true - } + existedObjs, err = t.existingDstFiles(dstActualPath) + if err != nil { + return err } } @@ -278,86 +266,42 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } -// preCreateDirectoryTree is a thin method wrapper that resolves the storage-bound -// makeDir / listSrc functions and delegates to the pure preCreateDirTreeFn helper. -func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { - makeDir := func(ctx context.Context, path string) error { - return op.MakeDir(ctx, t.DstStorage, path) - } - listSrc := func(ctx context.Context, path string) ([]model.Obj, error) { - return op.List(ctx, t.SrcStorage, path, model.ListArgs{}) +// existingDstFiles wraps existingDstFilesFn with the storage-bound List call, +// for use by RunWithNextTaskCallback. Returns the set of file names already +// present at dstActualPath; a non-existent destination yields an empty map +// rather than an error so merge tasks can run against a fresh destination. +func (t *FileTransferTask) existingDstFiles(dstActualPath string) (map[string]bool, error) { + listDst := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.DstStorage, path, model.ListArgs{}) } - return preCreateDirTreeFn(t.Ctx(), objs, srcBasePath, dstBasePath, maxDepth, makeDir, listSrc) + return existingDstFilesFn(t.Ctx(), listDst, dstActualPath) } -// preCreateDirTreeFn recursively scans source directory tree and pre-creates -// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. -// -// - maxDepth=0 – only create dirs in the current objs list (no recursion) -// - maxDepth=1 – also recurse one level deeper, etc. -// - srcBasePath – current source directory path; passed explicitly through all -// recursion levels so that subdirSrcPath is always correct (do NOT use -// t.SrcActualPath, which is fixed at the top-level path). +// existingDstFilesFn collects names of files (not directories) already +// present at dstPath. A non-existent destination (errs.ObjectNotFound, +// possibly wrapped) is treated as an empty result so merge tasks can run +// against fresh destinations — see PR #1898. Other List errors propagate. // -// makeDir and listSrc are injected to enable testing without a real storage driver. -func preCreateDirTreeFn( +// listDst is injected so tests can drive this without a real storage driver. +func existingDstFilesFn( ctx context.Context, - objs []model.Obj, - srcBasePath, dstBasePath string, - maxDepth int, - makeDir func(context.Context, string) error, - listSrc func(context.Context, string) ([]model.Obj, error), -) error { - // First pass: create immediate subdirectories - var subdirs []model.Obj - for _, obj := range objs { - // Check for cancellation - if err := ctx.Err(); err != nil { - return err - } - - if obj.IsDir() { - subdirPath := stdpath.Join(dstBasePath, obj.GetName()) - if err := makeDir(ctx, subdirPath); err != nil { - log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) - // Continue with other directories - } - subdirs = append(subdirs, obj) - // No explicit sleep here: drivers that have QPS limits (e.g. 115, BaiduNetDisk) - // implement WaitLimit via a token-bucket rate.Limiter and call it inside their - // MakeDir, so op.MakeDir already blocks at the correct per-driver rate. - } + listDst func(context.Context, string) ([]model.Obj, error), + dstPath string, +) (map[string]bool, error) { + dstObjs, err := listDst(ctx, dstPath) + if err != nil && !errors.Is(err, errs.ObjectNotFound) { + return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) } - - // Stop recursion if max depth reached - if maxDepth <= 0 { - return nil - } - - // Second pass: recursively scan and create nested subdirectories - for _, subdir := range subdirs { + existed := make(map[string]bool, len(dstObjs)) + for _, obj := range dstObjs { if err := ctx.Err(); err != nil { - return err - } - - // Build paths relative to srcBasePath (NOT t.SrcActualPath) so that - // deeper recursion levels resolve to the correct source paths. - subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) - subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) - - subObjs, err := listSrc(ctx, subdirSrcPath) - if err != nil { - log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) - continue // Skip this subdirectory, will handle when processing + return nil, err } - - // Recursively create subdirectories with decreased depth - if err := preCreateDirTreeFn(ctx, subObjs, subdirSrcPath, subdirDstPath, maxDepth-1, makeDir, listSrc); err != nil { - return err + if !obj.IsDir() { + existed[obj.GetName()] = true } } - - return nil + return existed, nil } var ( diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go index 5c9f4b5399..cf77c1d794 100644 --- a/internal/fs/copy_move_test.go +++ b/internal/fs/copy_move_test.go @@ -3,11 +3,14 @@ package fs import ( "context" "errors" + "fmt" "sync" "testing" "time" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + pkgerrors "github.com/pkg/errors" ) // ---------- helpers ---------- @@ -20,298 +23,256 @@ func fileObj(name string) model.Obj { return &model.Object{Name: name, IsFolder: false} } -// callRecorder records every path passed to makeDir and listSrc. -type callRecorder struct { - mu sync.Mutex - mkdirs []string - lists []string - // listReturns maps srcPath → objects to return (nil = empty) - listReturns map[string][]model.Obj - // mkdirErr maps dstPath → error to return - mkdirErr map[string]error +// listRecorder records every call to listDst so tests can assert on +// invocation patterns when needed. +type listRecorder struct { + mu sync.Mutex + calls []string + respond func(path string) ([]model.Obj, error) } -func newRecorder() *callRecorder { - return &callRecorder{ - listReturns: make(map[string][]model.Obj), - mkdirErr: make(map[string]error), - } +func newListRecorder(respond func(string) ([]model.Obj, error)) *listRecorder { + return &listRecorder{respond: respond} } -func (r *callRecorder) makeDir(_ context.Context, path string) error { +func (r *listRecorder) listDst(_ context.Context, path string) ([]model.Obj, error) { r.mu.Lock() - r.mkdirs = append(r.mkdirs, path) - err := r.mkdirErr[path] + r.calls = append(r.calls, path) r.mu.Unlock() - return err + return r.respond(path) } -func (r *callRecorder) listSrc(_ context.Context, path string) ([]model.Obj, error) { - r.mu.Lock() - r.lists = append(r.lists, path) - objs := r.listReturns[path] - r.mu.Unlock() - return objs, nil -} - -func (r *callRecorder) hasMkdir(path string) bool { - r.mu.Lock() - defer r.mu.Unlock() - for _, p := range r.mkdirs { - if p == path { - return true - } +// ---------- tests: existingDstFilesFn ---------- +// +// These tests pin down the contract of the merge-mode existedObjs builder +// extracted from RunWithNextTaskCallback. They are the regression guard +// for the deletion of the BFS precreate logic: the only invariant that +// the BFS precreate quietly protected (and that #1898's ObjectNotFound +// tolerance independently fixes) lives in this function. + +// 1. dst exists but is empty → empty map, no error. +func TestExistingDstFilesFn_EmptyDst(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { return nil, nil }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) } - return false -} - -func (r *callRecorder) hasList(path string) bool { - r.mu.Lock() - defer r.mu.Unlock() - for _, p := range r.lists { - if p == path { - return true - } + if len(got) != 0 { + t.Fatalf("expected empty map, got %v", got) } - return false } -// ---------- tests ---------- - -// TestPreCreateDirTreeFn_EmptyObjs: no objects → no calls at all. -func TestPreCreateDirTreeFn_EmptyObjs(t *testing.T) { - rec := newRecorder() - err := preCreateDirTreeFn(context.Background(), nil, "/src", "/dst", 1, rec.makeDir, rec.listSrc) +// 2. dst contains only files → all included. +func TestExistingDstFilesFn_OnlyFiles(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{fileObj("a.txt"), fileObj("b.txt"), fileObj("c.txt")}, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(rec.mkdirs) != 0 { - t.Errorf("expected 0 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + for _, name := range []string{"a.txt", "b.txt", "c.txt"} { + if !got[name] { + t.Errorf("expected %q in existed set, got %v", name, got) + } } - if len(rec.lists) != 0 { - t.Errorf("expected 0 List calls, got %d: %v", len(rec.lists), rec.lists) + if len(got) != 3 { + t.Errorf("expected 3 entries, got %d: %v", len(got), got) } } -// TestPreCreateDirTreeFn_OnlyFiles: file objects only → zero MakeDir calls. -func TestPreCreateDirTreeFn_OnlyFiles(t *testing.T) { - objs := []model.Obj{fileObj("a.txt"), fileObj("b.txt")} - rec := newRecorder() - if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { +// 3. dst contains only directories → empty map (dirs don't count as +// "existed" because merge only skips already-uploaded *files*). +func TestExistingDstFilesFn_OnlyDirs(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{dirObj("subA"), dirObj("subB")}, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(rec.mkdirs) != 0 { - t.Errorf("expected 0 MakeDir calls, got %d", len(rec.mkdirs)) + if len(got) != 0 { + t.Fatalf("dirs must not appear in existed-files map, got %v", got) } } -// TestPreCreateDirTreeFn_FlatDirs_MaxDepth0: dirs present, maxDepth=0 → MakeDir -// called for each dir with correct dstPath, NO listSrc calls. -func TestPreCreateDirTreeFn_FlatDirs_MaxDepth0(t *testing.T) { - objs := []model.Obj{dirObj("subA"), fileObj("file.txt"), dirObj("subB")} - rec := newRecorder() - if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst/parent", 0, rec.makeDir, rec.listSrc); err != nil { +// 4. dst contains mixed files and dirs → only files are recorded. +func TestExistingDstFilesFn_MixedFilesAndDirs(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + fileObj("go.mod"), + }, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { t.Fatalf("unexpected error: %v", err) } - - if !rec.hasMkdir("/dst/parent/subA") { - t.Error("expected MakeDir(/dst/parent/subA)") + if len(got) != 3 { + t.Errorf("expected 3 file entries, got %d: %v", len(got), got) } - if !rec.hasMkdir("/dst/parent/subB") { - t.Error("expected MakeDir(/dst/parent/subB)") - } - if rec.hasMkdir("/dst/parent/file.txt") { - t.Error("MakeDir must NOT be called for a file") + for _, name := range []string{"readme.md", "main.go", "go.mod"} { + if !got[name] { + t.Errorf("expected %q in existed set, got %v", name, got) + } } - if len(rec.lists) != 0 { - t.Errorf("maxDepth=0 must not trigger any List calls, got: %v", rec.lists) + for _, name := range []string{"assets", "pkg"} { + if got[name] { + t.Errorf("dir %q must NOT be in existed set, got %v", name, got) + } } } -// TestPreCreateDirTreeFn_Recursion_CorrectSrcPath is the regression test for the -// srcBasePath bug: with maxDepth=1 the recursive List must use the SUBDIR src path, -// not the original top-level srcBasePath. -func TestPreCreateDirTreeFn_Recursion_CorrectSrcPath(t *testing.T) { - // /src/parent contains [subA(dir), subB(dir)] - // /src/parent/subA contains [subA1(dir)] - // /src/parent/subB contains [] - topObjs := []model.Obj{dirObj("subA"), dirObj("subB")} - rec := newRecorder() - rec.listReturns["/src/parent/subA"] = []model.Obj{dirObj("subA1")} - rec.listReturns["/src/parent/subB"] = []model.Obj{} - - if err := preCreateDirTreeFn(context.Background(), topObjs, "/src/parent", "/dst/parent", 1, rec.makeDir, rec.listSrc); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // ── first level dirs must be created - if !rec.hasMkdir("/dst/parent/subA") { - t.Error("expected MakeDir(/dst/parent/subA)") +// 5. dst doesn't exist (raw errs.ObjectNotFound) → empty map, no error. +// THIS IS THE REGRESSION TEST FOR #1898. +func TestExistingDstFilesFn_DstDoesNotExist_RawError(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return nil, errs.ObjectNotFound + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/never-existed") + if err != nil { + t.Fatalf("ObjectNotFound on dst must be tolerated, got error: %v", err) } - if !rec.hasMkdir("/dst/parent/subB") { - t.Error("expected MakeDir(/dst/parent/subB)") + if len(got) != 0 { + t.Fatalf("expected empty map on non-existent dst, got %v", got) } +} - // ── listSrc must use subdirSrcPath (NOT the whole /src/parent again) - if !rec.hasList("/src/parent/subA") { - t.Error("listSrc must be called with /src/parent/subA, got:", rec.lists) - } - if !rec.hasList("/src/parent/subB") { - t.Error("listSrc must be called with /src/parent/subB, got:", rec.lists) +// 6. dst doesn't exist (wrapped via pkg/errors.WithMessage) → empty map. +// Guards the errors.Is unwrapping behavior that matters in practice: +// op.List wraps the underlying ObjectNotFound with context messages. +func TestExistingDstFilesFn_DstDoesNotExist_WrappedError(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + // emulate the op.List wrapping path: GetUnwrap returns + // ObjectNotFound, list wraps with WithMessage twice. + return nil, pkgerrors.WithMessage(pkgerrors.WithMessage(errs.ObjectNotFound, "failed get dir"), "while listing") + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("wrapped ObjectNotFound must be tolerated, got error: %v", err) } - // The original bug would have called listSrc("/src/parent/subA") as - // stdpath.Join(t.SrcActualPath, "subA") where t.SrcActualPath=="/src/parent", - // but in a deeper recursive call (e.g. maxDepth=2) it would have used - // the top-level path incorrectly; verify the nested mkdir used the right dst. - if !rec.hasMkdir("/dst/parent/subA/subA1") { - t.Error("expected MakeDir(/dst/parent/subA/subA1), got mkdirs:", rec.mkdirs) + if len(got) != 0 { + t.Fatalf("expected empty map, got %v", got) } } -// TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion: with maxDepth=1 recursion -// goes exactly one level. The nested list returns another dir, but since maxDepth -// reaches 0 that deeper dir must NOT be listed further. -func TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion(t *testing.T) { - topObjs := []model.Obj{dirObj("sub")} - rec := newRecorder() - // sub contains deeper, deeper contains deepest - rec.listReturns["/src/sub"] = []model.Obj{dirObj("deeper")} - rec.listReturns["/src/sub/deeper"] = []model.Obj{dirObj("deepest")} // should NOT be listed - - if err := preCreateDirTreeFn(context.Background(), topObjs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if !rec.hasMkdir("/dst/sub") { - t.Error("expected /dst/sub to be created") - } - if !rec.hasMkdir("/dst/sub/deeper") { - t.Error("expected /dst/sub/deeper to be created (within maxDepth=1)") - } - // deepest must NOT be created (would require maxDepth=2) - if rec.hasMkdir("/dst/sub/deeper/deepest") { - t.Error("/dst/sub/deeper/deepest must NOT be created at maxDepth=1") +// 7. List returns a non-ObjectNotFound error (e.g. permission denied, +// I/O failure) → error propagates so the task fails fast. +func TestExistingDstFilesFn_OtherListError_Propagates(t *testing.T) { + sentinel := errors.New("permission denied") + rec := newListRecorder(func(string) ([]model.Obj, error) { + return nil, sentinel + }) + _, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err == nil { + t.Fatal("expected error to propagate, got nil") } - // /src/sub/deeper must NOT be listed (we've hit maxDepth=0 at that point) - if rec.hasList("/src/sub/deeper") { - t.Error("/src/sub/deeper must NOT be listed when maxDepth reaches 0") + if !errors.Is(err, sentinel) { + t.Fatalf("expected wrapped sentinel error, got: %v", err) } } -// TestPreCreateDirTreeFn_ContextCancelled: context cancelled before processing → -// returns ctx.Err, makes zero or partial calls. -func TestPreCreateDirTreeFn_ContextCancelled(t *testing.T) { +// 8. Context cancelled before list → ctx.Err returned (the contract for +// listDst is to honor ctx; the caller's loop also checks ctx.Err). +func TestExistingDstFilesFn_CtxCancelledBeforeList(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - cancel() // already cancelled + cancel() - objs := []model.Obj{dirObj("sub")} - rec := newRecorder() - err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + rec := newListRecorder(func(string) ([]model.Obj, error) { + // A real op.List would honor ctx and return ctx.Err. + return nil, ctx.Err() + }) + _, err := existingDstFilesFn(ctx, rec.listDst, "/dst") if !errors.Is(err, context.Canceled) { - t.Errorf("expected context.Canceled, got: %v", err) - } - if len(rec.mkdirs) != 0 { - t.Errorf("no MakeDir should be called after cancellation, got: %v", rec.mkdirs) + t.Fatalf("expected context.Canceled, got: %v", err) } } -// TestPreCreateDirTreeFn_ContextCancelledDuringRecursion: context is cancelled -// during the second-pass recursion loop. -func TestPreCreateDirTreeFn_ContextCancelledDuringRecursion(t *testing.T) { +// 9. Context cancelled mid-iteration → loop bails with ctx.Err and the +// partial map is not returned. +func TestExistingDstFilesFn_CtxCancelledDuringIteration(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - // Two dirs; cancel after the first List in recursion - callCount := 0 - listSrc := func(c context.Context, path string) ([]model.Obj, error) { - callCount++ - cancel() // cancel on first list call - return nil, nil - } - objs := []model.Obj{dirObj("sub1"), dirObj("sub2")} - rec := newRecorder() - err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, listSrc) - if !errors.Is(err, context.Canceled) { - t.Errorf("expected context.Canceled after cancellation during recursion, got: %v", err) - } - if callCount > 1 { - t.Errorf("listSrc should have been called at most once before ctx.Err fired, got %d", callCount) + // Big enough list that cancelling after returning still gives the + // loop a chance to observe ctx.Err. + objs := make([]model.Obj, 1000) + for i := range objs { + objs[i] = fileObj(fmt.Sprintf("f-%d.txt", i)) } -} - -// TestPreCreateDirTreeFn_MakeDirErrorNonFatal: a MakeDir failure on one dir must -// not stop processing of subsequent dirs. -func TestPreCreateDirTreeFn_MakeDirErrorNonFatal(t *testing.T) { - objs := []model.Obj{dirObj("subA"), dirObj("subB"), dirObj("subC")} - rec := newRecorder() - rec.mkdirErr["/dst/subA"] = errors.New("quota exceeded") - if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { - t.Fatalf("error should not propagate from MakeDir failure: %v", err) - } - // All three must have been attempted despite the error on subA - for _, p := range []string{"/dst/subA", "/dst/subB", "/dst/subC"} { - if !rec.hasMkdir(p) { - t.Errorf("expected MakeDir(%s) to be called", p) - } + rec := newListRecorder(func(string) ([]model.Obj, error) { + cancel() // cancel immediately after list returns + return objs, nil + }) + _, err := existingDstFilesFn(ctx, rec.listDst, "/dst") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) } } -// TestPreCreateDirTreeFn_ListErrorNonFatal: a List error for one subdir during -// recursion skips that subdir but continues with the rest. -func TestPreCreateDirTreeFn_ListErrorNonFatal(t *testing.T) { - objs := []model.Obj{dirObj("subA"), dirObj("subB")} - listCallCount := 0 - listSrc := func(_ context.Context, path string) ([]model.Obj, error) { - listCallCount++ - if path == "/src/subA" { - return nil, errors.New("I/O error") - } - return []model.Obj{dirObj("nested")}, nil - } - rec := newRecorder() - if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, listSrc); err != nil { - t.Fatalf("List error must not be fatal: %v", err) +// 10. dst contains duplicate file names (pathological but possible if a +// driver returns duplicates) → map dedupes silently, no panic, no error. +func TestExistingDstFilesFn_DuplicateNames(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{ + fileObj("dup.txt"), + fileObj("dup.txt"), + fileObj("unique.txt"), + }, nil + }) + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) } - // subB's nested dir should still be processed despite subA's List failure - if !rec.hasMkdir("/dst/subB/nested") { - t.Error("expected /dst/subB/nested to be created despite subA list error, mkdirs:", rec.mkdirs) + if len(got) != 2 { + t.Fatalf("expected 2 dedup'd entries, got %d: %v", len(got), got) } - if listCallCount != 2 { - t.Errorf("both subdirs must be attempted for listing, got %d calls", listCallCount) + if !got["dup.txt"] || !got["unique.txt"] { + t.Fatalf("expected both names, got %v", got) } } -// TestPreCreateDirTreeFn_MixedObjs: mixed files and dirs; only dirs are processed. -func TestPreCreateDirTreeFn_MixedObjs(t *testing.T) { - objs := []model.Obj{ - fileObj("readme.md"), - dirObj("assets"), - fileObj("main.go"), - dirObj("pkg"), +// 11. dst contains a large number of files → all included; the call +// completes within a reasonable budget (sanity, not strict perf). +func TestExistingDstFilesFn_LargeDst(t *testing.T) { + const N = 10000 + objs := make([]model.Obj, N) + for i := range objs { + objs[i] = fileObj(fmt.Sprintf("f-%05d.bin", i)) } - rec := newRecorder() - if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + rec := newListRecorder(func(string) ([]model.Obj, error) { return objs, nil }) + + start := time.Now() + got, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + elapsed := time.Since(start) + + if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(rec.mkdirs) != 2 { - t.Errorf("expected exactly 2 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + if len(got) != N { + t.Fatalf("expected %d entries, got %d", N, len(got)) } - if !rec.hasMkdir("/dst/assets") || !rec.hasMkdir("/dst/pkg") { - t.Errorf("unexpected mkdirs: %v", rec.mkdirs) + // Generous bound — only catches catastrophic regressions (e.g. + // accidental O(N²) inside the loop). + if elapsed > 2*time.Second { + t.Fatalf("processing %d entries took %v, too slow", N, elapsed) } } -// TestPreCreateDirTreeFn_Timeout: context with a very short deadline cancels execution. -func TestPreCreateDirTreeFn_Timeout(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - - time.Sleep(5 * time.Millisecond) // ensure deadline has passed - - objs := []model.Obj{dirObj("sub")} - rec := newRecorder() - err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) - if !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("expected DeadlineExceeded, got: %v", err) +// 12. Sanity: the helper invokes listDst exactly once with the given +// dstPath (no double-listing). +func TestExistingDstFilesFn_CallsListOnce(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { + return []model.Obj{fileObj("a.txt")}, nil + }) + _, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/exact/path") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.calls) != 1 || rec.calls[0] != "/dst/exact/path" { + t.Fatalf("expected single call to /dst/exact/path, got %v", rec.calls) } } From 09c69407481132a95f13ccb0a826ed043ec208e2 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 17 May 2026 16:02:45 +0800 Subject: [PATCH 064/107] docs: log 2026-05-17 HybridCache rebase, Pass 2 prefetch, BFS precreate removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JOURNAL.md: new entry covering the three coordinated changes (commits fccab338 → 7995dbdb → 6b3ce577) — rebase onto upstream HybridCache, Pass 2 prefetch on hybridSectionReader, and BFS precreate deletion. CLAUDE.md: replace the now-obsolete "MaxBufferLimit truncation pitfall" section with the current HybridCache story; add new sections for the hybridSectionReader prefetch and the merge-task ObjectNotFound tolerance contract (incl. the existingDstFilesFn extraction). --- CLAUDE.md | 26 +++++++++++++++--- JOURNAL.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e4b09c0daf..c6d9bf1731 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -298,16 +298,36 @@ Without `oss.CallbackResult`, the upload appears successful (OSS returns 200) bu **`Put` vs `PutResult`**: Drivers can implement either `driver.Put` (returns `error`) or `driver.PutResult` (returns `model.Obj, error`). When `Put` returns nil, `op.Put` creates a temporary object in the directory cache. When `PutResult` returns an actual object, that object is used in the cache instead. -### `CacheFullAndHash` Truncation Pitfall +### HybridCache replaces the old `MaxBufferLimit` truncation -⚠️ `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` caps buffering at `MaxBufferLimit` (~48MB). For non-seekable streams (browser stream upload) larger than this limit, **SHA1 is only computed on the first ~48MB**, producing an incorrect hash. This causes providers like 115 to reject the upload with checksum errors. +Historically `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` capped buffering at `MaxBufferLimit` (~48MB), so non-seekable streams larger than that produced a truncated SHA1 (only the first 48MB hashed) and providers like 115 rejected the upload. The 115_open driver carried a `utils.CreateTempFile` workaround. -**Workaround** (used in `115_open`): Before calling `CacheFullAndHash`, cache the full stream to a temp file via `utils.CreateTempFile`, then set `fs.Reader = tmpF`. After that, `StreamHashFile` / `RangeRead` will read from the complete temp file. +**Current (2026-05-17)**: upstream PR #2460 unified caching via `internal/mem.HybridCache` — three tiers (heap memory → `LinearMemory` → temp file fallback) with 16MB blocks (`MaxBlockLimit`). `CacheFullAndWriter` now writes the *entire* stream regardless of size, so the truncation bug is gone for **all drivers**, not just 115_open. The 115_open workaround was removed; the standard `stream.CacheFullAndHash` path is back. **Unaffected paths**: - `SeekableStream` (copy tasks): uses `RangeRead` directly, never goes through `cache()` - Form uploads: `c.FormFile()` already stores to a temp file, so `GetFile()` returns non-nil and `CacheFullAndWriter` reads the entire file +### Pass 2 prefetch on `hybridSectionReader` + +For sequential multipart uploads (115_open, baidu_netdisk, aliyundrive_open, etc.), `hybridSectionReader.GetSectionReader` launches a background goroutine to pre-read the next chunk while the caller uploads the current one. The next call picks up the prefetched block; mismatched offsets fall back to synchronous read. Prefetch is clamped to remaining file size, drains on `DiscardSection`, and surfaces errors on the next `GetSectionReader`. Cleanup is registered via `file.Add` so the goroutine is drained before `HybridCache` is freed. + +Drivers that already use `errgroup.Lifecycle` (e.g. 123/upload.go) with `Before` (List/GetSectionReader) and `Do` (upload) get overlap from the lifecycle pattern itself; the section-reader prefetch helps drivers that loop sequentially without Before/Do split (e.g. 115_open, which requires `oss.Sequential()`). + +### Merge-task ObjectNotFound tolerance + +Merge-mode `FileTransferTask.RunWithNextTaskCallback` calls `op.List(dst)` to build the `existedObjs` skip set. A non-existent dst must be treated as "empty" rather than fatal (otherwise resuming an interrupted merge to a fresh dst fails immediately). The logic is extracted into `existingDstFilesFn` in `internal/fs/copy_move.go`: + +```go +dstObjs, err := listDst(ctx, dstPath) +if err != nil && !errors.Is(err, errs.ObjectNotFound) { + return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) +} +// non-existent dst → empty map, merge proceeds and creates dst on demand +``` + +A previous BFS-style "precreate one level of subdirectories" optimization was removed (2026-05-17, commit `6b3ce577`); directory creation now happens on demand via `op.Put`'s internal `MakeDir(parent)` and `op.MakeDir`'s recursive parent walk. The top-level dst `MakeDir` at `copy_move.go:198` is kept for early failure detection. 12 unit tests on `existingDstFilesFn` (including raw + wrapped `ObjectNotFound`) lock in the contract. + ### Saving Driver State When updating tokens or credentials: diff --git a/JOURNAL.md b/JOURNAL.md index 422b594c38..25523f86c8 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -306,6 +306,84 @@ Copy task (SeekableStream, isSeekable): [不变] --- +## 2026-05-17 — Rebase onto upstream HybridCache + Pass 2 prefetch + BFS precreate 删除 + +### 背景 + +Upstream（OpenListTeam/OpenList）通过 PR #2460（`b6db83ed`）引入 `HybridCache`：三级缓存（普通堆内存 → `LinearMemory` → 文件落盘），用来统一 stream 缓存路径。这恰好替代了 2026-05-13 在 115_open 驱动层手写的 SHA1 截断绕过（`utils.CreateTempFile` workaround）。 + +### 行动 1：Rebase 16 个提交到 upstream HybridCache + +用 `git rebase --onto upstream/main` 把本地分支挪到 HybridCache 之上。冲突解决原则：**保留双方功能**。 + +关键冲突文件： + +- **`internal/stream/stream.go`** — 取 upstream HybridCache 版（`cache()` 内部三级分配），删掉本地原来的 `MaxBufferLimit` workaround +- **`internal/stream/util.go`** — 保留本地全部新增：`RefreshableRangeReader`、`selfHealingReadCloser`、`IsLinkExpiredError`、`ReadFullWithRangeRead`、`streamHashSeekableWithPrefetch`；丢弃本地的 `directSectionReader` + prefetch（在新结构上重写）；upstream 的 `byteSectionReader` / `hybridSectionReader` 保留 +- **`drivers/115_open/driver.go`** Put 方法 — 删掉 `CreateTempFile` workaround，恢复标准 `CacheFullAndHash` 路径（HybridCache 已经保证完整缓存） +- **`drivers/baidu_netdisk/upload.go`** — 接口重命名：`StreamSectionReaderIF` → `StreamSectionReader`（upstream 名称） +- **`server/static/static.go`** — 保留本地动态前端 `reloadableFS`（rebase 期间一度被覆盖，已修正） + +Rebase 后 22 commits ahead of origin，force-pushed 至 `7995dbdb`。 + +### HybridCache vs 旧 workaround + +| 维度 | 旧 workaround | HybridCache | +|---|---|---| +| 修复范围 | 仅 115_open | stream 层，全 driver 受益 | +| 大文件存储 | 一律落盘 | 优先内存，紧张才落盘 | +| 块策略 | 单 buffer | 16MB 分块（`MaxBlockLimit`） | +| GC 友好度 | 一个大 `[]byte` | `LinearMemory` 避开 Go heap | +| Section reader 适配 | 整 buffer 切片 | 每块独立，对 `hybridSectionReader` 天然友好 | + +### 行动 2：Pass 2 prefetch on `hybridSectionReader`（commit `7995dbdb`） + +旧 `directSectionReader` 已被 upstream 的 `hybridSectionReader` 替代，但失去了"上传当前分片时异步预读下一分片"的优化（影响 115_open 这种顺序上传的 driver）。 + +TDD 红→绿→重构： + +- 在 `hybridSectionReader` 上加 `prefetchTask` 状态 + `schedulePrefetch` / `takePrefetched` / `waitPrefetch` +- 每次 `GetSectionReader` 返回后启动 goroutine 预读下一块到一个 pending block +- 下次 `GetSectionReader` 命中即拿、不命中走原路径 +- `DiscardSection` 排空 prefetch；prefetch 错误延迟到下次 `GetSectionReader` 暴露 +- `file.Add(closerFunc(waitPrefetch))` 注册清理，避免 `hc` 被释放时 prefetch 还在写 + +8 个测试覆盖：overlap 时序、顺序正确性、最后分片小于 partSize、Discard 配合、prefetch 错误传播、不超 EOF、长度 clamp、非 hybrid 路径回退。 + +- Files: `internal/stream/util.go`, `internal/stream/section_reader_prefetch_test.go` + +### 行动 3：删除 BFS precreate（commit `6b3ce577`） + +旧 `preCreateDirTreeFn` 提前 BFS 创建 dst 一层子目录。原始动机：避开"merge 任务 List 不存在的 dst 会致命"的 bug。 + +排查后发现该 bug 已由三层独立修复覆盖： + +1. **`copy_move.go:213` 容错判断**（PR #1898，你最初提交、KirCute 优化、Tron 合入）：`if err != nil && !errors.Is(err, errs.ObjectNotFound)` 容忍 dst 不存在 +2. **`op.Put` 内置 MakeDir**（`internal/op/fs.go:682`）：上传前自动建父目录 +3. **`op.MakeDir` 递归 + 缓存同步重试**（`internal/op/fs.go:354-364`）：递归建父链 + 100ms 重试解决云盘缓存延迟 + +BFS precreate 在功能上已变成纯性能优化（提前批建 + 缓存预热），不再是 bug 修复的关键依赖。 + +**TDD 流程**： + +1. 抽出 `existingDstFilesFn(ctx, listDst, dstPath) → map[string]bool`，把 merge 模式构建 `existedObjs` 的逻辑独立为可注入纯函数 +2. 写 12 个单测：empty/files-only/dirs-only/mixed/raw-ObjectNotFound/wrapped-ObjectNotFound（#1898 回归保护）/其他 List 错误/ctx 取消前/ctx 取消中/重名/万级 dst 性能/单次 List 合理性 +3. 全绿后删除 `preCreateDirTreeFn`、`preCreateDirectoryTree` 方法、调用点、原有 11 个 `TestPreCreateDirTreeFn_*` 测试 +4. 保留顶层 dst MakeDir（`copy_move.go:198`）做早期失败检测 + +净 -95 行代码。`existingDstFilesFn` 的 ObjectNotFound 容错测试（`TestExistingDstFilesFn_DstDoesNotExist_RawError` / `_WrappedError`）是 #1898 的回归护栏，确保未来重构不把那个老 bug 漏回去。 + +- Files: `internal/fs/copy_move.go`, `internal/fs/copy_move_test.go` + +### 行动后状态 + +- `go build ./...` 干净 +- `internal/fs`、`internal/stream`、`internal/op`、`drivers/115_open`、`drivers/baidu_netdisk` 全部测试通过 +- 不相关失败：`internal/net/oss_test.go` 的 HTTPS proxy 测试（rebase 前就失败）、`pkg/aria2/rpc` 测试（需要本地 aria2 服务） +- Branch `feat/dynamic-frontend` HEAD 推至 `6b3ce577` + +--- + ## Architecture Notes ### Upload Data Flow (Current) From a708d07b03adfb3b5cb87f75ff7dad6d62097391 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 17 May 2026 17:46:03 +0800 Subject: [PATCH 065/107] =?UTF-8?q?test(fs):=20pin=20merge=20skip-decision?= =?UTF-8?q?=20contract=20=E2=80=94=20subdirs=20are=20never=20skipped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge skip rule at copy_move.go:221 has two halves whose combination is load-bearing: 1. existingDstFilesFn must only include FILES (not dirs) from dst 2. the call-site guard `!obj.IsDir() && existedObjs[name]` must keep the !obj.IsDir() clause If either half drifts (e.g. dirs leak into existedObjs, or the IsDir guard is removed), src subdirectories matching dst dirs of the same name would be silently skipped — every file inside them would never get copied. That bug is invisible to "did the task complete?" checks; only shows up as quietly missing deep files. New tests: - TestMergeSkipDecision_DirsAreNeverSkipped: 7-case matrix covering every src kind × dst state combo, including the cross-type name collisions (src dir / dst file and vice versa) - TestMergeSkipDecision_EmptyDst: nothing skipped against empty dst - TestMergeSkipDecision_DeepTreeRecursionContract: 3-level src tree against partial dst, asserts correct skip/spawn at each level — the canonical "deep files missing" regression guard --- internal/fs/copy_move_test.go | 218 ++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go index cf77c1d794..37869604bc 100644 --- a/internal/fs/copy_move_test.go +++ b/internal/fs/copy_move_test.go @@ -262,6 +262,224 @@ func TestExistingDstFilesFn_LargeDst(t *testing.T) { } } +// shouldSkipForMerge mirrors the skip decision at copy_move.go:221 so the +// test below pins both the helper (existingDstFilesFn) AND the call-site +// guard as a single contract. Any future refactor that touches either +// half of this contract will be caught. +func shouldSkipForMerge(srcObj model.Obj, existedFiles map[string]bool) bool { + return !srcObj.IsDir() && existedFiles[srcObj.GetName()] +} + +// TestMergeSkipDecision_DirsAreNeverSkipped is the explicit anti-regression +// test for the worry "when a subdir exists in dst, the src subdir is +// skipped and its contents are not merged". It walks every (src kind, dst +// state) combination and asserts the skip decision. +// +// Why this matters: if existingDstFilesFn ever started including dirs, or +// if the call-site guard dropped the `!obj.IsDir()` clause, src subdirs +// matching dst subdirs would silently stop recursing — and every file +// inside them would never get copied. That bug is invisible to a casual +// "did the task complete?" check and only shows up as quietly missing +// deep files. Lock it down. +func TestMergeSkipDecision_DirsAreNeverSkipped(t *testing.T) { + // dst already contains a mix: one file, two dirs. + dstContents := []model.Obj{ + fileObj("root_file.txt"), + dirObj("subA"), + dirObj("subB"), + } + rec := newListRecorder(func(string) ([]model.Obj, error) { return dstContents, nil }) + + existed, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Sanity: the dirs in dst must NOT be in existedObjs. If this fails, + // every src dir matching a dst dir name would be skipped. + if existed["subA"] || existed["subB"] { + t.Fatalf("CRITICAL: dirs in dst leaked into existed-files map — "+ + "matching src subdirs would be skipped and lose their contents. "+ + "got %v", existed) + } + if !existed["root_file.txt"] { + t.Fatalf("file in dst missing from existed map: %v", existed) + } + + // Exhaustive skip-decision matrix for every src-object kind against + // the populated existedObjs. + cases := []struct { + name string + srcObj model.Obj + wantSkip bool + why string + }{ + { + name: "src_file_matches_dst_file", + srcObj: fileObj("root_file.txt"), + wantSkip: true, + why: "resume semantics: already-uploaded file is skipped", + }, + { + name: "src_dir_matches_dst_dir", + srcObj: dirObj("subA"), + wantSkip: false, + why: "MUST recurse into matching subdir to merge its contents", + }, + { + name: "src_dir_matches_dst_dir_B", + srcObj: dirObj("subB"), + wantSkip: false, + why: "MUST recurse — every dst dir must trigger recursion regardless of name", + }, + { + name: "src_dir_no_match", + srcObj: dirObj("brand_new_dir"), + wantSkip: false, + why: "new dir → recurse and create", + }, + { + name: "src_file_no_match", + srcObj: fileObj("brand_new_file.txt"), + wantSkip: false, + why: "new file → upload", + }, + { + name: "src_dir_matches_dst_FILE_name", + srcObj: dirObj("root_file.txt"), + wantSkip: false, + why: "even with a name collision against a dst FILE, src dir must NOT be skipped " + + "(the conflict will surface later when MakeDir runs, not silently)", + }, + { + name: "src_file_matches_dst_DIR_name", + srcObj: fileObj("subA"), + wantSkip: false, + why: "src file colliding with a dst dir name must NOT be skipped " + + "(dirs are not in existed map, and op.Put will surface the conflict)", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := shouldSkipForMerge(c.srcObj, existed) + if got != c.wantSkip { + t.Fatalf("skip(src=%s,IsDir=%v) = %v, want %v — %s", + c.srcObj.GetName(), c.srcObj.IsDir(), got, c.wantSkip, c.why) + } + }) + } +} + +// TestMergeSkipDecision_EmptyDst: with no existedObjs, NOTHING is skipped +// regardless of obj kind — every src item gets a sub-task. +func TestMergeSkipDecision_EmptyDst(t *testing.T) { + existed := map[string]bool{} + for _, obj := range []model.Obj{ + fileObj("a.txt"), + dirObj("d1"), + fileObj("b.bin"), + dirObj("d2"), + } { + if shouldSkipForMerge(obj, existed) { + t.Errorf("nothing should be skipped against empty dst, but %s was", obj.GetName()) + } + } +} + +// TestMergeSkipDecision_DeepTreeRecursionContract simulates a 3-level src +// tree against a partial dst and asserts that the dir-recursion contract +// holds at every level. This is the "deep dir files missing" regression +// guard the user asked for. +func TestMergeSkipDecision_DeepTreeRecursionContract(t *testing.T) { + // Three levels of nesting, with files at each level. Dst already has + // the dir skeleton from a previous interrupted run, plus one file at + // the deepest level (simulating partial completion). + level0Src := []model.Obj{fileObj("root.txt"), dirObj("L1")} + level1Src := []model.Obj{fileObj("a.txt"), dirObj("L2")} + level2Src := []model.Obj{fileObj("deep1.txt"), fileObj("deep2.txt")} + + // Dst state per level + level0Dst := []model.Obj{fileObj("root.txt"), dirObj("L1")} // root.txt already uploaded + level1Dst := []model.Obj{dirObj("L2")} // L1 exists, no files yet + level2Dst := []model.Obj{fileObj("deep1.txt")} // deep1 already uploaded + + cases := []struct { + level string + srcObjs []model.Obj + dstObjs []model.Obj + mustSkip []string + mustSpawn []string + }{ + { + level: "L0", + srcObjs: level0Src, + dstObjs: level0Dst, + mustSkip: []string{"root.txt"}, // file already there + mustSpawn: []string{"L1"}, // dir must recurse + }, + { + level: "L1", + srcObjs: level1Src, + dstObjs: level1Dst, + mustSkip: []string{}, // a.txt not in dst + mustSpawn: []string{"a.txt", "L2"}, // upload file + recurse dir + }, + { + level: "L2", + srcObjs: level2Src, + dstObjs: level2Dst, + mustSkip: []string{"deep1.txt"}, // already uploaded + mustSpawn: []string{"deep2.txt"}, // must still upload + }, + } + + for _, c := range cases { + t.Run(c.level, func(t *testing.T) { + rec := newListRecorder(func(string) ([]model.Obj, error) { return c.dstObjs, nil }) + existed, err := existingDstFilesFn(context.Background(), rec.listDst, "/dst/"+c.level) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var spawned, skipped []string + for _, obj := range c.srcObjs { + if shouldSkipForMerge(obj, existed) { + skipped = append(skipped, obj.GetName()) + } else { + spawned = append(spawned, obj.GetName()) + } + } + + if !sameStrings(skipped, c.mustSkip) { + t.Errorf("%s: skipped = %v, want %v", c.level, skipped, c.mustSkip) + } + if !sameStrings(spawned, c.mustSpawn) { + t.Errorf("%s: spawned = %v, want %v", c.level, spawned, c.mustSpawn) + } + }) + } +} + +func sameStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + m := map[string]int{} + for _, s := range a { + m[s]++ + } + for _, s := range b { + m[s]-- + } + for _, v := range m { + if v != 0 { + return false + } + } + return true +} + // 12. Sanity: the helper invokes listDst exactly once with the given // dstPath (no double-listing). func TestExistingDstFilesFn_CallsListOnce(t *testing.T) { From edcf676e2bc384faa22e6d78aeeb571f55cbcd1f Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 19 May 2026 21:18:32 +0800 Subject: [PATCH 066/107] docs: log 2026-05-19 rebase onto hybrid_cache extraction + prefetch squash --- JOURNAL.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 25523f86c8..40e75604e5 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -384,6 +384,87 @@ BFS precreate 在功能上已变成纯性能优化（提前批建 + 缓存预热 --- +## 2026-05-19 — Rebase onto `hybrid_cache` 包抽取 + prefetch 双 commit squash + +### 背景 + +上游 PR #2477（commit `9eae6258`）做了一次破坏性重构： + +- 把 `internal/mem`（`HybridCache` / `LinearMemory`）抽到独立的 `internal/hybrid_cache` 包（别名 `hcache`） +- 引入 `BackingStore` 抽象：`BufferStore`（内存）+ `FileStore`（磁盘），由 `HybridCache` 内部自动切换，外部代码不再分支 +- 配置字段重命名：`conf.CacheThreshold` → `conf.AutoMemoryLimit`（json `cache_threshold` → `auto_memory_limit`、env 同步） +- 删除 `pkg/buffer/bytes.go` / `pkg/buffer/file.go`，`pkg/buffer/buffer.go` + `type.go` 重塑 +- `internal/stream/util.go` 从 ~960 行精简至 298 行——上游只保留 `GetRangeReaderFromLink` / `CacheFullAndHash` / `NewStreamSectionReader` / `hybridSectionReader` 等核心 helpers，所有本地扩展需要重新落上去 + +同批次另有两个修复（`daad21ef` 139 driver `Connection` 头、`31b41f99` qBittorrent 5.2 204 登录修复），均不与本地冲突。 + +### 行动 1：Rebase 20 个本地 commit 到 `op/main` @ `31b41f99` + +`git tag pre-rebase-2026-05-19 HEAD` 后 `git rebase op/main`，命中两处冲突： + +**冲突点 1 — `internal/stream/stream.go`（`fccab338`）** + +上游把 `cache(maxCacheSize int64)` 重命名为 `ensureCache(size int64)`，函数体已转为新 `hcache.HybridCache`。处理：直接取上游版，本地 fccab338 patch 的语义已被上游覆盖。 + +**冲突点 2 — `internal/stream/util.go`（`fccab338`）** + +本地有独立的 `byteSectionReader` 分支（小文件走单独缓冲池）。上游新设计下 `BackingStore` 自动按文件大小选择 `BufferStore` / `FileStore`，`NewStreamSectionReader` 简化为： + +```go +if file.GetFile() != nil { return &cachedSectionReader{...} } +// 否则全部走 +return &hybridSectionReader{...} +``` + +处理：删除 `byteSectionReader` / `bytesRefReadSeeker` 类型，以及 `TestHybridSectionReader_NonPrefetchPath` 测试（断言"小文件不走 hybrid"已失效）。 + +**冲突点 3 — `internal/stream/util.go`（`7995dbdb` Pass 2 prefetch）** + +`hybridSectionReader` 结构体内的 `hc` 字段：本地 `*mem.HybridCache` vs 上游 `*hcache.HybridCache`。处理：保留本地新增的 `fileSize int64` 字段（Pass 2 prefetch 用于 EOF clamp），类型切到 `*hcache.HybridCache`。 + +**冲突点 4 — `internal/stream/section_reader_prefetch_test.go`** + +`conf.CacheThreshold` 三处引用全部改成 `conf.AutoMemoryLimit`。 + +其余 16 个 commit 干净回放（包括 `163e5c81` 的 util.go selfHealingReadCloser EOF 调整、`07410ad3` 的 Pass 1 hash prefetch、`4e91d93a` 的 `NewOSSUploadHttpClient` 等），上游 patch 已覆盖大部分 `mem.` → `hcache.` 翻译工作。 + +### 行动 2：Squash 两个 prefetch commit + +`588b7024 perf(stream): add double-buffer prefetch to StreamHashFile` 和 `a32544fa perf(stream): pass 2 prefetch on hybridSectionReader` 同属"上传 pipeline 重叠优化"主题但相隔 5 个 commit。用 `git rebase -i op/main` 加自定义 `GIT_SEQUENCE_EDITOR` / `GIT_EDITOR`（Python 脚本临时丢在 `.git/`）做 reorder + squash，得到 `01364250 perf(stream): two-pass prefetch on upload pipeline`，叙事完整。 + +之间的 4 个非 stream commit（`8a2fb995` / `9389b219` / `610c3cf9` / `65c713a0`）reorder 时无冲突——它们碰的是 driver 和 `server/static`，不沾 util.go。 + +未 squash 的 `902c8b82 feat(stream)` 是综合提交（selfHealing + link refresh + hash 工具 + 当时被丢弃的旧 prefetch），独立保留以维持叙事。 + +### 测试结果 + +| 包 | 结果 | +|---|---| +| `internal/stream` | ok（7 个 `TestHybridSectionReader_*` prefetch、2 个 `TestRefreshableRangeReader_*`、2 个 `TestSelfHealingReadCloser_*`、`TestStreamHashFile_SeekablePrefetchProducesSameHash` 全部通过） | +| `internal/fs` | ok | +| `internal/op` | ok | +| `drivers/115_open` | ok（含 `TestNewOSSUploadHttpClientHasLongerTimeout`） | +| `internal/hybrid_cache` | ok | +| `internal/frontend` | ok | +| `internal/net` | 唯一失败 `TestNewOSSClientUsesEnvironmentHTTPSProxy`（白名单，rebase 前就失败） | + +### 行动后状态 + +- `feat/dynamic-frontend` HEAD `39c74672`，19 commits ahead of `op/main`（原 20，squash -1） +- `git push --force-with-lease origin feat/dynamic-frontend` 成功（`7e140903...39c74672`） +- 本地 tag `pre-rebase-2026-05-19` 保留作为回滚点 +- `env.md` 更新 "Current upstream base" 到 `31b41f99` + +### 设计要点：byteSectionReader 消失为何不损失功能 + +旧 `byteSectionReader` 的优化逻辑：小文件用 `pool.Pool[[]byte]` 复用 buffer，避免每个 section 都分配新切片。 + +上游 `BackingStore` 的等价机制：`BufferStore` 内部就是块池化（按 `blockSize` 分块，块从 `LinearMemory` 复用），且自动按 `AutoMemoryLimit`/文件大小决策。也就是说"小文件走纯内存"这个语义保留了，只是不再暴露为独立类型。 + +代价：失去了"按文件大小选择 reader 实现"的显式分支，调试时需要看 `BackingStore` 内部状态。换取：调用方代码统一、Pass 2 prefetch 可以盲目挂在 `hybridSectionReader` 上不用考虑 byte 分支。 + +--- + ## Architecture Notes ### Upload Data Flow (Current) @@ -394,7 +475,7 @@ Pass 1: Hash Calculation (with prefetch) chunk N+1: async prefetch (network I/O) ← overlapped Pass 2: Multipart Upload (with prefetch) - directSectionReader.GetSectionReader + hybridSectionReader.GetSectionReader chunk N: upload to cloud (network I/O) chunk N+1: async prefetch (network I/O) ← overlapped ``` From e0b854d9641f43d3caa7424972917fa6dc518b26 Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 19 May 2026 21:18:47 +0800 Subject: [PATCH 067/107] fix(115_open): bind link cache TTL to CDN ?t= expiry timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 115 CDN signed URLs carry a Unix-timestamp `t=...` query parameter that marks when the URL stops accepting requests — typically just a few minutes, well below OpenList's default link-cache TTL (30 min). When the cache outlives the URL, OP serves a stale URL and 115 responds with HTTP 200 + Content-Length: 0, which clients (mpv, ffmpeg, …) see as a corrupt or empty stream. Since 115's link cache is keyed per User-Agent (`LinkCacheMode: LinkCacheUA`), the failure is sticky and per-UA: the "libmpv" entry stays poisoned while other UAs keep working through the webpage, masking the bug as "only mpv fails". Parse the `t=` parameter and set `model.Link.Expiration` so OP refreshes the link before the CDN actually rejects. A 60-second safety margin covers clock skew and in-flight requests; any timestamp at-or-near expiry clamps to a 1-second minimum so the cache entry evicts on next lookup instead of being held indefinitely. 10 unit tests pin the helper contract (future / past / about-to-expire / missing / malformed / empty / negative / invalid-URL / real-world / safety-margin). --- drivers/115_open/driver.go | 12 ++- drivers/115_open/link_expiry.go | 40 +++++++ drivers/115_open/link_expiry_test.go | 150 +++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 drivers/115_open/link_expiry.go create mode 100644 drivers/115_open/link_expiry_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 9d9ae3add9..b4829c81ec 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -194,12 +194,20 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) totalDuration := time.Since(start) log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) - return &model.Link{ + link := &model.Link{ URL: u.URL.URL, Header: http.Header{ "User-Agent": []string{ua}, }, - }, nil + } + // Tie the cache TTL to the CDN's own `t=` expiry so OP never serves a + // URL that 115's CDN has already invalidated. Without this, OP would + // hand out a dead URL and 115 responds with 200 + Content-Length: 0, + // which downstream clients see as a corrupt/empty stream. + if ttl, ok := parseCDNExpiry(u.URL.URL); ok { + link.Expiration = &ttl + } + return link, nil } func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { diff --git a/drivers/115_open/link_expiry.go b/drivers/115_open/link_expiry.go new file mode 100644 index 0000000000..f92f401c4a --- /dev/null +++ b/drivers/115_open/link_expiry.go @@ -0,0 +1,40 @@ +package _115_open + +import ( + "net/url" + "strconv" + "time" +) + +// 115 CDN download URLs carry `?t=` marking when the signed URL stops +// serving bytes. The real lifetime is often a few minutes, well below +// OpenList's default link-cache TTL — when the cache outlives the URL, OP +// hands out an expired CDN link and the response is "200 OK + empty body". +// +// parseCDNExpiry extracts that timestamp and turns it into a TTL suitable +// for model.Link.Expiration, with a safety margin so OP refreshes slightly +// before the CDN actually rejects. +const ( + cdnExpirySafetyMargin = 60 * time.Second + cdnExpiryMinimum = 1 * time.Second +) + +func parseCDNExpiry(rawURL string) (time.Duration, bool) { + parsed, err := url.Parse(rawURL) + if err != nil { + return 0, false + } + tStr := parsed.Query().Get("t") + if tStr == "" { + return 0, false + } + tUnix, err := strconv.ParseInt(tStr, 10, 64) + if err != nil { + return 0, false + } + remaining := time.Until(time.Unix(tUnix, 0)) - cdnExpirySafetyMargin + if remaining < cdnExpiryMinimum { + return cdnExpiryMinimum, true + } + return remaining, true +} diff --git a/drivers/115_open/link_expiry_test.go b/drivers/115_open/link_expiry_test.go new file mode 100644 index 0000000000..4a19686ce0 --- /dev/null +++ b/drivers/115_open/link_expiry_test.go @@ -0,0 +1,150 @@ +package _115_open + +import ( + "fmt" + "testing" + "time" +) + +// 115 CDN download URLs carry a Unix timestamp in `t=...` that marks when +// the signed URL stops accepting requests. The actual lifetime is typically +// just a few minutes — much shorter than OpenList's default link-cache TTL. +// `parseCDNExpiry` reads `t=` and returns the time-to-live to plug into +// model.Link.Expiration so OP refreshes the link before the CDN does. + +func TestParseCDNExpiry_FutureTimestamp(t *testing.T) { + future := time.Now().Add(30 * time.Minute).Unix() + url := fmt.Sprintf("https://cdnfhnfdfs.115cdn.net/group518/foo.mkv?t=%d&u=123&s=52428800", future) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true for URL with future t=") + } + // Expect 30min minus the safety margin. Allow ±5s slack for the + // time.Now() jitter between Unix() above and time.Until() inside. + want := 30*time.Minute - cdnExpirySafetyMargin + if d < want-5*time.Second || d > want+5*time.Second { + t.Fatalf("d = %v, want ~%v", d, want) + } +} + +func TestParseCDNExpiry_PastTimestamp_ClampedToMinimum(t *testing.T) { + past := time.Now().Add(-10 * time.Minute).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", past) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true even for expired t= (caller decides what to do)") + } + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } +} + +func TestParseCDNExpiry_AboutToExpire_ClampedToMinimum(t *testing.T) { + // 30 seconds in the future, less than safety margin (60s) → would yield + // a negative duration. Should clamp. + soon := time.Now().Add(30 * time.Second).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", soon) + + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false, want true") + } + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } +} + +func TestParseCDNExpiry_MissingT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?u=123&s=52428800") + if ok { + t.Fatalf("ok = true, want false for URL without t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_MalformedT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=notanumber&u=123") + if ok { + t.Fatalf("ok = true, want false for malformed t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_EmptyT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=&u=123") + if ok { + t.Fatalf("ok = true, want false for empty t=") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_NegativeT(t *testing.T) { + d, ok := parseCDNExpiry("https://cdn.example.com/foo?t=-1") + if ok { + // negative parses as Int64 but maps to a past time → ok=true with clamp + if d != cdnExpiryMinimum { + t.Fatalf("d = %v, want clamp to cdnExpiryMinimum=%v", d, cdnExpiryMinimum) + } + } else { + // alternative acceptable behavior: reject negative as malformed. + // Either policy is defensible; pin whichever the implementation chose. + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } + } +} + +func TestParseCDNExpiry_InvalidURL(t *testing.T) { + // url.Parse is permissive — most "garbage in" still parses, but the + // query bag is empty so t= is missing. + d, ok := parseCDNExpiry("not_a_url_at_all") + if ok { + t.Fatalf("ok = true, want false for garbage URL with no query") + } + if d != 0 { + t.Fatalf("d = %v, want 0", d) + } +} + +func TestParseCDNExpiry_RealWorld115URL(t *testing.T) { + // Anchored on the actual URL the user pasted, but with t= rewritten to a + // known future point so the test isn't time-bombed. + future := time.Now().Add(2 * time.Hour).Unix() + url := fmt.Sprintf( + "https://cdnfhnfdfs.115cdn.net/group518/M00/5B/9F/tzyQp1JGFCUAAAAIj4qFUklVFs09176478/Django%%20Unchained%%202012.mkv?t=%d&u=103088508&s=52428800&d=vip-2559104837-cqghjg71avvncddhi-1-100195313&c=2&f=1&k=44423951d519b201c3b42e03556e724b&us=62914560&uc=10&v=1", + future, + ) + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false on real-world URL") + } + if d <= 0 || d > 2*time.Hour { + t.Fatalf("d = %v, want in (0, 2h]", d) + } +} + +func TestParseCDNExpiry_SafetyMarginApplied(t *testing.T) { + // Verify the margin actually shrinks the returned duration. Use a fixed + // offset much larger than the safety margin so jitter doesn't matter. + raw := 1 * time.Hour + future := time.Now().Add(raw).Unix() + url := fmt.Sprintf("https://cdn.example.com/foo?t=%d", future) + d, ok := parseCDNExpiry(url) + if !ok { + t.Fatalf("ok = false") + } + if d >= raw { + t.Fatalf("d = %v, expected strictly less than raw=%v (safety margin not applied)", d, raw) + } + if d < raw-2*cdnExpirySafetyMargin { + t.Fatalf("d = %v, expected ≥ raw-2*margin=%v (margin too aggressive)", d, raw-2*cdnExpirySafetyMargin) + } +} From 52ed5884d7db9f6acfe224322e6760588f9f48bf Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 19 May 2026 21:19:03 +0800 Subject: [PATCH 068/107] fix(stream): treat 200 OK + Content-Length: 0 as expired link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a 115 CDN signed URL expires, the server answers any request with HTTP 200 + Content-Length: 0 (a "soft success") rather than 4xx. The existing selfHealingReadCloser catches this AFTER body read returns (0, io.EOF) and triggers a refresh, but only when a Refresher is set and ServeHTTP has already written response headers — the client gets HTTP 206 + Content-Length: ${expected} headers immediately followed by zero bytes, breaking range-aware clients that pre-allocate buffers. Move the detection up to the request level: when the closure in GetRangeReaderFromLink asks for ≥1 byte but the response promises 0, return an "expired" error before ever exposing the empty body. The RefreshableRangeReader path now refreshes one step earlier, and callers without a Refresher fail loudly instead of streaming silence. ContentLength == -1 (chunked transfer) is excluded so legitimately chunked responses are not flagged. 4 tests cover: - soft-expired + Refresher → refresh fires once, fresh body delivered - soft-expired + no Refresher → IsLinkExpiredError-classified error - healthy 206 → no false positive - chunked transfer → no false positive --- internal/stream/util.go | 13 ++ internal/stream/util_test.go | 226 +++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) diff --git a/internal/stream/util.go b/internal/stream/util.go index 6dffcc5c70..e097b3cc3b 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -391,6 +391,19 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, } return nil, fmt.Errorf("http request failure, err:%w", err) } + // "Soft 200" expired-link guard: when we asked for ≥1 byte but the + // server promised 0 (e.g. 115 CDN for a stale `?t=` URL — 200 OK + + // Content-Length: 0), the body would be empty and any downstream + // client would interpret it as a corrupt/empty file. Surface this + // as an explicit "expired" error so RefreshableRangeReader can + // trigger a refresh, and callers without a Refresher fail loudly + // instead of streaming silence. ContentLength == -1 means the + // server used chunked transfer encoding and is excluded. + if httpRange.Length > 0 && response.ContentLength == 0 { + response.Body.Close() + return nil, fmt.Errorf("link expired: server returned status %d with Content-Length: 0 (expected %d bytes from %s)", + response.StatusCode, httpRange.Length, link.URL) + } if ServerDownloadLimit != nil { response.Body = &RateLimitReader{ Ctx: ctx, diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go index c39ccbfb4f..4bdd1f5068 100644 --- a/internal/stream/util_test.go +++ b/internal/stream/util_test.go @@ -6,14 +6,28 @@ import ( "encoding/hex" "errors" "io" + "net/http" + "net/http/httptest" + "strconv" "sync" + "sync/atomic" "testing" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) +// ensureConfForHTTP makes sure conf.Conf is non-nil before any test reaches +// net.HttpClient — that helper deferences conf.Conf.TlsInsecureSkipVerify +// during a sync.Once init and panics on a nil pointer otherwise. +func ensureConfForHTTP() { + if conf.Conf == nil { + conf.Conf = &conf.Config{} + } +} + func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { data := []byte("0123456789abcdef") var refreshes int @@ -260,6 +274,218 @@ func (f *flakyReadCloser) Close() error { return nil } +// TestRangeReaderFromLink_SoftExpiredLink_TriggersRefresh covers the +// 115 CDN "soft 200 + empty body" failure mode: when a signed URL has +// expired, 115 still answers with HTTP 200 and Content-Length: 0 instead +// of a 4xx. Without explicit detection, OP forwards an empty stream to +// the client (mpv sees "Failed to recognize file format"). This test +// pins the contract that GetRangeReaderFromLink-derived readers treat +// "non-zero range requested → zero bytes promised" as an expired link +// and let RefreshableRangeReader trigger a refresh + retry. +func TestRangeReaderFromLink_SoftExpiredLink_TriggersRefresh(t *testing.T) { + ensureConfForHTTP() + data := []byte("The quick brown fox jumps over the lazy dog") + size := int64(len(data)) + + var expiredHits int32 + expired := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&expiredHits, 1) + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusOK) + })) + defer expired.Close() + + var freshHits int32 + fresh := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&freshHits, 1) + rangeHeader := r.Header.Get("Range") + start := int64(0) + length := size + if rangeHeader != "" { + ranges, err := http_range.ParseRange(rangeHeader, size) + if err == nil && len(ranges) == 1 { + start = ranges[0].Start + length = ranges[0].Length + w.Header().Set("Content-Range", ranges[0].ContentRange(size)) + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : start+length]) + return + } + } + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data[start : start+length]) + })) + defer fresh.Close() + + link := &model.Link{URL: expired.URL} + var refreshes int32 + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return &model.Link{URL: fresh.URL}, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("body mismatch: got %q, want %q", got, data) + } + if atomic.LoadInt32(&refreshes) != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + if atomic.LoadInt32(&expiredHits) < 1 { + t.Fatalf("expired server never hit (= %d), expected at least once", expiredHits) + } + if atomic.LoadInt32(&freshHits) < 1 { + t.Fatalf("fresh server never hit (= %d), expected at least once after refresh", freshHits) + } +} + +// TestRangeReaderFromLink_NormalResponse_NoFalsePositive guards against the +// soft-expired check ever firing on a healthy 206 response. +func TestRangeReaderFromLink_NormalResponse_NoFalsePositive(t *testing.T) { + ensureConfForHTTP() + data := []byte("0123456789abcdef") + size := int64(len(data)) + + var refreshes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ranges, _ := http_range.ParseRange(r.Header.Get("Range"), size) + if len(ranges) == 1 { + ra := ranges[0] + w.Header().Set("Content-Range", ra.ContentRange(size)) + w.Header().Set("Content-Length", strconv.FormatInt(ra.Length, 10)) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[ra.Start : ra.Start+ra.Length]) + return + } + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + _, _ = w.Write(data) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return link, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 4, Length: 6}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data[4:10]) { + t.Fatalf("body mismatch: got %q, want %q", got, data[4:10]) + } + if atomic.LoadInt32(&refreshes) != 0 { + t.Fatalf("refreshes = %d, want 0 (healthy response must not trigger refresh)", refreshes) + } +} + +// TestRangeReaderFromLink_ChunkedResponse_NoFalsePositive guards against +// false positives on responses without a Content-Length (chunked transfer). +// Go's http.Response sets ContentLength = -1 for chunked, which must not +// be treated as "0 bytes promised". +func TestRangeReaderFromLink_ChunkedResponse_NoFalsePositive(t *testing.T) { + ensureConfForHTTP() + data := []byte("chunked-payload-bytes-here-yo!") + size := int64(len(data)) + + var refreshes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Force chunked: do not set Content-Length, write headers then body + // in two flushes. + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + _, _ = w.Write(data[:len(data)/2]) + if flusher != nil { + flusher.Flush() + } + _, _ = w.Write(data[len(data)/2:]) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + atomic.AddInt32(&refreshes, 1) + return link, nil, nil + } + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + rc, err := rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err != nil { + t.Fatalf("RangeRead: %v", err) + } + defer rc.Close() + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("body mismatch: got %q, want %q", got, data) + } + if atomic.LoadInt32(&refreshes) != 0 { + t.Fatalf("refreshes = %d, want 0 (chunked response must not be treated as expired)", refreshes) + } +} + +// TestRangeReaderFromLink_SoftExpiredLink_NoRefresher_ReturnsError verifies +// that when a soft-expired link is encountered without a Refresher set, +// the error surfaces cleanly to the caller instead of returning an empty +// body that the client then misinterprets as a corrupt file. +func TestRangeReaderFromLink_SoftExpiredLink_NoRefresher_ReturnsError(t *testing.T) { + ensureConfForHTTP() + size := int64(100) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + link := &model.Link{URL: server.URL} // no Refresher + + rrr, err := GetRangeReaderFromLink(size, link) + if err != nil { + t.Fatalf("GetRangeReaderFromLink: %v", err) + } + _, err = rrr.RangeRead(context.Background(), http_range.Range{Start: 0, Length: size}) + if err == nil { + t.Fatalf("RangeRead returned nil error; expected an 'expired link' error so callers see a real failure instead of an empty stream") + } + if !IsLinkExpiredError(err) { + t.Fatalf("error %q is not classified as expired by IsLinkExpiredError", err) + } +} + func sliceForRange(data []byte, httpRange http_range.Range) []byte { start := int(httpRange.Start) length := int(httpRange.Length) From 113b6aefcf52ca0c3cfd7d83357b57cc8e18dc90 Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 19 May 2026 21:48:47 +0800 Subject: [PATCH 069/107] fix(115_open): driver.Get defers files to GetFiles path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libmpv "Failed to recognize file format" symptom came from one underlying mistake: 115's GetFolderInfoByPath endpoint (yuque rl8zrhe2nag21dfw, "Get folder details") is folder-oriented. Its struct carries Count / FolderCount / PlayLong and the Size field is byte-accurate only for directories; for FILE paths Size is empty or "0". driver.Get nevertheless called it for every path. The resulting Obj had FS = 0, op.Get returned it to ServeHTTP, and the empty-file + Range branch in internal/net/serve.go answered with 200 OK + Content-Length: 0. mpv couldn't recognize the empty stream. Webpage playback masked the bug because op.List warmed dirCache with GetFilesResp_File entries (FS arrives as int64 there and is always correct). mpv direct access skipped op.List, hit driver.Get cold, and broke. Fix: - Introduce fromFolderInfo gateway used by driver.Get: * folder → return Obj from folderInfoToObj (fast path, single API call) * file → return errs.NotImplement so op.Get falls through to its list-based path, which pulls FS out of GetFilesResp_File.FS (int64, reliable) - folderInfoToObj extracts only the fields the gateway actually uses (FileID, FileName, FileCategory, Sha1, PickCode). Size is not parsed — files are rejected before any caller would read FS. - log.Debugf in driver.Get prints the raw resp.Size / FileCategory so future diagnoses don't need to theorize. Trade-off: cold-cache file access costs 3 WaitLimit-gated SDK calls (wasted GetFolderInfoByPath on the file + GetFolderInfoByPath on the parent for its FileID + GetFiles to list the parent) plus the eventual DownURL = 4 calls. With default limit_rate=1 req/s that's ~3s of pure rate-limit wait. Steady-state access within dirCache TTL (5 min) collapses back to a single DownURL. Bumping limit_rate to 5–10 in the storage config keeps the worst case under 1s. Verified in production with curl + libmpv UA + Range: bytes=0-1023 → 206 Partial Content / Content-Length: 1024 / Content-Range bytes 0-1023/36767958354 / Last-Modified populated (proving the Obj came from the List path, not the bogus folderInfoToObj of an earlier attempt). 5 unit tests pin the contract: folder fast path, file/folder gateway behavior, and an existing TestGetReturnsObjForExistingFolder updated to set file_category="0" so it actually exercises the folder branch. --- drivers/115_open/driver.go | 10 ++---- drivers/115_open/driver_test.go | 7 ++-- drivers/115_open/get.go | 46 +++++++++++++++++++++++++ drivers/115_open/get_test.go | 59 +++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 drivers/115_open/get.go create mode 100644 drivers/115_open/get_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index b4829c81ec..982eef6d9e 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -222,13 +222,9 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { } return nil, err } - return &Obj{ - Fid: resp.FileID, - Fn: resp.FileName, - Fc: resp.FileCategory, - Sha1: resp.Sha1, - Pc: resp.PickCode, - }, nil + log.Debugf("[115] GetFolderInfoByPath(%s) => Size=%q FileCategory=%q FileID=%s", + path, resp.Size, resp.FileCategory, resp.FileID) + return fromFolderInfo(resp) } func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 27eeb3d25e..4898103271 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -849,9 +849,10 @@ func TestCheckUploadCallbackInvalidJSON(t *testing.T) { func TestGetReturnsObjForExistingFolder(t *testing.T) { driver, _ := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { writeSDKSuccess(t, w, map[string]any{ - "file_id": "99999", - "file_name": "my_folder", - "pick_code": "pc-123", + "file_id": "99999", + "file_name": "my_folder", + "pick_code": "pc-123", + "file_category": "0", // folder; Fix 4 rejects file responses with NotImplement }) }) driver.parentPath = "" diff --git a/drivers/115_open/get.go b/drivers/115_open/get.go new file mode 100644 index 0000000000..d36f0c97fe --- /dev/null +++ b/drivers/115_open/get.go @@ -0,0 +1,46 @@ +package _115_open + +import ( + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// folderInfoToObj extracts the fields driver.Get needs from a +// GetFolderInfoByPath response. We only look at FileID + FileName + +// FileCategory — Sha1 / PickCode come along because the struct already +// carries them, but Size is intentionally dropped: the API is +// folder-oriented and resp.Size is empty/garbage for file paths. +// fromFolderInfo rejects files outright so the parsed value would never +// be read anyway. +func folderInfoToObj(resp *sdk.GetFolderInfoResp) *Obj { + return &Obj{ + Fid: resp.FileID, + Fn: resp.FileName, + Fc: resp.FileCategory, + Sha1: resp.Sha1, + Pc: resp.PickCode, + } +} + +// fromFolderInfo is the gateway used by driver.Get. Files are rejected +// with errs.NotImplement so op.Get falls through to its list-based +// path — that response carries GetFilesResp_File.FS (int64) and is +// always correct. Folders take the fast path: one +// GetFolderInfoByPath call is enough to build the Obj that +// op.list will pass to driver.List as the parent directory. +// +// Trade-off: cold-cache file access pays one wasted +// GetFolderInfoByPath (this call) + one folder GetFolderInfoByPath +// (parent) + one GetFiles (list parent) + the eventual DownURL = 4 +// WaitLimit-gated SDK calls. Default limit_rate is 1 req/s so this +// is ~3s of pure rate-limit wait on top of network. Steady-state +// access within dirCache TTL (5 min) collapses back to a single +// DownURL call. +func fromFolderInfo(resp *sdk.GetFolderInfoResp) (model.Obj, error) { + obj := folderInfoToObj(resp) + if !obj.IsDir() { + return nil, errs.NotImplement + } + return obj, nil +} diff --git a/drivers/115_open/get_test.go b/drivers/115_open/get_test.go new file mode 100644 index 0000000000..638e65b69f --- /dev/null +++ b/drivers/115_open/get_test.go @@ -0,0 +1,59 @@ +package _115_open + +import ( + "errors" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/errs" +) + +// driver.Get's only job after Fix 4 is "is this a folder, and if so what's +// its FileID/name". Files are rejected with errs.NotImplement so op.Get +// falls back to the list-based path — that response carries +// GetFilesResp_File.FS as int64 and is always correct. The 115 +// "Get folder info" API (yuque rl8zrhe2nag21dfw) is folder-oriented and +// resp.Size is unreliable for file paths anyway. + +func TestFolderInfoToObj_Folder(t *testing.T) { + resp := &sdk.GetFolderInfoResp{ + FileID: "1234", FileName: "Movies", FileCategory: "0", + } + obj := folderInfoToObj(resp) + if !obj.IsDir() { + t.Fatalf("IsDir() = false, want true (FileCategory 0 = folder)") + } + if obj.GetID() != "1234" || obj.GetName() != "Movies" { + t.Fatalf("obj = %+v, want id=1234 name=Movies", obj) + } +} + +func TestFromFolderInfo_FileReturnsNotImplement(t *testing.T) { + // File path response: even with sha1/pickcode populated, fast-path + // must defer to list — Size is not byte-accurate for files. + resp := &sdk.GetFolderInfoResp{ + FileID: "3370024725891437210", FileName: "Django Unchained 2012.mkv", + FileCategory: "1", Sha1: "17C6301550DE8E22D477EB9BA3901A99B9961494", + PickCode: "cqghjg71avvncddhi", Size: "36767958354", + } + obj, err := fromFolderInfo(resp) + if obj != nil { + t.Fatalf("obj = %+v, want nil (files must defer to List)", obj) + } + if !errors.Is(err, errs.NotImplement) { + t.Fatalf("err = %v, want errs.NotImplement (so op.Get falls through to list path)", err) + } +} + +func TestFromFolderInfo_FolderReturnsObj(t *testing.T) { + resp := &sdk.GetFolderInfoResp{ + FileID: "1234", FileName: "Movies", FileCategory: "0", + } + obj, err := fromFolderInfo(resp) + if err != nil { + t.Fatalf("unexpected err = %v", err) + } + if obj == nil || !obj.IsDir() || obj.GetID() != "1234" { + t.Fatalf("obj = %+v, want non-nil folder with ID 1234", obj) + } +} From 8392b003c5a5c4f65d76b6e7af4823b671d4350d Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 19 May 2026 22:18:20 +0800 Subject: [PATCH 070/107] ci(docker): auto-trigger on feat/dynamic-frontend + per-variant image names Two changes that go together because the second was useless without the first. Push to feat/dynamic-frontend (the active branch) was not in the trigger list, so commits like Fix 3 b80688c9 never produced a fresh :front-rolling image -- the user kept pulling stale binaries and the mpv "Failed to recognize file format" symptom appeared unfixed when in fact CI had simply not run. - Replace push:main + pull_request:copy triggers with push:feat/dynamic-frontend so every commit on the active branch rebuilds the four docker variants. - Drop the shared IMAGE_NAME env and give each matrix row its own image_name: openlist, openlist-ffmpeg, openlist-aria2, openlist-aio. Tags collapse from openlist:front-rolling-{variant} to {image_name}:front-rolling, so the pull path matches the image intent (ghcr.io/.../openlist-aio:front-rolling, etc.). - :latest now applies to each image when its frontend_channel is "latest", rather than only openlist:latest. - Build cache key reuses the per-variant image_name so caches don't collide across variants. --- .github/workflows/test_docker.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index f1947f4d1b..d21b60e38f 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -18,10 +18,7 @@ on: - both push: branches: - - main - pull_request: - branches: - - copy + - feat/dynamic-frontend concurrency: @@ -32,7 +29,6 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} - IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME_PREFIX: 'binaries_docker_release' # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 @@ -131,27 +127,27 @@ jobs: strategy: matrix: frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} - # 构建所有变体 + # 四种变体，各自独立 image 名（推送到独立 GHCR repo） image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - image_tag_suffix: "" + image_name: "openlist" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - image_tag_suffix: "-ffmpeg" + image_name: "openlist-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - image_tag_suffix: "-aria2" + image_name: "openlist-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - image_tag_suffix: "-aio" + image_name: "openlist-aio" steps: - name: Checkout uses: actions/checkout@v6 @@ -175,10 +171,10 @@ jobs: uses: docker/metadata-action@v5 with: images: | - ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} + ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }} tags: | - type=raw,value=front-${{ matrix.frontend_channel }}${{ matrix.image_tag_suffix }} - type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' && matrix.image == 'latest' }} + type=raw,value=front-${{ matrix.frontend_channel }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' }} - name: Build and push uses: docker/build-push-action@v6 @@ -192,5 +188,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }} - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }}:buildcache-front-${{ matrix.frontend_channel }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ matrix.image_name }}:buildcache-front-${{ matrix.frontend_channel }},mode=max From da8facd1d1a28622ccf2b499903d674e27e48063 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 13:32:27 +0800 Subject: [PATCH 071/107] feat(server): add COOP/COEP headers for SharedArrayBuffer support Required by movi-player's FFmpeg WASM which uses SharedArrayBuffer for multi-threaded decoding. Without these headers, browsers block SharedArrayBuffer and the player fails to initialize. --- server/router.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/router.go b/server/router.go index 7330bd2c33..3d44a9e3da 100644 --- a/server/router.go +++ b/server/router.go @@ -248,6 +248,10 @@ func Cors(r *gin.Engine) { config.AllowHeaders = conf.Conf.Cors.AllowHeaders config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) + r.Use(func(c *gin.Context) { + c.Header("Cross-Origin-Opener-Policy", "same-origin") + c.Header("Cross-Origin-Embedder-Policy", "credentialless") + }) } func InitS3(e *gin.Engine) { From 60b8cc6231960d6a3e80a6dda3bdfe9bf852ce57 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 13:38:01 +0800 Subject: [PATCH 072/107] test(fs): pin skip-existing name-clearing contract for copy/move --- server/handles/fsmanage_test.go | 100 ++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 server/handles/fsmanage_test.go diff --git a/server/handles/fsmanage_test.go b/server/handles/fsmanage_test.go new file mode 100644 index 0000000000..e2de6508c6 --- /dev/null +++ b/server/handles/fsmanage_test.go @@ -0,0 +1,100 @@ +package handles + +import ( + "testing" +) + +// TestClearSkippedNames verifies the contract from upstream fix #2520: +// When SkipExisting is true and a file already exists at the destination, +// that name must be cleared (set to "") so the task loop skips it. +// Valid (non-skipped) names must be set to full srcPath. +// +// This is a contract test for the name-filtering loop in FsMove/FsCopy. +// It tests the logic in isolation without needing gin/fs dependencies. +func TestClearSkippedNames(t *testing.T) { + // Simulate the name-filtering loop logic from FsMove/FsCopy. + // existsAtDst simulates whether a file already exists at the destination. + filterNames := func(srcDir string, names []string, overwrite, skipExisting bool, existsAtDst func(name string) bool) []string { + result := make([]string, len(names)) + copy(result, names) + for i, name := range result { + srcPath := srcDir + "/" + name + // First: set to srcPath (the fix moves this before skip check) + result[i] = srcPath + if !overwrite { + if existsAtDst(name) { + if !skipExisting { + // Would return error in real code + result[i] = "ERROR" + return result + } + // Skip: must clear to "" + result[i] = "" + continue + } + } + } + return result + } + + tests := []struct { + name string + srcDir string + names []string + overwrite bool + skipExisting bool + existsAtDst func(string) bool + want []string + }{ + { + name: "skip existing files clears their names", + srcDir: "/src", + names: []string{"a.mkv", "b.mkv", "c.mkv"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(n string) bool { return n == "b.mkv" }, + want: []string{"/src/a.mkv", "", "/src/c.mkv"}, + }, + { + name: "no skipping when overwrite is true", + srcDir: "/src", + names: []string{"a.mkv", "b.mkv"}, + overwrite: true, + skipExisting: false, + existsAtDst: func(string) bool { return true }, + want: []string{"/src/a.mkv", "/src/b.mkv"}, + }, + { + name: "all files skipped results in all empty", + srcDir: "/src", + names: []string{"x.mp4", "y.mp4"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(string) bool { return true }, + want: []string{"", ""}, + }, + { + name: "no existing files keeps all paths", + srcDir: "/data", + names: []string{"movie.mkv"}, + overwrite: false, + skipExisting: true, + existsAtDst: func(string) bool { return false }, + want: []string{"/data/movie.mkv"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := filterNames(tt.srcDir, tt.names, tt.overwrite, tt.skipExisting, tt.existsAtDst) + if len(got) != len(tt.want) { + t.Fatalf("length mismatch: got %d, want %d", len(got), len(tt.want)) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("names[%d] = %q, want %q", i, got[i], tt.want[i]) + } + } + }) + } +} From de89a81f31176037ed28190417f106e857eb2205 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 15:06:39 +0800 Subject: [PATCH 073/107] fix(frontend): use embedded dist on startup, let watcher upgrade The dynamic frontend cache in the data volume was overriding newer embedded dist from Docker images. On networks where GitHub API is unreachable, the stale cache persisted indefinitely. Now initStatic() always starts with the embedded dist. The watcher still runs in the background and hot-swaps to a newer version from GitHub when available. Also default FrontendRepo to Ironboxplus fork. --- internal/conf/var.go | 2 +- internal/frontend/fetcher_test.go | 117 +++++++++++++++++++++++++++++- server/static/static.go | 19 ++--- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/internal/conf/var.go b/internal/conf/var.go index b11f3086ed..76de345873 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,7 +12,7 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" - FrontendRepoDefault string = "OpenListTeam/OpenList-Frontend" + FrontendRepoDefault string = "Ironboxplus/OpenList-Frontend" ) var ( diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go index bae99f47fb..3ed6d95466 100644 --- a/internal/frontend/fetcher_test.go +++ b/internal/frontend/fetcher_test.go @@ -283,8 +283,8 @@ func TestLegacyConfigJSONGetsDefaultFrontendRepo(t *testing.T) { if err := json.Unmarshal([]byte(`{"site_url":"https://example.com"}`), cfg); err != nil { t.Fatalf("unmarshal legacy config: %v", err) } - if cfg.FrontendRepo != defaultFrontendRepo { - t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, defaultFrontendRepo) + if cfg.FrontendRepo != conf.FrontendRepoDefault { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, conf.FrontendRepoDefault) } } @@ -487,6 +487,119 @@ func TestResolveTagCommitSHA_RealGitHubWithProxy10808(t *testing.T) { } } +func TestStaleCacheOverriddenByNewerRelease(t *testing.T) { + oldFiles := map[string]string{"./dist/index.html": "old"} + oldTar := createTestTarGz(t, oldFiles) + newFiles := map[string]string{"./dist/index.html": "new"} + newTar := createTestTarGz(t, newFiles) + + callCount := 0 + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case fmt.Sprintf("/repos/%s/releases/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling", + "assets": [{"name": "openlist-frontend-dist.tar.gz", "browser_download_url": "%s/download/dist.tar.gz"}] + }`, ts.URL))) + case fmt.Sprintf("/repos/%s/git/ref/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"object":{"type":"commit","sha":"aaaaaaaaaaaabbbbbbbbbbbbccccccccccccdddd"}}`)) + case "/download/dist.tar.gz": + callCount++ + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + if callCount == 1 { + _, _ = w.Write(oldTar) + } else { + _, _ = w.Write(newTar) + } + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + destDir := GetDistPath() + _ = os.MkdirAll(destDir, 0755) + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + // Write a stale cached version + _ = writeVersion("rolling@stale000000000") + _ = os.MkdirAll(filepath.Join(destDir, distDirName), 0755) + _ = os.WriteFile(filepath.Join(destDir, distDirName, "index.html"), []byte("stale"), 0644) + + // Fetch should detect version mismatch and download + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag with stale cache: %v", err) + } + if !result.Downloaded { + t.Error("expected Downloaded=true when cache is stale") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "old" { + t.Errorf("index.html: got %q, want old content", string(data)) + } +} + +func TestWatcherTriggersCallbackOnNewVersion(t *testing.T) { + files := map[string]string{"./dist/index.html": "watcher"} + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case fmt.Sprintf("/repos/%s/releases/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling", + "assets": [{"name": "openlist-frontend-dist.tar.gz", "browser_download_url": "%s/download/dist.tar.gz"}] + }`, ts.URL))) + case fmt.Sprintf("/repos/%s/git/ref/tags/rolling", getFrontendRepo()): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"object":{"type":"commit","sha":"watchertest1234567890watchertest1234567890"}}`)) + case "/download/dist.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + _, _ = w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + destDir := GetDistPath() + _ = os.MkdirAll(destDir, 0755) + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + // The watcher's check() calls FetchFromRolling which uses api.github.com. + // For this test, call fetchFromTag directly and verify Downloaded triggers callback logic. + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + if !result.Downloaded { + t.Fatal("expected Downloaded=true for watcher callback trigger") + } + if result.Version != "rolling@watchertest1" { + t.Errorf("version: got %q", result.Version) + } +} + func TestEnsureDistOnceFailureDoesNotLock(t *testing.T) { ResetFetchState() _ = os.MkdirAll(GetDistPath(), 0755) diff --git a/server/static/static.go b/server/static/static.go index d6bdcb5871..81695c9fb7 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -59,28 +59,17 @@ func (r *reloadableFS) swap(f fs.FS) { var staticFS = &reloadableFS{} func initStatic() { - utils.Log.Debug("Initializing static file system...") - // 1. User explicitly configured dist_dir if conf.Conf.DistDir != "" { staticFS.swap(os.DirFS(conf.Conf.DistDir)) utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) return } - // 2. Try dynamic dist (fetched from rolling release) - if frontend.HasValidDist() { - distPath := filepath.Join(frontend.GetDistPath(), "dist") - staticFS.swap(os.DirFS(distPath)) - utils.Log.Infof("Using dynamically fetched dist: %s", distPath) - return - } - // 3. Fall back to embedded dist; the Watcher (started after initStatic) - // will fetch the rolling release in the background and hot-swap via ReloadStatic. dist, err := fs.Sub(public.Public, "dist") if err != nil { - utils.Log.Fatalf("failed to read dist dir: %v", err) + utils.Log.Fatalf("failed to read embedded dist dir: %v", err) } staticFS.swap(dist) - utils.Log.Debug("Using embedded dist directory") + utils.Log.Infof("Using embedded dist directory") } func replaceStrings(content string, replacements map[string]string) string { @@ -168,8 +157,10 @@ func UpdateIndex() { // ReloadStatic reloads the static files from disk (called by the watcher after an update) func ReloadStatic() { utils.Log.Info("[static] reloading static files after frontend update...") + distPath := filepath.Join(frontend.GetDistPath(), "dist") + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Switched to dynamically fetched dist: %s", distPath) siteConfig := getSiteConfig() - initStatic() initIndex(siteConfig) } From a0e03418e782d4f799eeb51b73fc3851dc7a81a6 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 15:08:15 +0800 Subject: [PATCH 074/107] docs: document frontend dist serving priority and watcher design --- CLAUDE.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index c6d9bf1731..55cdf77a81 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ docker build -f Dockerfile . # Build docker image ``` **Build Script Details** (`build.sh`): -- Fetches frontend from OpenListTeam/OpenList-Frontend releases +- Fetches frontend from `$FRONTEND_REPO` (default: `Ironboxplus/OpenList-Frontend`) releases and embeds into `public/dist/` - Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` - Supports `dev`, `beta`, and release builds - Downloads prebuilt frontend distribution automatically @@ -245,6 +245,34 @@ Handles multiple scenarios: 3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader 4. Transparent proxy (forwards to `link.URL`) +### Frontend Dist Serving + +**Location**: `server/static/static.go`, `internal/frontend/` + +The frontend dist has two sources, with a strict priority: + +1. **Embedded dist** (`public/dist/` via `go:embed`): Baked into the binary at build time. Always used on startup. +2. **Dynamic dist** (fetched by watcher): The `frontend.Watcher` checks GitHub every 30 minutes for a newer rolling release. If found, it downloads to `data/frontend_dist/` and hot-swaps the serving FS via `ReloadStatic()`. + +**Key design rule**: `initStatic()` always starts with the embedded dist (or `dist_dir` if configured). The cached dynamic dist in the data volume is never read on startup — only the watcher can activate it after verifying a newer version exists on GitHub. This prevents stale cache from overriding a newer Docker image. + +**Configuration** (`config.json`): +- `dist_dir`: Override with a custom local directory (highest priority, skips embedded) +- `frontend_repo`: GitHub repo for the watcher to check (default: `Ironboxplus/OpenList-Frontend`) + +**Startup flow**: +``` +initStatic() → embedded dist (or dist_dir) + ↓ +StartWatcher(ReloadStatic) → background goroutine + ↓ (every 30min) +FetchFromRolling() → compare cache version vs GitHub rolling tag commit + ↓ (if newer) +downloadAndExtract() → atomic swap in data/frontend_dist/dist/ + ↓ +ReloadStatic() → swap staticFS to new dist, re-render index.html +``` + ### Startup Sequence **Location**: `internal/bootstrap/run.go` From 91650d18375ccae3e36b2f09a1d8da0923d793e7 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 15:27:08 +0800 Subject: [PATCH 075/107] fix(ci): default frontend repo to Ironboxplus fork, fix rolling cache key - All workflows now default to Ironboxplus/OpenList-Frontend instead of upstream, so builds embed the correct frontend without needing vars. - build.sh default also updated. - test_docker.yml: rolling cache key now includes the release asset name so cache is busted when a new rolling release is published. --- .github/workflows/beta_release.yml | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/test_docker.yml | 11 +++++++---- build.sh | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 496fd0e0b6..ba64b12f42 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -140,7 +140,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling - github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} env: GOFLAGS: ${{ matrix.goflags }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e59c5364e5..ea8fb5a443 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest - github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} output: openlist$ext - name: Verify musl binary is static diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index d21b60e38f..226f1eee5c 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -5,7 +5,7 @@ on: frontend_repo: description: 'Frontend repo, e.g. Ironboxplus/OpenList-Frontend' required: false - default: 'OpenListTeam/OpenList-Frontend' + default: 'Ironboxplus/OpenList-Frontend' type: string frontend_channel: description: 'Frontend release channel to build' @@ -28,14 +28,14 @@ concurrency: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 - FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'Ironboxplus/OpenList-Frontend' }} REGISTRY: ghcr.io ARTIFACT_NAME_PREFIX: 'binaries_docker_release' # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 IMAGE_PUSH: 'true' - # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) + # 👇 使用默认的前端仓库 (Ironboxplus/OpenList-Frontend) # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' jobs: @@ -73,7 +73,10 @@ jobs: if [ "$web_version" = "latest" ]; then frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') else - frontend_version="$web_version" + # For rolling/dev channels, resolve the actual commit to bust stale caches + release_json=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/tags/$web_version" 2>/dev/null || echo '{}') + asset_name=$(echo "$release_json" | jq -r '.assets[0].name // empty') + frontend_version="${web_version}-${asset_name:-unknown}" fi echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" diff --git a/build.sh b/build.sh index d37ddad730..4e7b4f608d 100644 --- a/build.sh +++ b/build.sh @@ -5,7 +5,7 @@ gitAuthor="The OpenList Projects Contributors " gitCommit=$(git log --pretty=format:"%h" -1) # Set frontend repository, default to OpenListTeam/OpenList-Frontend -frontendRepo="${FRONTEND_REPO:-OpenListTeam/OpenList-Frontend}" +frontendRepo="${FRONTEND_REPO:-Ironboxplus/OpenList-Frontend}" githubAuthArgs="" if [ -n "$GITHUB_TOKEN" ]; then From 0c589a0e00b94f8cae337c6afeccb1c4db6d0544 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 27 May 2026 15:28:49 +0800 Subject: [PATCH 076/107] fix(ci): push trigger builds rolling only (no latest release in fork) --- .github/workflows/test_docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 226f1eee5c..0dc51ef0f1 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["rolling"]') }} steps: - name: Checkout uses: actions/checkout@v6 @@ -129,7 +129,7 @@ jobs: packages: write strategy: matrix: - frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["rolling"]') }} # 四种变体，各自独立 image 名（推送到独立 GHCR repo） image: ["latest", "ffmpeg", "aria2", "aio"] include: From 5574dd1c6a6bf673dd848604e94e97473354af67 Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 28 May 2026 02:52:53 +0800 Subject: [PATCH 077/107] docs: consolidate Claude-generated MDs to parent workspace directory Move JOURNAL.md, OVERVIEW.md, COMPATIBILITY_REPORT.md to parent E:\Go\Openlist\. Update CLAUDE.md to workspace-wide aggregated version covering backend, frontend, and movi-player. --- CLAUDE.md | 118 +++++++++- COMPATIBILITY_REPORT.md | 204 ----------------- JOURNAL.md | 486 ---------------------------------------- OVERVIEW.md | 79 ------- 4 files changed, 110 insertions(+), 777 deletions(-) delete mode 100644 COMPATIBILITY_REPORT.md delete mode 100644 JOURNAL.md delete mode 100644 OVERVIEW.md diff --git a/CLAUDE.md b/CLAUDE.md index 55cdf77a81..680d02ba88 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,14 +1,50 @@ -# CLAUDE.md +# OpenList Workspace — CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file covers the entire OpenList workspace, which contains three main repositories: -## Core Development Principles +- **OP/** — OpenList backend (Go, main project) +- **OpenList-Frontend/** — SolidJS frontend +- **movi-player/** — FFmpeg WASM video player (local fork) + +Each section below is labeled by component. The backend section is the primary development guide. + +--- + +## Environment Configuration + +| Item | Value | +|------|-------| +| OS | Windows 11 Pro for Workstations (10.0.26100) | +| Go | go1.25.4 windows/amd64 | +| Node.js | v22.19.0 | +| Project Root | `E:\Go\Openlist\` | + +### Repository Map + +| Directory | Repo | Branch | Remote | +|-----------|------|--------|--------| +| `OP/` | [Ironboxplus/OpenList](https://github.com/Ironboxplus/OpenList) | `feat/dynamic-frontend` | origin (fork), op (upstream: OpenListTeam/OpenList) | +| `OpenList-Frontend/` | [Ironboxplus/OpenList-Frontend](https://github.com/Ironboxplus/OpenList-Frontend) | `main` | origin (upstream: OpenListTeam/OpenList-Frontend), ironbox (fork) | +| `movi-player/` | Local fork of [MrUjjwalG/movi-player](https://github.com/MrUjjwalG/movi-player) | `main` | — | +| `115-sdk-go/` | [Ironboxplus/115-sdk-go](https://github.com/Ironboxplus/115-sdk-go) | — | — | + +### Module Replacements (Backend `go.mod`) + +| Module | Replace Target | Notes | +|--------|---------------|-------| +| `github.com/OpenListTeam/115-sdk-go` | `github.com/Ironboxplus/115-sdk-go v0.2.8` | Concurrent refresh fix, ErrDataEmpty, FlexString CID | +| `github.com/ProtonMail/go-proton-api` | `github.com/henrybear327/go-proton-api v1.0.0` | Community fork | +| `github.com/cronokirby/saferith` | `github.com/Da3zKi7/saferith v0.33.0-fixed` | Bug fix fork | + +--- + +## [Backend] Core Development Principles 1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal 2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file 3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload -## Build and Development Commands +## [Backend] Build and Development Commands ```bash # Development @@ -38,7 +74,7 @@ docker build -f Dockerfile . # Build docker image **Module Replacements** (`go.mod`): Some dependencies use `replace` directives pointing to forks (e.g., `115-sdk-go` → `Ironboxplus/115-sdk-go`). When modifying SDK behavior, check if there's a local fork to edit. -## Architecture Overview +## [Backend] Architecture Overview ### Driver System (Storage Abstraction) @@ -286,7 +322,7 @@ Order of initialization: 6. `InitTaskManager()` - Start background tasks 7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers -## Common Patterns +## [Backend] Common Patterns ### Error Handling @@ -394,7 +430,7 @@ default: } ``` -## Important Conventions +## [Backend] Important Conventions **Naming**: - Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) @@ -416,7 +452,7 @@ default: - Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` - Include driver name in logs: `log.Infof("[driver_name] message")` -## Project Context +## [Backend] Project Context OpenList is a community-driven fork of AList, focused on: - Long-term governance and trust @@ -428,3 +464,69 @@ OpenList is a community-driven fork of AList, focused on: - Archive extraction **License**: AGPL-3.0 + +--- + +## Frontend (OpenList-Frontend) + +SolidJS + Vite + Hope UI. Builds to `dist/` which gets embedded into the backend binary via `go:embed`. + +### Build Commands + +```bash +pnpm install # Install dependencies +pnpm dev # Dev server (port 5173) +pnpm build # Production build +pnpm test # Run vitest tests +``` + +### Preview System + +Previews registered in `src/pages/home/previews/index.ts`. Each preview declares `prior: true/false` for priority ordering. Current video priority: movi-player (default) > Artplayer (fallback). + +### Movi Player Integration + +movi-player (FFmpeg WASM + WebCodecs) is the default video player, with full subtitle support: +- **SRT/VTT**: movi-player native parsing +- **ASS (external)**: JASSUB (libass WASM) overlay canvas rendering with font fallback +- **PGS/SUP (external)**: libpgs overlay canvas rendering +- **Embedded subtitles**: movi-player WASM demuxer handles all embedded formats + +Requires COOP/COEP headers (`Cross-Origin-Opener-Policy: same-origin`, `Cross-Origin-Embedder-Policy: credentialless`) for SharedArrayBuffer. Set in backend `server/router.go`. + +--- + +## Movi Player Fork (`movi-player/`) + +Clone of [MrUjjwalG/movi-player](https://github.com/MrUjjwalG/movi-player). FFmpeg WASM + WebCodecs browser-side video decoder. + +### Fork Changes + +- `src/core/MoviPlayer.ts`: Added `parseASS()` + `parseASSTime()` for external ASS subtitle support +- Format detection in `selectSubtitleLang()` extended for `.ass`/`.ssa` + +### Build + +```bash +npm run build:wasm # Requires Docker (C → WASM) +npm run build:ts # TypeScript only (needs dist/wasm/movi.js) +npm run build # Full build (wasm + ts) +``` + +### Known Limitations + +| Limitation | Reason | +|-----------|--------| +| External ASS: text only (no styles) | SubtitleCue architecture is plain text. Full ASS → use JASSUB overlay | +| External PGS: not supported in movi-player native | Binary format needs demuxer. Handled by libpgs overlay | +| Dolby Vision: purple tint | WASM decoder lacks DV enhancement layer | +| Requires COOP/COEP headers | SharedArrayBuffer for WASM threads | + +--- + +## 115-sdk-go Fork + +Fork of upstream SDK at `github.com/Ironboxplus/115-sdk-go`. Key changes: +- **Concurrent token refresh**: `authRequest` uses `sync.Mutex` + double-check to prevent multiple goroutines from racing on `RefreshToken` +- **ErrDataEmpty sentinel**: `GetFolderInfoByPath` returning `data:[]` for non-existent paths → returns `ErrDataEmpty` instead of unmarshal error +- **FlexString CID**: handles numeric/string JSON interop for category IDs diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md deleted file mode 100644 index 0b8be3b526..0000000000 --- a/COMPATIBILITY_REPORT.md +++ /dev/null @@ -1,204 +0,0 @@ -# Rebase兼容性分析报告 - -## 提交概览 -共引入 **21个commits**，主要涉及以下模块： - -### 核心功能改动 - -#### 1. **链接刷新机制** (`internal/stream/util.go`) -**Commits**: -- `4c33ffa4` feat(link): add link refresh capability for expired download links -- `f38fe180` fix(stream): 修复链接过期检测逻辑，避免将上下文取消视为链接过期 -- `7cf362c6` fix(stream): 更新过期链接检查逻辑，支持所有4xx客户端错误 -- `03fbaf1c` refactor(stream): 移除过时的链接刷新逻辑，添加自愈读取器以处理0字节读取 - -**核心代码**: -```go -// 新增常量 -MAX_LINK_REFRESH_COUNT = 50 // 链接最大刷新次数 -MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead重试次数（从3提升到5） - -// 新增函数 -IsLinkExpiredError(err error) bool // 判断是否为链接过期错误 - -// 新增结构 -RefreshableRangeReader struct { - link *model.Link - size int64 - innerReader model.RangeReaderIF - mu sync.Mutex - refreshCount int // 防止无限循环 -} - -selfHealingReadCloser struct { - // 检测0字节读取，自动刷新链接 -} -``` - -**功能说明**: -1. **链接过期检测**: 识别多种云盘的过期错误（expired, token expired, access denied, 4xx状态码等） -2. **自动刷新**: 检测到过期时自动调用Refresher获取新链接，最多刷新50次 -3. **自愈机制**: 处理某些云盘返回200但内容为空的情况（0字节读取检测） -4. **并发安全**: 使用sync.Mutex保护共享状态 -5. **Context隔离**: 刷新时使用WithoutCancel避免用户取消操作影响刷新 - -**潜在风险**: -- ✅ Context.WithoutCancel需要Go 1.21+ -- ✅ 并发场景下的锁竞争 -- ✅ refreshCount可能在某些场景下不递增导致无限循环 - ---- - -#### 2. **目录预创建优化** (`internal/fs/copy_move.go`) -**Commit**: `ce0da112` fix(copy_move): 将预创建子目录的深度从2级调整为1级 - -**核心代码**: -```go -func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { - // 第一轮：创建直接子目录 - for _, obj := range objs { - if obj.IsDir() { - subdirPath := stdpath.Join(dstBasePath, obj.GetName()) - op.MakeDir(t.Ctx(), t.DstStorage, subdirPath) - subdirs = append(subdirs, obj) - } - } - - // 停止递归条件 - if maxDepth <= 0 { - return nil - } - - // 第二轮：递归创建嵌套目录 - for _, subdir := range subdirs { - subObjs := op.List(...) - preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1) - } -} -``` - -**功能说明**: -1. **深度控制**: 默认maxDepth=1，只预创建2级目录（当前+子级） -2. **防止深度递归**: 避免在大型项目中递归过深导致栈溢出或性能问题 -3. **错误容忍**: MakeDir失败时继续处理其他目录 -4. **Context感知**: 每次循环检查ctx.Err()支持取消操作 - -**潜在风险**: -- ✅ op.MakeDir和op.List调用需要存储初始化 -- ✅ 大量目录时的性能问题 -- ✅ Context取消时的资源清理 - ---- - -#### 3. **网络优化** (`drivers/`, `internal/net/`) -**Commits**: -- `b9dafa65` feat(network): 增加对慢速网络的支持，调整超时和重试机制 -- `bce47884` fix(driver): 增加夸克分片大小调整逻辑，支持重试机制 -- `0b8471f6` feat(quark_open): 添加速率限制和重试逻辑 - -**功能说明**: -1. 提升RangeRead重试次数: 3 → 5 -2. 调整网络超时参数 -3. 添加分片上传重试逻辑 - ---- - -#### 4. **驱动修复** -**Commits**: -- `da2812c0` fix(google_drive): 更新Put方法以支持可重复读取流和不可重复读取流的MD5校验 -- `5a6bad90` feat(google_drive): 添加文件夹创建的锁机制和重试逻辑 -- `a54b2388` feat(google_drive): 添加处理重复文件名的功能 -- `9ef22ec9` fix(driver): fix file copy failure to 123pan due to incorrect etag -- `0ead87ef` fix(alias): update storage retrieval method in listRoot function -- `311f6246` fix: 修复500 panic和NaN问题 - ---- - -## 兼容性评估 - -### ✅ 编译兼容性 -- 构建成功，无语法错误 -- 依赖版本无冲突 - -### ✅ API兼容性 -- 新增函数不破坏现有接口 -- RefreshableRangeReader实现model.RangeReaderIF接口 -- 向后兼容旧代码 - -### ⚠️ 运行时兼容性 -**需要验证的场景**: -1. **并发安全**: RefreshableRangeReader的并发读取 -2. **资源泄漏**: Context取消时goroutine是否正确退出 -3. **边界条件**: - - refreshCount达到50次的行为 - - 0字节读取检测的准确性 - - maxDepth=0时的目录创建 -4. **错误处理**: - - nil Refresher时的处理 - - 链接刷新失败时的回退机制 -5. **性能**: - - 大文件下载时的刷新开销 - - 深层目录结构的预创建性能 - ---- - -## 测试需求 - -### 必须测试的场景 - -#### Stream包测试 -1. **IsLinkExpiredError准确性** - - 各种云盘的过期错误格式 - - Context取消不应判断为过期 - - HTTP 4xx/5xx的区分 - -2. **RefreshableRangeReader可靠性** - - 正常读取流程 - - 自动刷新触发和成功 - - 达到最大刷新次数 - - 并发读取安全性 - - Context取消的正确处理 - -3. **selfHealingReadCloser** - - 0字节读取检测 - - 刷新重试机制 - - 资源正确关闭 - -#### FS包测试 -1. **preCreateDirectoryTree** - - 深度控制正确性(0, 1, 2级) - - 大量目录的性能 - - Context取消的响应 - - 错误容忍性 - ---- - -## 风险等级: **中等** - -**原因**: -- ✅ 新功能设计合理，有明确的边界和错误处理 -- ⚠️ 并发场景需要充分测试 -- ⚠️ 链接刷新逻辑复杂，需要验证各种边界情况 -- ⚠️ 依赖op包的函数需要正确的初始化 - ---- - -## 推送建议: **通过测试后可推送** - -**前置条件**: -1. 完成全面的单元测试（见下方测试代码） -2. 验证并发安全性 -3. 确认Context取消不会导致资源泄漏 -4. 性能测试通过（大文件、深层目录） - -**建议测试命令**: -```bash -# 单元测试 -go test ./internal/stream ./internal/fs -v -count=1 -race - -# 压力测试 -go test ./internal/stream -run Stress -v -count=10 - -# 完整测试套件 -go test ./... -short -count=1 -``` diff --git a/JOURNAL.md b/JOURNAL.md deleted file mode 100644 index 40e75604e5..0000000000 --- a/JOURNAL.md +++ /dev/null @@ -1,486 +0,0 @@ -# Development Journal - -Chronological log of all changes in this fork, from earliest to latest. - ---- - -## 2026-04-25 — Initial Feature Batch (rebased onto up/main) - -### `17be63fb` feat(stream): link refresh, self-healing reader, seekable prefetch, and upload hash rework -- Added `RefreshableRangeReader`: wraps `RangeReader` with automatic link refresh on expiry (max 50 attempts) -- Added `selfHealingReadCloser`: detects 0-byte reads, connection resets, and `io.ErrUnexpectedEOF`; reconnects from current offset transparently -- Added `IsLinkExpiredError()`: checks error strings + HTTP 4xx status codes for link expiry -- Added 2-window async prefetch in `directSectionReader`: while uploading chunk N, prefetch chunk N+1 via goroutine -- Reworked `StreamHashFile` to use `RangeRead` for `SeekableStream` (never consumes `Reader`) -- Added `ReadFullWithRangeRead` with retry (max 5 attempts, 1-5s backoff) -- Files: `internal/stream/util.go`, `internal/stream/stream.go` - -### `1db9136f` feat(google_drive): duplicate filename handling, folder lock, retry, and MD5 checksum -- Added `mkdirLocks` (`sync.Map`) to prevent concurrent creation of duplicate folders -- Added existence check with retry before folder creation -- Added 500ms consistency delay after folder creation -- Added MD5 hash computation for upload integrity (via `StreamHashFile`) -- Added `chunkUpload` with retry and `RangeRead`-based streaming -- Files: `drivers/google_drive/driver.go`, `drivers/google_drive/util.go` - -### `f9bc1567` feat(115_open): permanent delete, proxy_range, offline task fixes, and error handling -- Added `RemoveWay` config option: "trash" (default) or "delete" (permanent) -- Implemented `removePermanently`: deletes from recycle bin after trash -- Added `findRecycleBinEntry` with paginated recycle bin search -- Added `matchRecycleBinEntry` with multi-strategy matching (ID → SHA1 → name+size) -- Added `findRecycleBinEntryWithRetry` (4 attempts, 300ms backoff) for eventual consistency -- Added `FlexString` CID handling (numeric/string JSON interop) -- Added `proxy_range` option exposure -- Files: `drivers/115_open/driver.go`, `drivers/115_open/meta.go`, `drivers/115_open/upload.go`, `drivers/115_open/driver_test.go` - -### `f1493fc9` feat(offline_download): multi-page task retrieval and task limit wait mechanism -- 115 `OfflineList` now paginates through all pages (was only page 1) -- Added task limit wait mechanism in offline download client -- Moved 115 offline task cleanup from `Run()` to `Update()` so it runs even if transfer fails -- Files: `internal/offline_download/115_open/client.go`, `internal/offline_download/tool/download.go` - -### `35130094` feat(drivers): baidu streaming upload, quark rate-limit/retry, 123pan etag fix -- **Baidu Netdisk**: extracted upload logic to `upload.go`, full streaming upload support -- **Quark Open**: added rate limiter, retry with chunk size adjustment on 413 error -- **123 Open**: fixed copy failure due to incorrect etag, added SHA1 rapid upload -- Normalized `StreamHashFile` progress weight to 100 across all drivers -- Files: `drivers/baidu_netdisk/upload.go`, `drivers/quark_open/driver.go`, `drivers/123_open/driver.go` - -### `7787ddbc` fix(core): copy_move depth, alias storage retrieval, sftp symlink, 500 panic -- Fixed `preCreateDirectoryTree` depth from 2 to 1 to avoid deep recursion -- Fixed alias storage retrieval method in `listRoot` -- Fixed SFTP symlink handling -- Fixed 500 panic and NaN issues -- Files: `internal/fs/copy_move.go`, `drivers/alias/util.go`, `drivers/sftp/types.go` - -### `4e735df0` feat(frontend): dynamic frontend fetching, CI upgrades, and build infrastructure -- Added `internal/frontend/fetcher.go`: auto-download frontend dist from GitHub releases -- Added `internal/frontend/watcher.go`: periodic check (30min) for new versions -- Added `FrontendRepoDefault` ldflags variable for build-time frontend repo injection -- CI: action version upgrades, frontend caching, version matrix builds -- `build.sh`: frontend repo configurable via `FRONTEND_REPO` env var -- Files: `internal/frontend/`, `.github/workflows/`, `build.sh`, `internal/conf/`, `server/static/` - -### `0bf12c5c` chore: add project docs and update dependencies (115-sdk-go fork) -- Added `CLAUDE.md` with comprehensive project guidance -- Added `COMPATIBILITY_REPORT.md` for 115-sdk-go fork analysis -- Updated 115-sdk-go dependency to `v0.2.5` (FlexString CID support) -- Files: `CLAUDE.md`, `COMPATIBILITY_REPORT.md`, `go.mod`, `go.sum` - ---- - -## 2026-05-08 — Rebase onto upstream + Code Review Fixes - -### Rebase onto `up/main` @ `c7c0cfae` -- `feat/dynamic-frontend` cleanly rebased (8 commits, 0 conflicts) -- Upstream additions included: path validation, SplitSeq perf, ObjectAlreadyExists check, qBittorrent login fix, custom share IDs, Getter interfaces (webdav/s3/115_open), about page logo fix - -### `0369bc0a` fix: bounded auth retry, mkdirLocks cleanup, EOF handling, tar size limit, dist swap lock, Go 1.26 vet -Code review identified 16 issues (2 CRITICAL, 5 HIGH). Fixed 6: -- **Google Drive Put 401 recursion** (CRITICAL): replaced infinite recursive `d.Put()` call with `putWithRetry()` + `maxPutAuthRetries=2` -- **mkdirLocks memory leak** (HIGH): added `defer mkdirLocks.Delete(lockKey)` after unlock -- **selfHealingReadCloser EOF** (HIGH): removed `io.EOF` from reconnect trigger, kept only `io.ErrUnexpectedEOF` -- **Frontend tar extraction** (HIGH): added `maxExtractFileSize=500MB` + `io.LimitReader` + `hdr.Size` check -- **Frontend dist swap TOCTOU** (HIGH): added `distSwapMu` mutex around rename window -- **RefreshableRangeReader concurrency** (CRITICAL): documented that local `reader` copy is safe after `innerReader` replacement -- **Go 1.26 vet**: fixed `fmt.Errorf` non-constant format strings across 8 drivers/packages -- Added tests: `drivers/google_drive/driver_test.go`, stream EOF tests, frontend oversized file test -- Files: 14 files changed, +222/-22 - ---- - -## 2026-05-09 — OSS Upload Fix + Hash Prefetch - -### `214881b3` fix(115_open): handle OSS upload timeout and PartAlreadyExist retry -- **Root cause**: `ResponseHeaderTimeout=15s` in shared `NewHttpClient()` was too short for uploading 20MB OSS parts. Timeout caused part to be uploaded but unconfirmed; retry hit `PartAlreadyExist` (409). -- **Fix 1**: Created `NewOSSUploadHttpClient()` with `ResponseHeaderTimeout=5min` dedicated to OSS uploads -- **Fix 2**: In `multpartUpload` retry, detect `PartAlreadyExist` → call `ListUploadedParts` to recover the part's ETag → treat as success -- Added `isPartAlreadyExistError()` helper -- Tests: `upload_test.go` (PartAlreadyExist detection), `oss_test.go` (upload client timeout) -- Files: `drivers/115_open/upload.go`, `internal/net/oss.go` - -### `94591821` perf(stream): add double-buffer prefetch to StreamHashFile for SeekableStream -- **Before**: hash calculation read 10MB chunks sequentially (network idle during hash computation) -- **After**: while hashing chunk N, goroutine prefetches chunk N+1 via `ReadFullWithRangeRead` -- Extracted `streamHashSeekableWithPrefetch()` with double-buffering pattern -- Hash values identical — no change to upload flow or rapid-upload logic -- `FileStream` path unchanged (sequential read) -- Test: `TestStreamHashFile_SeekablePrefetchProducesSameHash` -- Files: `internal/stream/util.go`, `internal/stream/util_test.go` - ---- - -## 2026-05-10 — 115 Rate Limiting + Concurrent Token Refresh Fix - -### 问题现象 - -复制任务（`/scnet/` → `/storage/my_115/`）全部失败，错误 `code: 0, message:` 和 `code: 40100000, message: 参数错误！`。目录确实存在，单任务正常，多 worker 并发时全部报错。 - -### 根因 1：Put 方法多个 SDK 调用未走限流器 - -`d1b72178` fix(115_open): rate-limit every SDK call in Put method - -- **发现**：`Put()` 入口只调一次 `WaitLimit`，后续 3-4 个 SDK 请求（`UploadInit` ×3 + `UploadGetToken`）直接发出，不经过限流器 -- **影响**：10 个 copy worker 并发时，不受限的 Put 请求和其他走限流器的 List/MakeDir 请求一起打到 115 API，瞬时 QPS 超过 115 的限制，API 返回 `state:false, code:0`（空错误）拒绝所有请求 -- **修复**：移除 Put 入口的单次 `WaitLimit`，在每个 SDK 调用（`UploadInit`、`UploadGetToken`）前单独加 `WaitLimit` -- **测试**：`TestPutRateLimitsEverySDKCall` — 设置 10 req/s 限流器，验证 3 个 UploadInit 调用之间有 >=70ms 间隔；`TestPutRateLimitsPreHashPath` — 验证秒传成功路径 -- Files: `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go` - -### 根因 2：SDK RefreshToken 无并发保护 - -`823f46ba` fix(deps): bump 115-sdk-go to v0.2.6 for concurrent refresh fix - -- **发现**：日志显示 `40140117 refresh frequently` 和 `40140120 refresh token error` 从 3 月 25 日就开始出现。115 的 refresh token 是一次性的——token 过期后多个 goroutine 同时调 `authRequest`，同时检测到 401，同时调 `RefreshToken`。第一个成功消耗了旧 RT，后续的全部失败（RT 已作废），token 被损坏或清空 -- **时间线**（`my_115` 实例）： - - 07:06:19 — 存储加载成功 - - 07:28:34 — copy workers 从 3 改成 10 - - 09:11:29 — 最后一条成功的 `[115] GetFiles` 日志 - - 09:11-09:45 — 35 分钟无 `[115]` 日志（全是文件上传，走 UploadInit 不产生 `[115]` 日志） - - 09:46:01 — 首次 `40100000 参数错误`（token 已失效/清空） - - 从 07:06 到 09:46 正好 ~2h40m，115 access token TTL 约 2h -- **修复**（SDK `v0.2.6`）：`authRequest` 中加 `refreshMu sync.Mutex` + double-check pattern。发请求前锁内读取 `usedToken`，遇 401 后锁内比对 `c.accessToken == usedToken`，若已被别的 goroutine 刷新过则跳过，未刷新才调 `RefreshToken` -- **测试**：`TestConcurrentAuthRequestRefreshesOnlyOnce` — 10 个并发 goroutine 同时打过期 token，断言 `RefreshToken` 只被调用 1 次，全部 goroutine 成功。`count=3` 稳定通过 -- Files: SDK `client.go`, `request.go`, `request_test.go`; OP `go.mod`, `go.sum` - ---- - -## 2026-05-11 — 115 GetFolderInfoByPath 空数据处理 - -### 问题现象 - -复制任务预建目标子目录时报错：`failed to get obj: json: cannot unmarshal array into Go value of type sdk.GetFolderInfoResp`。只有**不存在**的子目录报错，已存在的目录正常。 - -### 根因 - -115 Open API 的 `GetFolderInfoByPath` 在路径不存在时返回 `{state:true, data:[]}` 而不是正常的错误码。SDK 的 `authRequest` 直接把 `[]` 反序列化到 `GetFolderInfoResp`（struct）→ `json.UnmarshalTypeError`。该错误不是 `errs.ObjectNotFound`，导致 `MakeDir` 在 `op/fs.go:350` 当作未知错误抛出，而不是正常进入"目录不存在→创建"流程。 - -### 修复 - -**SDK v0.2.8**（`cf4f508` fix: return ErrDataEmpty when API responds with empty data）： -- 新增 `ErrDataEmpty` sentinel error -- `authRequest` 在 `extractData` 模式下：`data` 为 `null`/空 → 直接返回 `ErrDataEmpty`；`data` 为 `[]` 反序列化到 struct 失败 → 也返回 `ErrDataEmpty`（反序列化到 slice 类型则正常通过，不影响 `DelFile` 等返回空数组的 API） - -**Driver 层**： -- `Open115.Get()` 捕获 `sdk.ErrDataEmpty` → 转为 `errs.ObjectNotFound` -- `MakeDir` 的 `errs.IsObjectNotFound` 检测通过 → 正常创建目录 - -### 测试 - -- SDK: `TestAuthRequestReturnsErrDataEmptyForEmptyArray`、`TestAuthRequestReturnsErrDataEmptyForNull`、`TestAuthRequestSucceedsForValidObject` -- Driver: `TestGetReturnsObjectNotFoundForEmptyData`、`TestGetReturnsObjForExistingFolder` -- 既有 27 个测试全部通过，含 `TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing`（验证 `DelFile` 的 `data:[]` 不受影响） - -### 教训：不要 force-push tag - -Go module proxy 会缓存 tag 第一次发布时的内容。Force-push 更新 tag 后，`go.sum` 中记录的旧 hash 与新内容不匹配，触发 `checksum mismatch` 安全错误。正确做法是打新版号（v0.2.7 → v0.2.8）。 - -- Files: SDK `error.go`, `request.go`, `request_test.go`; OP `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go`, `go.mod`, `go.sum` - ---- - -## 已完成调查：FileStream.cache truncated stream（2026-04-07） - -### 现象 - -- `failed to read all data: (expect =50331648, actual =41592644) unexpected EOF` -- 调用链：`FsStream -> fs.PutDirectly -> op.Put -> FileStream.cache` -- 115_open 与 google_drive 均出现 - -### 根因 - -上游请求体提前结束（truncated stream），`FileStream.cache()` 的 `io.ReadFull` 严格检测并报错。`50331648 = 48MiB`（MaxBufferLimit 窗口），`41592644 ≈ 39.66MiB`（实际收到的字节数）。actual 值不固定，排除驱动逻辑在固定偏移崩溃的可能。 - -`3b2f9d55` 将"超限时落盘"改为"裁剪到 MaxBufferLimit"，使错误暴露更早，但非根因。 - -### 处置 - -P0：在上传入口增加"声明长度 vs 实际接收长度"日志；排查反向代理超时。 - ---- - -## 2026-05-13 — 115 Open 上传静默失败修复（双 bug） - -### 问题现象 - -前端上传文件显示成功（进度 100%，PUT `/api/fs/put` 返回 HTTP 200），但刷新页面后文件消失。日志无任何错误。 - -### 日志分析 - -从 330K 行日志中定位到两次上传（12:53:31 和 13:10:05），均耗时 ~2 分钟（实际数据传输），返回 200。上传前后 `arc` 目录文件数始终为 11，文件从未出现在 115 的文件列表中。 - -对比历史上传： - -- **2026-04-06**：3 次上传（10s/22s/25s）全部成功，`PUB` 目录从 2 增至 5 文件 -- **2026-04-14**：第 1 次上传失败（5→5 文件），第 2 次成功（5→6 文件） -- **2026-05-13**：两次上传均失败（11→11 文件） - -### Bug 1：OSS 回调未校验（静默失败） - -对比非 Open 版 115 驱动（`drivers/115/util.go`）与 Open 版（`drivers/115_open/upload.go`）： - -**非 Open 版**（正确）： - -```go -var bodyBytes []byte -bucket.CompleteMultipartUpload(imur, parts, - oss.Callback(...), - oss.CallbackResult(&bodyBytes), // ← 捕获回调响应 -) -var uploadResult UploadResult -json.Unmarshal(bodyBytes, &uploadResult) -return &uploadResult, uploadResult.Err(...) // ← 校验 state 字段 -``` - -**Open 版**（有 bug）： - -```go -// callbackRespBytes := make([]byte, 1024) ← 注释掉了！ -_, err = bucket.CompleteMultipartUpload(imur, parts, - oss.Callback(...), - // oss.CallbackResult(&callbackRespBytes), ← 注释掉了！ -) -if err != nil { return err } // ← 只检查 OSS 层错误 -return nil // ← 115 回调失败被忽略 -``` - -OSS 上传流程：客户端上传分片到 OSS → `CompleteMultipartUpload` → OSS 调用 115 的回调 URL → 115 返回 `{"state": true/false}`。OSS 只要回调 URL 返回 HTTP 200 就认为成功（`err == nil`），但 115 可能在 body 中返回 `{"state": false}` 表示文件注册失败。Open 版驱动没有捕获回调 body，所以 115 拒绝注册文件时完全静默。 - -`op.Put`（`internal/op/fs.go:714-731`）在 `err == nil` 时把文件临时加入目录缓存 → 前端显示成功 → 用户刷新后缓存过期，文件消失。 - -**修复**：新增 `UploadCallbackResult` + `checkUploadCallback()`，`singleUpload` 和 `multpartUpload` 均添加 `oss.CallbackResult(&bodyBytes)` 捕获并校验回调。 - -### Bug 2：Stream 模式大文件 SHA1 截断（根因） - -加了回调校验后看到真正的错误：`code=10002, message=校验文件失败，请重新上传。` - -关键线索：Form 模式（`PUT /api/fs/form`）和 Copy task 均成功，只有 Stream 模式（`PUT /api/fs/put`）失败。 - -**根因**：`CacheFullAndHash` → `CacheFullAndWriter` → `cache()` 将缓存上限截断为 `MaxBufferLimit`（~48MB）： - -```go -func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { - if maxCacheSize > int64(conf.MaxBufferLimit) { - maxCacheSize = int64(conf.MaxBufferLimit) // ← 截断！ - } - // ...只读取前 48MB 到 peekBuff -} -``` - -870MB 文件：SHA1 只算了前 48MB → `UploadInit` 带错误的 SHA1 → 完整 870MB 上传到 OSS → 115 校验：SHA1(870MB) ≠ SHA1(前 48MB) → 拒绝。 - -**为什么 Form 模式不受影响**：`c.FormFile()` 已将文件存入 `*os.File` 临时文件，`GetFile()` 返回非 nil，`CacheFullAndWriter` 走第一个分支直接读整个文件算 hash。 - -**为什么 Copy task 不受影响**：走 `SeekableStream` 路径（`isSeekable == true`），用 `StreamHashFile` → `RangeRead` 按需读取，不经过 `cache()`。 - -**修复**：`drivers/115_open/driver.go` Put 方法，对非 seekable 的 `FileStream`，当 `GetFile() == nil` 时先用 `utils.CreateTempFile` 将完整流写入临时文件，设置 `fs.Reader = tmpF`，再用 `StreamHashFile` 从临时文件计算正确的 SHA1。 - -### 上传数据流（修复后） - -``` -Stream 上传 (FileStream, !isSeekable): - HTTP Body → CreateTempFile(磁盘) → StreamHashFile(临时文件) → UploadInit → 分片上传(临时文件) → 清理 - -Copy task (SeekableStream, isSeekable): [不变] - Pass 1: RangeRead(源存储) → StreamHashFile → SHA1 - Pass 2: RangeRead(源存储) → directSectionReader → 分片上传 -``` - -### 附带修复：前端下载阻塞启动 - -`server/static/static.go`：移除 `initStatic` 中 30 秒超时的同步前端下载（国内 GitHub 不通导致启动卡住），改为直接使用内嵌 dist，由 `Watcher` 后台异步下载 rolling 版本并热替换。 - -### 测试 - -新增 4 个回调校验测试，既有 30 个测试全部通过： - -- `TestCheckUploadCallbackSuccess` — state=true 正常通过 -- `TestCheckUploadCallbackStateFalse` — state=false 返回含 code/message 的错误 -- `TestCheckUploadCallbackEmptyBody` — 空响应返回错误 -- `TestCheckUploadCallbackInvalidJSON` — 非法 JSON 返回错误 - -### `cache()` 截断是否影响其他驱动 - -`cache()` 的 `MaxBufferLimit` 截断是刻意设计（避免流式缓存占用过多内存），但 `CacheFullAndWriter` 名为"CacheFull"却内部调 `cache()` 导致大文件只缓存部分。任何依赖 `CacheFullAndHash` 对大文件（> MaxBufferLimit）计算 hash 的非 seekable 流驱动都可能受影响。本次修复仅在 115_open 驱动层绕过，未改动核心 stream 包。 - -- Files: `drivers/115_open/driver.go`, `drivers/115_open/driver_test.go`, `drivers/115_open/upload.go`, `server/static/static.go` - ---- - -## 2026-05-17 — Rebase onto upstream HybridCache + Pass 2 prefetch + BFS precreate 删除 - -### 背景 - -Upstream（OpenListTeam/OpenList）通过 PR #2460（`b6db83ed`）引入 `HybridCache`：三级缓存（普通堆内存 → `LinearMemory` → 文件落盘），用来统一 stream 缓存路径。这恰好替代了 2026-05-13 在 115_open 驱动层手写的 SHA1 截断绕过（`utils.CreateTempFile` workaround）。 - -### 行动 1：Rebase 16 个提交到 upstream HybridCache - -用 `git rebase --onto upstream/main` 把本地分支挪到 HybridCache 之上。冲突解决原则：**保留双方功能**。 - -关键冲突文件： - -- **`internal/stream/stream.go`** — 取 upstream HybridCache 版（`cache()` 内部三级分配），删掉本地原来的 `MaxBufferLimit` workaround -- **`internal/stream/util.go`** — 保留本地全部新增：`RefreshableRangeReader`、`selfHealingReadCloser`、`IsLinkExpiredError`、`ReadFullWithRangeRead`、`streamHashSeekableWithPrefetch`；丢弃本地的 `directSectionReader` + prefetch（在新结构上重写）；upstream 的 `byteSectionReader` / `hybridSectionReader` 保留 -- **`drivers/115_open/driver.go`** Put 方法 — 删掉 `CreateTempFile` workaround，恢复标准 `CacheFullAndHash` 路径（HybridCache 已经保证完整缓存） -- **`drivers/baidu_netdisk/upload.go`** — 接口重命名：`StreamSectionReaderIF` → `StreamSectionReader`（upstream 名称） -- **`server/static/static.go`** — 保留本地动态前端 `reloadableFS`（rebase 期间一度被覆盖，已修正） - -Rebase 后 22 commits ahead of origin，force-pushed 至 `7995dbdb`。 - -### HybridCache vs 旧 workaround - -| 维度 | 旧 workaround | HybridCache | -|---|---|---| -| 修复范围 | 仅 115_open | stream 层，全 driver 受益 | -| 大文件存储 | 一律落盘 | 优先内存，紧张才落盘 | -| 块策略 | 单 buffer | 16MB 分块（`MaxBlockLimit`） | -| GC 友好度 | 一个大 `[]byte` | `LinearMemory` 避开 Go heap | -| Section reader 适配 | 整 buffer 切片 | 每块独立，对 `hybridSectionReader` 天然友好 | - -### 行动 2：Pass 2 prefetch on `hybridSectionReader`（commit `7995dbdb`） - -旧 `directSectionReader` 已被 upstream 的 `hybridSectionReader` 替代，但失去了"上传当前分片时异步预读下一分片"的优化（影响 115_open 这种顺序上传的 driver）。 - -TDD 红→绿→重构： - -- 在 `hybridSectionReader` 上加 `prefetchTask` 状态 + `schedulePrefetch` / `takePrefetched` / `waitPrefetch` -- 每次 `GetSectionReader` 返回后启动 goroutine 预读下一块到一个 pending block -- 下次 `GetSectionReader` 命中即拿、不命中走原路径 -- `DiscardSection` 排空 prefetch；prefetch 错误延迟到下次 `GetSectionReader` 暴露 -- `file.Add(closerFunc(waitPrefetch))` 注册清理，避免 `hc` 被释放时 prefetch 还在写 - -8 个测试覆盖：overlap 时序、顺序正确性、最后分片小于 partSize、Discard 配合、prefetch 错误传播、不超 EOF、长度 clamp、非 hybrid 路径回退。 - -- Files: `internal/stream/util.go`, `internal/stream/section_reader_prefetch_test.go` - -### 行动 3：删除 BFS precreate（commit `6b3ce577`） - -旧 `preCreateDirTreeFn` 提前 BFS 创建 dst 一层子目录。原始动机：避开"merge 任务 List 不存在的 dst 会致命"的 bug。 - -排查后发现该 bug 已由三层独立修复覆盖： - -1. **`copy_move.go:213` 容错判断**（PR #1898，你最初提交、KirCute 优化、Tron 合入）：`if err != nil && !errors.Is(err, errs.ObjectNotFound)` 容忍 dst 不存在 -2. **`op.Put` 内置 MakeDir**（`internal/op/fs.go:682`）：上传前自动建父目录 -3. **`op.MakeDir` 递归 + 缓存同步重试**（`internal/op/fs.go:354-364`）：递归建父链 + 100ms 重试解决云盘缓存延迟 - -BFS precreate 在功能上已变成纯性能优化（提前批建 + 缓存预热），不再是 bug 修复的关键依赖。 - -**TDD 流程**： - -1. 抽出 `existingDstFilesFn(ctx, listDst, dstPath) → map[string]bool`，把 merge 模式构建 `existedObjs` 的逻辑独立为可注入纯函数 -2. 写 12 个单测：empty/files-only/dirs-only/mixed/raw-ObjectNotFound/wrapped-ObjectNotFound（#1898 回归保护）/其他 List 错误/ctx 取消前/ctx 取消中/重名/万级 dst 性能/单次 List 合理性 -3. 全绿后删除 `preCreateDirTreeFn`、`preCreateDirectoryTree` 方法、调用点、原有 11 个 `TestPreCreateDirTreeFn_*` 测试 -4. 保留顶层 dst MakeDir（`copy_move.go:198`）做早期失败检测 - -净 -95 行代码。`existingDstFilesFn` 的 ObjectNotFound 容错测试（`TestExistingDstFilesFn_DstDoesNotExist_RawError` / `_WrappedError`）是 #1898 的回归护栏，确保未来重构不把那个老 bug 漏回去。 - -- Files: `internal/fs/copy_move.go`, `internal/fs/copy_move_test.go` - -### 行动后状态 - -- `go build ./...` 干净 -- `internal/fs`、`internal/stream`、`internal/op`、`drivers/115_open`、`drivers/baidu_netdisk` 全部测试通过 -- 不相关失败：`internal/net/oss_test.go` 的 HTTPS proxy 测试（rebase 前就失败）、`pkg/aria2/rpc` 测试（需要本地 aria2 服务） -- Branch `feat/dynamic-frontend` HEAD 推至 `6b3ce577` - ---- - -## 2026-05-19 — Rebase onto `hybrid_cache` 包抽取 + prefetch 双 commit squash - -### 背景 - -上游 PR #2477（commit `9eae6258`）做了一次破坏性重构： - -- 把 `internal/mem`（`HybridCache` / `LinearMemory`）抽到独立的 `internal/hybrid_cache` 包（别名 `hcache`） -- 引入 `BackingStore` 抽象：`BufferStore`（内存）+ `FileStore`（磁盘），由 `HybridCache` 内部自动切换，外部代码不再分支 -- 配置字段重命名：`conf.CacheThreshold` → `conf.AutoMemoryLimit`（json `cache_threshold` → `auto_memory_limit`、env 同步） -- 删除 `pkg/buffer/bytes.go` / `pkg/buffer/file.go`，`pkg/buffer/buffer.go` + `type.go` 重塑 -- `internal/stream/util.go` 从 ~960 行精简至 298 行——上游只保留 `GetRangeReaderFromLink` / `CacheFullAndHash` / `NewStreamSectionReader` / `hybridSectionReader` 等核心 helpers，所有本地扩展需要重新落上去 - -同批次另有两个修复（`daad21ef` 139 driver `Connection` 头、`31b41f99` qBittorrent 5.2 204 登录修复），均不与本地冲突。 - -### 行动 1：Rebase 20 个本地 commit 到 `op/main` @ `31b41f99` - -`git tag pre-rebase-2026-05-19 HEAD` 后 `git rebase op/main`，命中两处冲突： - -**冲突点 1 — `internal/stream/stream.go`（`fccab338`）** - -上游把 `cache(maxCacheSize int64)` 重命名为 `ensureCache(size int64)`，函数体已转为新 `hcache.HybridCache`。处理：直接取上游版，本地 fccab338 patch 的语义已被上游覆盖。 - -**冲突点 2 — `internal/stream/util.go`（`fccab338`）** - -本地有独立的 `byteSectionReader` 分支（小文件走单独缓冲池）。上游新设计下 `BackingStore` 自动按文件大小选择 `BufferStore` / `FileStore`，`NewStreamSectionReader` 简化为： - -```go -if file.GetFile() != nil { return &cachedSectionReader{...} } -// 否则全部走 -return &hybridSectionReader{...} -``` - -处理：删除 `byteSectionReader` / `bytesRefReadSeeker` 类型，以及 `TestHybridSectionReader_NonPrefetchPath` 测试（断言"小文件不走 hybrid"已失效）。 - -**冲突点 3 — `internal/stream/util.go`（`7995dbdb` Pass 2 prefetch）** - -`hybridSectionReader` 结构体内的 `hc` 字段：本地 `*mem.HybridCache` vs 上游 `*hcache.HybridCache`。处理：保留本地新增的 `fileSize int64` 字段（Pass 2 prefetch 用于 EOF clamp），类型切到 `*hcache.HybridCache`。 - -**冲突点 4 — `internal/stream/section_reader_prefetch_test.go`** - -`conf.CacheThreshold` 三处引用全部改成 `conf.AutoMemoryLimit`。 - -其余 16 个 commit 干净回放（包括 `163e5c81` 的 util.go selfHealingReadCloser EOF 调整、`07410ad3` 的 Pass 1 hash prefetch、`4e91d93a` 的 `NewOSSUploadHttpClient` 等），上游 patch 已覆盖大部分 `mem.` → `hcache.` 翻译工作。 - -### 行动 2：Squash 两个 prefetch commit - -`588b7024 perf(stream): add double-buffer prefetch to StreamHashFile` 和 `a32544fa perf(stream): pass 2 prefetch on hybridSectionReader` 同属"上传 pipeline 重叠优化"主题但相隔 5 个 commit。用 `git rebase -i op/main` 加自定义 `GIT_SEQUENCE_EDITOR` / `GIT_EDITOR`（Python 脚本临时丢在 `.git/`）做 reorder + squash，得到 `01364250 perf(stream): two-pass prefetch on upload pipeline`，叙事完整。 - -之间的 4 个非 stream commit（`8a2fb995` / `9389b219` / `610c3cf9` / `65c713a0`）reorder 时无冲突——它们碰的是 driver 和 `server/static`，不沾 util.go。 - -未 squash 的 `902c8b82 feat(stream)` 是综合提交（selfHealing + link refresh + hash 工具 + 当时被丢弃的旧 prefetch），独立保留以维持叙事。 - -### 测试结果 - -| 包 | 结果 | -|---|---| -| `internal/stream` | ok（7 个 `TestHybridSectionReader_*` prefetch、2 个 `TestRefreshableRangeReader_*`、2 个 `TestSelfHealingReadCloser_*`、`TestStreamHashFile_SeekablePrefetchProducesSameHash` 全部通过） | -| `internal/fs` | ok | -| `internal/op` | ok | -| `drivers/115_open` | ok（含 `TestNewOSSUploadHttpClientHasLongerTimeout`） | -| `internal/hybrid_cache` | ok | -| `internal/frontend` | ok | -| `internal/net` | 唯一失败 `TestNewOSSClientUsesEnvironmentHTTPSProxy`（白名单，rebase 前就失败） | - -### 行动后状态 - -- `feat/dynamic-frontend` HEAD `39c74672`，19 commits ahead of `op/main`（原 20，squash -1） -- `git push --force-with-lease origin feat/dynamic-frontend` 成功（`7e140903...39c74672`） -- 本地 tag `pre-rebase-2026-05-19` 保留作为回滚点 -- `env.md` 更新 "Current upstream base" 到 `31b41f99` - -### 设计要点：byteSectionReader 消失为何不损失功能 - -旧 `byteSectionReader` 的优化逻辑：小文件用 `pool.Pool[[]byte]` 复用 buffer，避免每个 section 都分配新切片。 - -上游 `BackingStore` 的等价机制：`BufferStore` 内部就是块池化（按 `blockSize` 分块，块从 `LinearMemory` 复用），且自动按 `AutoMemoryLimit`/文件大小决策。也就是说"小文件走纯内存"这个语义保留了，只是不再暴露为独立类型。 - -代价：失去了"按文件大小选择 reader 实现"的显式分支，调试时需要看 `BackingStore` 内部状态。换取：调用方代码统一、Pass 2 prefetch 可以盲目挂在 `hybridSectionReader` 上不用考虑 byte 分支。 - ---- - -## Architecture Notes - -### Upload Data Flow (Current) -``` -Pass 1: Hash Calculation (with prefetch) - StreamHashFile → ReadFullWithRangeRead - chunk N: hash computation (CPU) - chunk N+1: async prefetch (network I/O) ← overlapped - -Pass 2: Multipart Upload (with prefetch) - hybridSectionReader.GetSectionReader - chunk N: upload to cloud (network I/O) - chunk N+1: async prefetch (network I/O) ← overlapped -``` - -### Proxy Architecture -- `conf.Conf.ProxyAddress` → global HTTP proxy for all server-side traffic -- Per-storage `WebProxy` / `DownProxyURL` → browser download only, NOT copy tasks -- OSS uploads use dedicated `NewOSSUploadHttpClient()` with longer timeouts diff --git a/OVERVIEW.md b/OVERVIEW.md deleted file mode 100644 index 01c71fb63d..0000000000 --- a/OVERVIEW.md +++ /dev/null @@ -1,79 +0,0 @@ -# OpenList Fork — Project Overview - -This is a fork of [OpenListTeam/OpenList](https://github.com/OpenListTeam/OpenList) (upstream), maintained at [Ironboxplus/OpenList](https://github.com/Ironboxplus/OpenList). - -## Branch Structure - -| Branch | Purpose | -|--------|---------| -| `feat/dynamic-frontend` | **Active development branch** (default). Rebased on `up/main`. | -| `copy` | Legacy branch with unsquashed commits. Superseded by `feat/dynamic-frontend`. | -| `main` | Synced from upstream via rebase workflow. | - -## Documentation Index - -| File | Description | -|------|-------------| -| [CLAUDE.md](CLAUDE.md) | AI coding guidance: architecture, driver system, stream/upload internals, conventions | -| [COMPATIBILITY_REPORT.md](COMPATIBILITY_REPORT.md) | 115-sdk-go fork compatibility analysis | -| [OVERVIEW.md](OVERVIEW.md) | This file — project index and high-level map | -| [JOURNAL.md](JOURNAL.md) | Chronological development log with all changes | -| [CONTRIBUTING.md](CONTRIBUTING.md) | Upstream contribution guidelines | -| [README.md](README.md) | Upstream project README | -| [SECURITY.md](SECURITY.md) | Security policy | - -## Key Subsystems Modified (vs Upstream) - -### 1. Full-Streaming Upload with Async Prefetch -- **Files**: `internal/stream/util.go`, `internal/stream/stream.go` -- Two-pass flow: hash calculation (with double-buffer prefetch) → multipart upload (with 2-window async prefetch) -- `SeekableStream` uses `RangeRead` exclusively, never consumes the `Reader` -- `selfHealingReadCloser`: transparent link refresh + reconnect-from-offset on stream interruption -- `RefreshableRangeReader`: auto-refresh expired download links (up to 50 retries) - -### 2. Dynamic Frontend Fetcher -- **Files**: `internal/frontend/fetcher.go`, `internal/frontend/watcher.go` -- Auto-downloads frontend dist from GitHub releases on startup (when `WebVersion` is rolling/beta/dev) -- `Watcher` polls every 30 min for new versions, hot-swaps dist directory -- Configurable `FrontendRepo` (default: `OpenListTeam/OpenList-Frontend`, overridable per deployment) -- `FrontendRepoDefault` injectable via ldflags at build time - -### 3. Driver Enhancements - -| Driver | Changes | -|--------|---------| -| 115 Open | Permanent delete with recycle-bin retry, FlexString CID, OSS upload timeout fix, PartAlreadyExist recovery, proxy_range, offline task multi-page | -| Google Drive | Duplicate filename handling, per-folder MakeDir lock, bounded 401 retry, MD5 checksum, mkdirLocks cleanup | -| Baidu Netdisk | Full streaming upload (extracted from monolithic driver) | -| Quark Open | Rate limiting, retry with chunk size adjustment | -| 123 Open | SHA1 rapid upload, etag fix, StreamHashFile progress normalization | -| Aliyundrive Open | StreamHashFile progress normalization | - -### 4. CI/CD & Build -- **Files**: `.github/workflows/`, `build.sh` -- Frontend version matrix (rolling + latest) for Docker builds -- `FrontendRepoDefault` x-flag in CI -- Action version upgrades (checkout v6, setup-go v6, cache v5) -- Frontend caching in CI to avoid redundant downloads -- Static linking verification for musl builds - -### 5. Offline Download -- **Files**: `internal/offline_download/115_open/`, `internal/offline_download/tool/` -- Multi-page task retrieval for 115 -- Task limit wait mechanism -- Cleanup moved to `Update()` to ensure it runs even if transfer fails - -## Upstream Sync - -Remote `up` points to `https://github.com/OpenListTeam/OpenList.git`. - -```bash -git fetch up -git rebase up/main # on feat/dynamic-frontend -``` - -Current base: `up/main` @ `c7c0cfae` (2026-05-06) - -## Global Proxy - -All server-side HTTP traffic (driver API calls, copy/move transfers, frontend fetching) uses `conf.Conf.ProxyAddress` (global setting). Per-storage `WebProxy`/`DownProxyURL` only affects browser-facing download behavior, NOT server-side copy tasks. From 7de3dedcab25f1d9221daaceeda40bb7ffaf7050 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 10 Jun 2026 23:47:53 +0800 Subject: [PATCH 078/107] chore(deps): bump Ironboxplus/115-sdk-go to v0.2.9 Pulls in two auth fixes: - token refresh detached from caller context (WithoutCancel + 30s timeout) so a canceled streaming request can no longer kill an in-flight refresh and lose the rotated one-time refresh_token - 40140117 (refresh frequently) and 40140120 (refresh token error) no longer misclassified as stale-access-token errors that trigger another refresh attempt --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10e319ae7a..04bda07d6b 100644 --- a/go.mod +++ b/go.mod @@ -312,6 +312,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.8 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.9 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index adb42d29fb..895fbcece9 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/Ironboxplus/115-sdk-go v0.2.8 h1:JyRGXDDXktItPJansGzyLpziF1UhY30xzC5LRYTgRp4= -github.com/Ironboxplus/115-sdk-go v0.2.8/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.9 h1:Y+LdB5/mytE27Tj3Hm2FOH2nu+CKSALQioA7QOtM5sw= +github.com/Ironboxplus/115-sdk-go v0.2.9/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From e66f5ad5069de0cbffa66c3254c5d3ccf429f2ce Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 14 Jun 2026 16:35:52 +0800 Subject: [PATCH 079/107] docs: update 115-sdk-go fork notes to v0.2.9 --- CLAUDE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 680d02ba88..eb536419eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ Each section below is labeled by component. The backend section is the primary d | Module | Replace Target | Notes | |--------|---------------|-------| -| `github.com/OpenListTeam/115-sdk-go` | `github.com/Ironboxplus/115-sdk-go v0.2.8` | Concurrent refresh fix, ErrDataEmpty, FlexString CID | +| `github.com/OpenListTeam/115-sdk-go` | `github.com/Ironboxplus/115-sdk-go v0.2.9` | Refresh context isolation, token error code filters, ErrDataEmpty, FlexString CID | | `github.com/ProtonMail/go-proton-api` | `github.com/henrybear327/go-proton-api v1.0.0` | Community fork | | `github.com/cronokirby/saferith` | `github.com/Da3zKi7/saferith v0.33.0-fixed` | Bug fix fork | @@ -526,7 +526,9 @@ npm run build # Full build (wasm + ts) ## 115-sdk-go Fork -Fork of upstream SDK at `github.com/Ironboxplus/115-sdk-go`. Key changes: +Fork of upstream SDK at `github.com/Ironboxplus/115-sdk-go`. Key changes (v0.2.9): - **Concurrent token refresh**: `authRequest` uses `sync.Mutex` + double-check to prevent multiple goroutines from racing on `RefreshToken` +- **Refresh context isolation** (v0.2.9): `RefreshToken` runs under `context.WithTimeout(context.WithoutCancel(ctx), 30s)` — once a refresh starts, caller cancellation cannot abort it. Fixes token loss when video player cancels mid-refresh +- **Token error code filters** (v0.2.9): `shouldRefreshToken(code)` excludes 40140117 (CodeRefreshFrequently) and 40140120 (CodeRefreshTokenError) from triggering refresh; prevents feeding rate limiter on unrecoverable auth failures - **ErrDataEmpty sentinel**: `GetFolderInfoByPath` returning `data:[]` for non-existent paths → returns `ErrDataEmpty` instead of unmarshal error - **FlexString CID**: handles numeric/string JSON interop for category IDs From 98707eeb22894a6d02446bccd159f453107045a2 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 14 Jun 2026 17:00:39 +0800 Subject: [PATCH 080/107] refactor(115_open): extract tested isObjectNotFound predicate for Get The rebase merged upstream #2596 (sdk.ErrObjectNotFound mapping) with our ErrDataEmpty handling into a union inside driver.Get. Extract it into a unit-testable helper so both not-found sentinels are covered directly: ErrObjectNotFound is shadowed by ErrDataEmpty for GetFolderInfoByPath (request.go collapses an empty/[] payload before folder-info unmarshal), so it is unreachable via an HTTP mock and an inline test could not exercise it. - isObjectNotFound(err): ErrObjectNotFound || ErrDataEmpty, both errors.Is-safe - TestIsObjectNotFound: both sentinels, wrapped forms, unrelated error, nil --- drivers/115_open/driver.go | 3 +-- drivers/115_open/get.go | 16 ++++++++++++++++ drivers/115_open/get_test.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 982eef6d9e..bcfde457f8 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -3,7 +3,6 @@ package _115_open import ( "context" "encoding/json" - "errors" "fmt" "net/http" stdpath "path" @@ -217,7 +216,7 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { path = stdpath.Join(d.parentPath, path) resp, err := d.client.GetFolderInfoByPath(ctx, path) if err != nil { - if errors.Is(err, sdk.ErrObjectNotFound) || errors.Is(err, sdk.ErrDataEmpty) { + if isObjectNotFound(err) { return nil, errs.ObjectNotFound } return nil, err diff --git a/drivers/115_open/get.go b/drivers/115_open/get.go index d36f0c97fe..3f18d18a1f 100644 --- a/drivers/115_open/get.go +++ b/drivers/115_open/get.go @@ -1,11 +1,27 @@ package _115_open import ( + "errors" + sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) +// isObjectNotFound reports whether err means the 115 API could not find the +// requested path, so driver.Get can translate it into errs.ObjectNotFound and +// let op.Get fall through instead of surfacing a raw SDK error. The SDK +// surfaces "not found" two ways and both must be honored: +// - ErrDataEmpty: GetFolderInfoByPath returned an empty array. This is the +// realistic case — request.go collapses an empty/`[]` payload to +// ErrDataEmpty before the folder-info unmarshal can run. +// - ErrObjectNotFound: upstream #2596's mapping of SDK not-found errors. +// Currently shadowed for this call path but kept as a forward-compatible +// second arm so a future SDK that surfaces it directly still works. +func isObjectNotFound(err error) bool { + return errors.Is(err, sdk.ErrObjectNotFound) || errors.Is(err, sdk.ErrDataEmpty) +} + // folderInfoToObj extracts the fields driver.Get needs from a // GetFolderInfoByPath response. We only look at FileID + FileName + // FileCategory — Sha1 / PickCode come along because the struct already diff --git a/drivers/115_open/get_test.go b/drivers/115_open/get_test.go index 638e65b69f..d53423f2aa 100644 --- a/drivers/115_open/get_test.go +++ b/drivers/115_open/get_test.go @@ -2,6 +2,7 @@ package _115_open import ( "errors" + "fmt" "testing" sdk "github.com/OpenListTeam/115-sdk-go" @@ -45,6 +46,34 @@ func TestFromFolderInfo_FileReturnsNotImplement(t *testing.T) { } } +// driver.Get maps "path not found" to errs.ObjectNotFound so op.Get can fall +// through cleanly instead of surfacing a raw SDK error. The SDK surfaces this +// two ways and both must be honored: ErrDataEmpty (GetFolderInfoByPath returns +// an empty array — the realistic case) and ErrObjectNotFound (upstream #2596's +// SDK not-found mapping, kept as a forward-compatible second arm). Any other +// error must NOT be swallowed. +func TestIsObjectNotFound(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"ErrDataEmpty (empty folder-info array)", sdk.ErrDataEmpty, true}, + {"ErrObjectNotFound (upstream #2596 mapping)", sdk.ErrObjectNotFound, true}, + {"wrapped ErrDataEmpty", fmt.Errorf("GetFolderInfoByPath: %w", sdk.ErrDataEmpty), true}, + {"wrapped ErrObjectNotFound", fmt.Errorf("GetFolderInfoByPath: %w", sdk.ErrObjectNotFound), true}, + {"unrelated error must propagate", errors.New("429 rate limited"), false}, + {"nil is not not-found", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isObjectNotFound(tt.err); got != tt.want { + t.Fatalf("isObjectNotFound(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + func TestFromFolderInfo_FolderReturnsObj(t *testing.T) { resp := &sdk.GetFolderInfoResp{ FileID: "1234", FileName: "Movies", FileCategory: "0", From 2b8e39ac8553d4420811a0f38bf35ec1e2b1eb04 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 21 Jun 2026 23:24:57 +0800 Subject: [PATCH 081/107] feat: yaegi plugin runtime, dynamic disk-usage header, favorites, loading progress Backend plugin system: swap wazero/WASM runtime for an in-process yaegi (Go-source) interpreter so admins can author plugins in plain Go. - internal/plugin: interp.go (Runtime: Load/Unload/hot-reload, curated host symbols), registry.go (hook event bus), hooks.go (Hook* constants + SettingGetter/Setter injection), manager.go (file watcher), manage.go (user-plugin CRUD + enable/disable) - bootstrap/plugin.go bridges op StorageHook/ObjsUpdateHook into the registry and wires setting capabilities (no op import cycle) - fire HookFsListAfter / HookFsLinkAfter / HookUserLoginAfter from handles - admin API: /admin/plugin list/get/save/delete/enable Header: FsListResp.mount_details (cache + singleflight backed, 429-safe, guest/HideStorageDetails gated) powers a dynamic disk-usage widget. Also: favorites (db/model/handles), storage loading progress (op/loading), 115_open video play, doc restructure (INDEX/OVERVIEW + docs/*). Tests: internal/plugin, server/handles, internal/op, internal/model green. --- CLAUDE.md | 511 ++----------------------- docs/architecture.md | 171 +++++++++ docs/frontend.md | 110 ++++++ docs/new-apis-2026-06-21.md | 73 ++++ docs/plugins.md | 99 +++++ docs/streaming-and-caching.md | 146 +++++++ docs/uploads.md | 56 +++ drivers/115_open/video.go | 56 +++ drivers/115_open/video_test.go | 48 +++ go.mod | 7 +- go.sum | 6 +- internal/bootstrap/db.go | 5 +- internal/bootstrap/plugin.go | 81 ++++ internal/bootstrap/run.go | 5 + internal/bootstrap/storage.go | 56 ++- internal/db/db.go | 2 +- internal/db/favorite.go | 44 +++ internal/driver/video.go | 20 + internal/model/favorite.go | 11 + internal/model/storage.go | 12 + internal/model/storage_details_test.go | 27 ++ internal/model/user.go | 12 + internal/model/user_test.go | 34 ++ internal/op/loading.go | 125 ++++++ internal/op/loading_test.go | 113 ++++++ internal/op/storage.go | 3 + internal/plugin/hooks.go | 60 +++ internal/plugin/interp.go | 206 ++++++++++ internal/plugin/interp_test.go | 140 +++++++ internal/plugin/manage.go | 159 ++++++++ internal/plugin/manage_test.go | 115 ++++++ internal/plugin/manager.go | 225 +++++++++++ internal/plugin/manager_test.go | 157 ++++++++ internal/plugin/registry.go | 120 ++++++ internal/plugin/registry_test.go | 101 +++++ server/handles/auth.go | 2 + server/handles/favorite.go | 117 ++++++ server/handles/favorite_test.go | 38 ++ server/handles/fsmanage.go | 2 + server/handles/fsread.go | 84 +++- server/handles/plugin.go | 148 +++++++ server/handles/storage.go | 6 + server/handles/user.go | 31 +- server/handles/user_test.go | 80 ++++ server/middlewares/auth.go | 12 + server/router.go | 35 +- 46 files changed, 3166 insertions(+), 505 deletions(-) create mode 100644 docs/architecture.md create mode 100644 docs/frontend.md create mode 100644 docs/new-apis-2026-06-21.md create mode 100644 docs/plugins.md create mode 100644 docs/streaming-and-caching.md create mode 100644 docs/uploads.md create mode 100644 drivers/115_open/video.go create mode 100644 drivers/115_open/video_test.go create mode 100644 internal/bootstrap/plugin.go create mode 100644 internal/db/favorite.go create mode 100644 internal/driver/video.go create mode 100644 internal/model/favorite.go create mode 100644 internal/model/storage_details_test.go create mode 100644 internal/model/user_test.go create mode 100644 internal/op/loading.go create mode 100644 internal/op/loading_test.go create mode 100644 internal/plugin/hooks.go create mode 100644 internal/plugin/interp.go create mode 100644 internal/plugin/interp_test.go create mode 100644 internal/plugin/manage.go create mode 100644 internal/plugin/manage_test.go create mode 100644 internal/plugin/manager.go create mode 100644 internal/plugin/manager_test.go create mode 100644 internal/plugin/registry.go create mode 100644 internal/plugin/registry_test.go create mode 100644 server/handles/favorite.go create mode 100644 server/handles/favorite_test.go create mode 100644 server/handles/plugin.go create mode 100644 server/handles/user_test.go diff --git a/CLAUDE.md b/CLAUDE.md index eb536419eb..3a9fad67ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,12 +1,6 @@ -# OpenList Workspace — CLAUDE.md +# OpenList Backend (OP) — CLAUDE.md -This file covers the entire OpenList workspace, which contains three main repositories: - -- **OP/** — OpenList backend (Go, main project) -- **OpenList-Frontend/** — SolidJS frontend -- **movi-player/** — FFmpeg WASM video player (local fork) - -Each section below is labeled by component. The backend section is the primary development guide. +Primary development guide for the Go backend. See [../INDEX.MD](../INDEX.MD) for the full subsystem→code map. --- @@ -24,11 +18,11 @@ Each section below is labeled by component. The backend section is the primary d | Directory | Repo | Branch | Remote | |-----------|------|--------|--------| | `OP/` | [Ironboxplus/OpenList](https://github.com/Ironboxplus/OpenList) | `feat/dynamic-frontend` | origin (fork), op (upstream: OpenListTeam/OpenList) | -| `OpenList-Frontend/` | [Ironboxplus/OpenList-Frontend](https://github.com/Ironboxplus/OpenList-Frontend) | `main` | origin (upstream: OpenListTeam/OpenList-Frontend), ironbox (fork) | +| `OpenList-Frontend/` | [Ironboxplus/OpenList-Frontend](https://github.com/Ironboxplus/OpenList-Frontend) | `main` | origin (upstream), ironbox (fork) | | `movi-player/` | Local fork of [MrUjjwalG/movi-player](https://github.com/MrUjjwalG/movi-player) | `main` | — | | `115-sdk-go/` | [Ironboxplus/115-sdk-go](https://github.com/Ironboxplus/115-sdk-go) | — | — | -### Module Replacements (Backend `go.mod`) +### Module Replacements (`go.mod`) | Module | Replace Target | Notes | |--------|---------------|-------| @@ -38,497 +32,54 @@ Each section below is labeled by component. The backend section is the primary d --- -## [Backend] Core Development Principles +## Core Development Principles 1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal 2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file 3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload -## [Backend] Build and Development Commands +## Build and Development Commands ```bash # Development go run main.go # Run backend server (default port 5244) -air # Hot reload during development (uses .air.toml) -./build.sh dev # Build development version with frontend +air # Hot reload during development +./build.sh dev # Build dev version with frontend ./build.sh release # Build release version # Testing go test ./... # Run all tests go test ./drivers/115_open/ -v # Run tests for a specific driver -go test ./drivers/115_open/ -run TestCheckUploadCallback -v # Run a single test go build ./drivers/115_open/... # Quick compile check for a package # Docker -docker-compose up # Run with docker-compose -docker build -f Dockerfile . # Build docker image -``` - -**Build Script Details** (`build.sh`): -- Fetches frontend from `$FRONTEND_REPO` (default: `Ironboxplus/OpenList-Frontend`) releases and embeds into `public/dist/` -- Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` -- Supports `dev`, `beta`, and release builds -- Downloads prebuilt frontend distribution automatically - -**Go Version**: Requires Go 1.24+ (CI uses 1.25.0) - -**Module Replacements** (`go.mod`): Some dependencies use `replace` directives pointing to forks (e.g., `115-sdk-go` → `Ironboxplus/115-sdk-go`). When modifying SDK behavior, check if there's a local fork to edit. - -## [Backend] Architecture Overview - -### Driver System (Storage Abstraction) - -OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. - -**Location**: `drivers/*/` - -**Core Interfaces** (`internal/driver/driver.go`): -- `Reader`: List directories, generate download links (REQUIRED) -- `Writer`: Upload, delete, move files (optional) -- `ArchiveDriver`: Extract archives (optional) -- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) - -**Driver Registration Pattern**: -```go -// In drivers/your_driver/meta.go -var config = driver.Config{ - Name: "YourDriver", - LocalSort: false, - NoCache: false, - DefaultRoot: "/", -} - -func init() { - op.RegisterDriver(func() driver.Driver { - return &YourDriver{} - }) -} -``` - -**Adding a New Driver**: -1. Copy `drivers/template/` to `drivers/your_driver/` -2. Implement `List()` and `Link()` methods (required) -3. Define `Addition` struct with configuration fields using struct tags: - - `json:"field_name"` - JSON field name - - `type:"select"` - Input type (select, string, text, bool, number) - - `required:"true"` - Required field - - `options:"a,b,c"` - Dropdown options - - `default:"value"` - Default value -4. Register driver in `init()` function - -**Example Driver Structure**: -```go -type YourDriver struct { - model.Storage - Addition - client *YourClient -} - -func (d *YourDriver) Init(ctx context.Context) error { - // Initialize client, login, etc. -} - -func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - // Return list of files/folders -} - -func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - // Return download URL or RangeReader -} -``` - -### Request Flow - -``` -HTTP Request (Gin Router) - ↓ -Middleware (Auth, CORS, Logging) - ↓ -Handler (server/handles/) - ↓ -fs.List/Get/Link (mount path → storage path conversion) - ↓ -op.List/Get/Link (caching, driver lookup) - ↓ -Driver.List/Link (storage-specific API calls) - ↓ -Response (JSON / Proxy / Redirect) -``` - -### Internal Package Structure - -| Package | Purpose | -|---------|---------| -| `bootstrap/` | Initialization sequence: config, DB, storages, servers | -| `conf/` | Configuration management | -| `db/` | Database models (SQLite/MySQL/Postgres) | -| `driver/` | Driver interface definitions | -| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | -| `op/` | Core operations with caching and driver management | -| `stream/` | Streaming, range readers, link refresh, rate limiting | -| `model/` | Data models (Obj, Link, Storage, User) | -| `cache/` | Multi-level caching (directories, links, users, settings) | -| `net/` | HTTP utilities, proxy config, download manager | - -### Link Generation and Caching - -**Link Types**: -1. **Direct URL** (`link.URL`): Simple redirect to storage provider -2. **RangeReader** (`link.RangeReader`): Custom streaming implementation -3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration - -**Cache System** (`internal/op/cache.go`): -- **Directory Cache**: Stores file listings with configurable TTL -- **Link Cache**: Stores download URLs (30min default) -- **User Cache**: Authentication data (1hr default) -- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format - -**Cache Key Pattern**: `{storageMountPath}/{relativePath}` - -**Invalidation**: Recursive tree deletion for directory operations - -### Range Reader and Streaming - -**Location**: `internal/stream/` - -**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. - -**Key Components**: - -1. **RangeReaderIF**: Core interface for range-based reading - ```go - type RangeReaderIF interface { - RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) - } - ``` - -2. **RefreshableRangeReader**: Wraps RangeReader with automatic link refresh - - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) - - Calls `link.Refresher(ctx)` to get new link - - Resumes download from current byte position - - Max 3 refresh attempts to prevent infinite loops - -3. **Multi-threaded Downloader** (`internal/net/downloader.go`): - - Splits file into parts based on `Concurrency` and `PartSize` - - Downloads parts in parallel - - Assembles final stream - -**Stream Types and Reader Management**: - -⚠️ **CRITICAL**: SeekableStream.Reader must NEVER be created early! - -- **FileStream**: One-time sequential stream (e.g., HTTP body) - - `Reader` is set at creation and consumed sequentially - - Cannot be rewound or re-read - -- **SeekableStream**: Reusable stream with RangeRead capability - - Has `rangeReader` for creating new readers on-demand - - `Reader` should ONLY be created when actually needed for sequential reading - - **DO NOT create Reader early** - use lazy initialization via `generateReader()` - -**Common Pitfall - Early Reader Creation**: -```go -// ❌ WRONG: Creating Reader early -if _, ok := rr.(*model.FileRangeReader); ok { - rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) - fs.Reader = rc // This will be consumed by intermediate operations! -} - -// ✅ CORRECT: Let generateReader() create it on-demand -// Reader will be created only when Read() is called -return &SeekableStream{FileStream: fs, rangeReader: rr}, nil -``` - -**Why This Matters**: -- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead -- If Reader is created early, it may be at EOF when HTTP upload actually needs it -- Result: `http: ContentLength=X with Body length 0` error - -**Hash Calculation for Uploads**: -```go -// For SeekableStream: Use RangeRead to avoid consuming Reader -if _, ok := file.(*SeekableStream); ok { - hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) - // StreamHashFile uses RangeRead internally, Reader remains unused -} - -// For FileStream: Must cache first, then calculate hash -_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) -``` - -**Link Refresh Pattern**: -```go -// In op.Link(), a refresher is automatically attached -link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { - // Get fresh link from storage driver - file, err := GetUnwrap(refreshCtx, storage, path) - newLink, err := storage.Link(refreshCtx, file, args) - return newLink, file, nil -} - -// RefreshableRangeReader uses this during streaming -if IsLinkExpiredError(err) && r.link.Refresher != nil { - newLink, _, err := r.link.Refresher(ctx) - // Resume from current position -} -``` - -**Proxy Function** (`server/common/proxy.go`): - -Handles multiple scenarios: -1. Multi-threaded download (`link.Concurrency > 0`) -2. Direct RangeReader (`link.RangeReader != nil`) -3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader -4. Transparent proxy (forwards to `link.URL`) - -### Frontend Dist Serving - -**Location**: `server/static/static.go`, `internal/frontend/` - -The frontend dist has two sources, with a strict priority: - -1. **Embedded dist** (`public/dist/` via `go:embed`): Baked into the binary at build time. Always used on startup. -2. **Dynamic dist** (fetched by watcher): The `frontend.Watcher` checks GitHub every 30 minutes for a newer rolling release. If found, it downloads to `data/frontend_dist/` and hot-swaps the serving FS via `ReloadStatic()`. - -**Key design rule**: `initStatic()` always starts with the embedded dist (or `dist_dir` if configured). The cached dynamic dist in the data volume is never read on startup — only the watcher can activate it after verifying a newer version exists on GitHub. This prevents stale cache from overriding a newer Docker image. - -**Configuration** (`config.json`): -- `dist_dir`: Override with a custom local directory (highest priority, skips embedded) -- `frontend_repo`: GitHub repo for the watcher to check (default: `Ironboxplus/OpenList-Frontend`) - -**Startup flow**: -``` -initStatic() → embedded dist (or dist_dir) - ↓ -StartWatcher(ReloadStatic) → background goroutine - ↓ (every 30min) -FetchFromRolling() → compare cache version vs GitHub rolling tag commit - ↓ (if newer) -downloadAndExtract() → atomic swap in data/frontend_dist/dist/ - ↓ -ReloadStatic() → swap staticFS to new dist, re-render index.html -``` - -### Startup Sequence - -**Location**: `internal/bootstrap/run.go` - -Order of initialization: -1. `InitConfig()` - Load config, environment variables -2. `Log()` - Initialize logging -3. `InitDB()` - Connect to database -4. `data.InitData()` - Initialize default data -5. `LoadStorages()` - Load and initialize all storage drivers -6. `InitTaskManager()` - Start background tasks -7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers - -## [Backend] Common Patterns - -### Error Handling - -Use custom errors from `internal/errs/`: -- `errs.NotImplement` - Feature not implemented -- `errs.ObjectNotFound` - File/folder not found -- `errs.NotFolder` - Path is not a directory -- `errs.StorageNotInit` - Storage driver not initialized - -**Link Expiry Detection**: -```go -// Checks error string for keywords: "expired", "invalid signature", "token expired" -// Also checks HTTP status: 401, 403, 410, 500 -if stream.IsLinkExpiredError(err) { - // Refresh link -} -``` - -### Upload and OSS Callback Validation - -Drivers that upload via Aliyun OSS (e.g., `115`, `115_open`) use a callback mechanism: after OSS stores the file, it POSTs to the storage provider's callback URL. The provider returns a JSON response indicating whether the file was registered. - -**Critical**: Always capture and validate the callback response: -```go -var bodyBytes []byte -_, err = bucket.CompleteMultipartUpload(imur, parts, - oss.Callback(base64.StdEncoding.EncodeToString([]byte(callback))), - oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(callbackVar))), - oss.CallbackResult(&bodyBytes), // ← MUST capture this -) -// Check both OSS error AND callback response -if err != nil { return err } -// Parse bodyBytes to verify {"state": true} +docker-compose up && docker build -f Dockerfile . ``` -Without `oss.CallbackResult`, the upload appears successful (OSS returns 200) but the file is never registered on the provider's side. The local cache shows the file temporarily, but it vanishes on refresh. - -**`Put` vs `PutResult`**: Drivers can implement either `driver.Put` (returns `error`) or `driver.PutResult` (returns `model.Obj, error`). When `Put` returns nil, `op.Put` creates a temporary object in the directory cache. When `PutResult` returns an actual object, that object is used in the cache instead. - -### HybridCache replaces the old `MaxBufferLimit` truncation - -Historically `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` capped buffering at `MaxBufferLimit` (~48MB), so non-seekable streams larger than that produced a truncated SHA1 (only the first 48MB hashed) and providers like 115 rejected the upload. The 115_open driver carried a `utils.CreateTempFile` workaround. - -**Current (2026-05-17)**: upstream PR #2460 unified caching via `internal/mem.HybridCache` — three tiers (heap memory → `LinearMemory` → temp file fallback) with 16MB blocks (`MaxBlockLimit`). `CacheFullAndWriter` now writes the *entire* stream regardless of size, so the truncation bug is gone for **all drivers**, not just 115_open. The 115_open workaround was removed; the standard `stream.CacheFullAndHash` path is back. - -**Unaffected paths**: -- `SeekableStream` (copy tasks): uses `RangeRead` directly, never goes through `cache()` -- Form uploads: `c.FormFile()` already stores to a temp file, so `GetFile()` returns non-nil and `CacheFullAndWriter` reads the entire file - -### Pass 2 prefetch on `hybridSectionReader` - -For sequential multipart uploads (115_open, baidu_netdisk, aliyundrive_open, etc.), `hybridSectionReader.GetSectionReader` launches a background goroutine to pre-read the next chunk while the caller uploads the current one. The next call picks up the prefetched block; mismatched offsets fall back to synchronous read. Prefetch is clamped to remaining file size, drains on `DiscardSection`, and surfaces errors on the next `GetSectionReader`. Cleanup is registered via `file.Add` so the goroutine is drained before `HybridCache` is freed. - -Drivers that already use `errgroup.Lifecycle` (e.g. 123/upload.go) with `Before` (List/GetSectionReader) and `Do` (upload) get overlap from the lifecycle pattern itself; the section-reader prefetch helps drivers that loop sequentially without Before/Do split (e.g. 115_open, which requires `oss.Sequential()`). - -### Merge-task ObjectNotFound tolerance - -Merge-mode `FileTransferTask.RunWithNextTaskCallback` calls `op.List(dst)` to build the `existedObjs` skip set. A non-existent dst must be treated as "empty" rather than fatal (otherwise resuming an interrupted merge to a fresh dst fails immediately). The logic is extracted into `existingDstFilesFn` in `internal/fs/copy_move.go`: - -```go -dstObjs, err := listDst(ctx, dstPath) -if err != nil && !errors.Is(err, errs.ObjectNotFound) { - return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) -} -// non-existent dst → empty map, merge proceeds and creates dst on demand -``` - -A previous BFS-style "precreate one level of subdirectories" optimization was removed (2026-05-17, commit `6b3ce577`); directory creation now happens on demand via `op.Put`'s internal `MakeDir(parent)` and `op.MakeDir`'s recursive parent walk. The top-level dst `MakeDir` at `copy_move.go:198` is kept for early failure detection. 12 unit tests on `existingDstFilesFn` (including raw + wrapped `ObjectNotFound`) lock in the contract. - -### Saving Driver State - -When updating tokens or credentials: -```go -d.AccessToken = newToken -op.MustSaveDriverStorage(d) // Persists to database -``` - -### Rate Limiting - -Use `rate.Limiter` for API rate limits: -```go -type YourDriver struct { - limiter *rate.Limiter -} - -func (d *YourDriver) Init(ctx context.Context) error { - d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec -} - -func (d *YourDriver) List(...) { - d.limiter.Wait(ctx) - // Make API call -} -``` - -### Context Cancellation - -Always respect context cancellation in long operations: -```go -select { -case <-ctx.Done(): - return nil, ctx.Err() -default: - // Continue operation -} -``` +- Requires Go 1.24+ (CI uses 1.25.0) +- `build.sh` fetches frontend from `$FRONTEND_REPO` (default: `Ironboxplus/OpenList-Frontend`) and embeds into `public/dist/` -## [Backend] Important Conventions +## Key Naming Conventions -**Naming**: -- Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) -- Packages: lowercase (e.g., `internal/op`) -- Interfaces: PascalCase with suffix (e.g., `Reader`, `Writer`) - -**Driver Configuration Fields**: -- Use `driver.RootPath` or `driver.RootID` for root folder -- Add `omitempty` to optional JSON fields -- Use descriptive help text in struct tags - -**Retries and Timeouts**: -- Use `github.com/avast/retry-go` for retry logic -- Set reasonable timeouts on HTTP clients (default 30s in `base.RestyClient`) -- For unstable APIs, implement exponential backoff - -**Logging**: -- Use `logrus` via `log` package -- Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` -- Include driver name in logs: `log.Infof("[driver_name] message")` - -## [Backend] Project Context - -OpenList is a community-driven fork of AList, focused on: -- Long-term governance and trust -- Support for 70+ cloud storage providers -- Web UI for file management -- Multi-protocol support (HTTP, WebDAV, FTP, SFTP, S3) -- Offline downloads (Aria2, Transmission) -- Full-text search -- Archive extraction - -**License**: AGPL-3.0 - ---- - -## Frontend (OpenList-Frontend) - -SolidJS + Vite + Hope UI. Builds to `dist/` which gets embedded into the backend binary via `go:embed`. - -### Build Commands - -```bash -pnpm install # Install dependencies -pnpm dev # Dev server (port 5173) -pnpm build # Production build -pnpm test # Run vitest tests -``` - -### Preview System - -Previews registered in `src/pages/home/previews/index.ts`. Each preview declares `prior: true/false` for priority ordering. Current video priority: movi-player (default) > Artplayer (fallback). - -### Movi Player Integration - -movi-player (FFmpeg WASM + WebCodecs) is the default video player, with full subtitle support: -- **SRT/VTT**: movi-player native parsing -- **ASS (external)**: JASSUB (libass WASM) overlay canvas rendering with font fallback -- **PGS/SUP (external)**: libpgs overlay canvas rendering -- **Embedded subtitles**: movi-player WASM demuxer handles all embedded formats - -Requires COOP/COEP headers (`Cross-Origin-Opener-Policy: same-origin`, `Cross-Origin-Embedder-Policy: credentialless`) for SharedArrayBuffer. Set in backend `server/router.go`. - ---- - -## Movi Player Fork (`movi-player/`) - -Clone of [MrUjjwalG/movi-player](https://github.com/MrUjjwalG/movi-player). FFmpeg WASM + WebCodecs browser-side video decoder. - -### Fork Changes - -- `src/core/MoviPlayer.ts`: Added `parseASS()` + `parseASSTime()` for external ASS subtitle support -- Format detection in `selectSubtitleLang()` extended for `.ass`/`.ssa` - -### Build - -```bash -npm run build:wasm # Requires Docker (C → WASM) -npm run build:ts # TypeScript only (needs dist/wasm/movi.js) -npm run build # Full build (wasm + ts) -``` - -### Known Limitations - -| Limitation | Reason | -|-----------|--------| -| External ASS: text only (no styles) | SubtitleCue architecture is plain text. Full ASS → use JASSUB overlay | -| External PGS: not supported in movi-player native | Binary format needs demuxer. Handled by libpgs overlay | -| Dolby Vision: purple tint | WASM decoder lacks DV enhancement layer | -| Requires COOP/COEP headers | SharedArrayBuffer for WASM threads | +- Drivers: lowercase with underscores (`baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (`internal/op`) +- Interfaces: PascalCase (`Reader`, `Writer`) +- Logging: `log.Infof("[driver_name] message")` via `logrus` +- Retries: `github.com/avast/retry-go`; HTTP default timeout 30s (`base.RestyClient`) --- -## 115-sdk-go Fork - -Fork of upstream SDK at `github.com/Ironboxplus/115-sdk-go`. Key changes (v0.2.9): -- **Concurrent token refresh**: `authRequest` uses `sync.Mutex` + double-check to prevent multiple goroutines from racing on `RefreshToken` -- **Refresh context isolation** (v0.2.9): `RefreshToken` runs under `context.WithTimeout(context.WithoutCancel(ctx), 30s)` — once a refresh starts, caller cancellation cannot abort it. Fixes token loss when video player cancels mid-refresh -- **Token error code filters** (v0.2.9): `shouldRefreshToken(code)` excludes 40140117 (CodeRefreshFrequently) and 40140120 (CodeRefreshTokenError) from triggering refresh; prevents feeding rate limiter on unrecoverable auth failures -- **ErrDataEmpty sentinel**: `GetFolderInfoByPath` returning `data:[]` for non-existent paths → returns `ErrDataEmpty` instead of unmarshal error -- **FlexString CID**: handles numeric/string JSON interop for category IDs +## Documentation Map + +| Document | Content | +|----------|---------| +| [../INDEX.MD](../INDEX.MD) | Subsystem → deep-dive doc → code path index | +| [../OVERVIEW.md](../OVERVIEW.md) | High-level system map | +| [../JOURNAL.md](../JOURNAL.md) | Chronological change log (source of truth) | +| [../PLAN.md](../PLAN.md) | Active development roadmap | +| [docs/architecture.md](docs/architecture.md) | Driver system, request flow, internal packages, startup, common patterns | +| [docs/streaming-and-caching.md](docs/streaming-and-caching.md) | Link types, cache system, RangeReader, SeekableStream pitfalls, HybridCache | +| [docs/uploads.md](docs/uploads.md) | OSS callback validation, Put vs PutResult, merge-task ObjectNotFound tolerance | +| [docs/plugins.md](docs/plugins.md) | Frontend plugin registry/slots + backend yaegi (Go-source) runtime | +| [docs/frontend.md](docs/frontend.md) | Frontend dist serving, movi-player, subtitle handling | +| [docs/new-apis-2026-06-21.md](docs/new-apis-2026-06-21.md) | Storage loading progress, permission bit 16, video_play, favorites, driver_name | diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000000..f6377223c7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,171 @@ +# Architecture — Driver System, Request Flow, Internal Packages, Startup + +Deep-dive reference for the OpenList backend's core structure. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` — JSON field name + - `type:"select"` — Input type (select, string, text, bool, number) + - `required:"true"` — Required field + - `options:"a,b,c"` — Dropdown options + - `default:"value"` — Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { /* initialize client */ } +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { /* list files */ } +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { /* return download URL */ } +``` + +--- + +## Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +--- + +## Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +--- + +## Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` — Load config, environment variables +2. `Log()` — Initialize logging +3. `InitDB()` — Connect to database (`_busy_timeout=5000` for SQLite parallel safety) +4. `data.InitData()` — Initialize default data +5. `LoadStorages()` — Load and initialize all storage drivers (parallel by driver type) +6. `InitTaskManager()` — Start background tasks +7. `InitPlugins()` — Start wazero plugin manager +8. `Start()` — Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +--- + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` — Feature not implemented +- `errs.ObjectNotFound` — File/folder not found +- `errs.NotFolder` — Path is not a directory +- `errs.StorageNotInit` — Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { /* refresh link */ } +``` + +### Saving Driver State + +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +```go +type YourDriver struct { limiter *rate.Limiter } + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +### Naming Conventions + +- Drivers: lowercase with underscores (`baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (`internal/op`) +- Interfaces: PascalCase with suffix (`Reader`, `Writer`) +- Driver config: use `driver.RootPath` or `driver.RootID` for root folder; add `omitempty` to optional JSON fields +- Logging: `log.Infof("[driver_name] message")` via `logrus` +- Retries: `github.com/avast/retry-go`; HTTP client timeout 30s via `base.RestyClient` diff --git a/docs/frontend.md b/docs/frontend.md new file mode 100644 index 0000000000..f5c31c1f88 --- /dev/null +++ b/docs/frontend.md @@ -0,0 +1,110 @@ +# Frontend — Dist Serving, Movi-Player Integration, Subtitle Handling + +Deep-dive reference for the OpenList frontend distribution pipeline and the movi-player video subsystem. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Frontend Dist Serving + +**Backend location**: `server/static/static.go`, `internal/frontend/` + +**Stack**: SolidJS + Vite + Hope UI. Builds to `dist/` which gets embedded into the backend binary via `go:embed`. + +The frontend dist has two sources, with a strict priority: + +1. **Embedded dist** (`public/dist/` via `go:embed`): Baked into the binary at build time. Always used on startup. +2. **Dynamic dist** (fetched by watcher): The `frontend.Watcher` checks GitHub every 30 minutes for a newer rolling release. If found, it downloads to `data/frontend_dist/` and hot-swaps the serving FS via `ReloadStatic()`. + +**Key design rule**: `initStatic()` always starts with the embedded dist (or `dist_dir` if configured). The cached dynamic dist in the data volume is **never** read on startup — only the watcher can activate it after verifying a newer version exists on GitHub. This prevents stale cache from overriding a newer Docker image. + +**Configuration** (`config.json`): +- `dist_dir`: Override with a custom local directory (highest priority, skips embedded) +- `frontend_repo`: GitHub repo for the watcher to check (default: `Ironboxplus/OpenList-Frontend`) + +**Startup flow**: +``` +initStatic() → embedded dist (or dist_dir) + ↓ +StartWatcher(ReloadStatic) → background goroutine + ↓ (every 30min) +FetchFromRolling() → compare cache version vs GitHub rolling tag commit + ↓ (if newer) +downloadAndExtract() → atomic swap in data/frontend_dist/dist/ + ↓ +ReloadStatic() → swap staticFS to new dist, re-render index.html +``` + +--- + +## Frontend Build Commands + +```bash +# From OpenList-Frontend/ +pnpm install # Install dependencies +pnpm dev # Dev server (port 5173) +pnpm build # Production build +pnpm test # Run vitest tests +``` + +--- + +## Preview System + +Previews registered in `src/pages/home/previews/index.ts`. Each preview declares `prior: true/false` for priority ordering. + +Current video priority: **movi-player** (default) > **Artplayer** (fallback). + +--- + +## Movi-Player Integration + +**npm package**: `movi-player ^0.3.2` +**Local reference clone**: `movi-player/` (clean upstream mirror, no local changes) + +movi-player (FFmpeg WASM + WebCodecs) is the default video player. It requires COOP/COEP headers for SharedArrayBuffer — set in backend `server/router.go`: +- `Cross-Origin-Opener-Policy: same-origin` +- `Cross-Origin-Embedder-Policy: credentialless` (chosen over `require-corp` to avoid breaking CDN resources) + +**Subtitle support**: +| Format | Handling | +|--------|---------| +| SRT/VTT (external) | movi-player native parsing | +| ASS (external) | JASSUB (libass WASM) overlay canvas rendering with font fallback | +| PGS/SUP (external) | libpgs overlay canvas rendering | +| All embedded formats | movi-player WASM demuxer | + +**Subtitle manager** (`subtitle-manager.ts`): orchestrates overlay canvases for ASS and PGS formats that movi-player cannot render natively. + +--- + +## Movi-Player npm Package Version + +- Current: **0.3.2** (referenced via `^0.3.2` in package.json) +- Key additions in 0.3.0+: TrueHD/MLP software audio decoding, HDR/DoVi hardware-decode retention, open-GOP HEVC fallback, multichannel passthrough + +**Known Limitations**: +| Limitation | Reason | +|-----------|--------| +| External ASS/PGS not rendered natively | Handled by JASSUB (ASS) and libpgs (PGS) overlay canvases | +| Dolby Vision: purple tint | WASM decoder lacks DV enhancement layer | +| Requires COOP/COEP headers | SharedArrayBuffer for WASM threads | + +--- + +## 115 Official Play Source (`115_video.tsx`) + +Frontend component that renders an Artplayer with quality selector for the `/api/fs/video_play` endpoint. Lists available resolutions sorted high→low. See [new-apis-2026-06-21.md](new-apis-2026-06-21.md) for the backend API details. + +--- + +## Other Frontend Subsystems (vs Upstream) + +| Feature | Key files | +|---------|-----------| +| VideoTreeList | `src/pages/home/previews/video/VideoTreeList.tsx` — tree-structured video browser replacing flat dropdown | +| Mobile toolbar | `src/utils/touch.ts` (triple-detection), `Icon.tsx` (icon+label on touch) | +| Header UserMenu | `user-menu.ts` pure-function entries; `UserMenu.tsx` avatar dropdown; Footer login removed | +| Motion presets | `src/utils/motion-presets.ts` — stagger/fade/scale entry animations | +| Persisted state | `src/utils/persisted.ts` — `createPersistedSignal`; storages driver filter persists across navigation | +| ECharts component | `src/components/EChart.tsx` — tree-shaken PieChart+BarChart, colorMode-aware | +| GridItem overflow | Respects `list_item_filename_overflow` (multi_line / scrollable / ellipsis), consistent with ListItem | diff --git a/docs/new-apis-2026-06-21.md b/docs/new-apis-2026-06-21.md new file mode 100644 index 0000000000..add97be04c --- /dev/null +++ b/docs/new-apis-2026-06-21.md @@ -0,0 +1,73 @@ +# New APIs and Features — 2026-06-21 Session + +Features added in the 2026-06-21 development session. See [INDEX.MD](../../INDEX.MD) for the full subsystem map and [JOURNAL.md](../../JOURNAL.md) for the chronological change log. + +--- + +## Storage Loading Progress — `GET /api/admin/storage/loading` + +Returns a live snapshot of storage initialization state. Requires admin auth. + +```go +// internal/op/loading.go +type LoadingProgress struct { /* thread-safe, states: pending/loading/loaded/failed */ } +func (lp *LoadingProgress) Snapshot() LoadingSnapshot +``` + +**Bootstrap behavior** (`internal/bootstrap/storage.go`): storages grouped by driver type — same driver type loads sequentially (shared rate-limiter / token-refresh safety), different driver types load in parallel. + +**SQLite safety**: `_busy_timeout=5000` set in `internal/bootstrap/db.go` to avoid `SQLITE_BUSY` under parallel writes. + +**Files**: `internal/op/loading.go`, `internal/bootstrap/storage.go`, `server/handles/storage.go` + +--- + +## Permission Bit 16 — `CanManageUserInfo` + +Defined in `internal/model/user.go`. Admin can grant a normal user the ability to edit other users' username/password. Privileged fields (role, permissions, base-path, etc.) are always forced back to original values by `applyUserUpdatePolicy` in `server/handles/user.go`. + +**Route split**: list/get/update under `/api/admin/user` guarded by `AuthUserInfoManage` middleware; create/delete/etc remain admin-only. Frontend hides privileged fields when the editor lacks full admin. + +**Files**: `internal/model/user.go`, `server/middlewares/auth.go`, `server/router.go`, `server/handles/user.go` + +--- + +## `POST /api/fs/video_play` — 115 Official Play Sources + +Optional `VideoPlayer` interface in `internal/driver/video.go`: + +```go +type VideoPlayer interface { + VideoPlay(ctx context.Context, file model.Obj) (*VideoPlayInfo, error) +} +``` + +`drivers/115_open/video.go` implements it via SDK `VideoPlay(pickCode)`. Returns available resolutions sorted high→low. Frontend `115_video.tsx` renders an Artplayer with a quality selector. + +**Files**: `internal/driver/video.go`, `drivers/115_open/video.go`, `server/handles/fsread.go` + +--- + +## Favorites API — `/api/me/favorites` + +User-scoped bookmark store. Model in `internal/model/favorite.go`, CRUD in `internal/db/favorite.go` (registered in `AutoMigrate`), handlers in `server/handles/favorite.go`. + +| Route | Purpose | +|-------|---------| +| `GET /api/me/favorites` | List current user's favorites | +| `POST /api/me/favorites/add` | Add path to favorites (idempotent) | +| `POST /api/me/favorites/delete` | Remove path from favorites | + +**Files**: `internal/model/favorite.go`, `internal/db/favorite.go`, `server/handles/favorite.go`, `server/router.go` + +--- + +## `StorageDetails.DriverName` + +`internal/model/storage.go` — `StorageDetails` now includes a `DriverName string` field populated by `GetStorageDetails` from `storage.Config().Name`. Emitted as `driver_name` in the JSON response. + +The `disk-usage` frontend plugin uses this to display the storage type label. + +**Note**: `/api/fs/list` returns `provider="unknown"` for mount details; `/api/fs/get` returns the real driver name. + +**Files**: `internal/model/storage.go`, `internal/op/` diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 0000000000..5abac61fdf --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,99 @@ +# Plugin System — Frontend Registry/Slots + Backend yaegi Runtime + +Two independent, hot-reloadable plugin planes. The **frontend** plane renders UI +via SolidJS slots; the **backend** plane runs admin-authored Go source in-process +via the [yaegi](https://github.com/traefik/yaegi) interpreter. See +[INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Frontend Plugin System (`OpenList-Frontend/src/plugins/`) + +| File | Role | +|------|------| +| `registry.ts` | Pure TS registry. `register(plugin)` / `unregister(id)` / `selectSlot(name)`. Enable/disable persisted in `localStorage`. Same `id` hot-replaces. | +| `PluginSlot.tsx` | Reactive slot renderer. `` renders all enabled plugins for that slot. | +| `loader.ts` | `loadExternalPlugins(manifest, importer?)` dynamically imports JS assets listed in the backend manifest. | +| `index.ts` | `installBuiltinPlugins()` + `installExternalPlugins()` (fetches `GET /api/plugin/manifest`). | + +**Built-in plugins**: `disk-usage` (header-right; reads `objStore.mountDetails`, updated on every navigation), `favorites` (header-right), `stats` (manage-dashboard ECharts), `media-preview` (behavior-only hover preview). Slot names in use: `header-right`, `manage-dashboard`. + +--- + +## Backend Plugin System (`OP/internal/plugin/`) + +Plugins are plain **`.go` source files** interpreted by yaegi — no compile step, +no WASM toolchain. (Previously this was a wazero/WASM runtime; swapped to yaegi +2026-06-21 so users can author plugins in plain Go.) + +| File | Role | +|------|------| +| `registry.go` | Hook event bus. `Subscribe(pluginID, hook, order, handler)` / `Fire(hook, payload)` / `UnsubscribeAll(pluginID)` / `HasHandlers(hook)`. One handler error never blocks others; `HookContext.Payload` is mutable across the chain. | +| `interp.go` | yaegi `Runtime`: `Load(name, source)` / `Unload` / `Loaded` / `Close`. One interpreter per plugin; same-name `Load` hot-reloads. Exposes `stdlib.Symbols` + a curated `hostSymbols` (this package's `API`, `Handler`, `HookContext`, and `Hook*` constants) — **nothing else of the host**, so plugins cannot import `internal/*`. Defines the `API` interface and per-plugin `pluginAPI`. | +| `hooks.go` | Well-known `Hook*` constants, the `FireHook`/`HasSubscribers` helpers, and injectable capability vars (`SettingGetter`/`SettingSetter`) — kept here so the plugin package never imports `internal/op` (which fires hooks → would be an import cycle). | +| `manager.go` | Watches `/plugins/go/*.go` (mtime+size change detection) and `/plugins/frontend/*.js`. `Start(interval)` polling goroutine; `Sync()` reconciles; tracks per-plugin load errors. `AssetPath` has traversal protection. | +| `manage.go` | User-plugin management: `ListGoPlugins` / `GetGoPluginSource` / `SaveGoPlugin` / `DeleteGoPlugin` / `SetGoPluginEnabled`. Enable/disable toggles the file between `.go` and `.go.disabled`. Name charset is restricted (traversal-proof). | +| `internal/bootstrap/plugin.go` | `InitPlugins()`: wires `SettingGetter/Setter` (via op), bridges `op.RegisterStorageHook` + `op.RegisterObjsUpdateHook` into the plugin registry, starts the watcher. | + +### Plugin contract + +A backend plugin is a single `.go` file: + +```go +package main + +import plugin "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +// Required. Runs once on (re)load. +func OnLoad(api plugin.API) { + api.Log("hello") + api.Subscribe(plugin.HookFsListAfter, 0, func(c *plugin.HookContext) error { + api.Logf("listed %v (%v items)", c.Payload["path"], c.Payload["count"]) + return nil + }) +} + +// Optional. Runs before the plugin is removed or hot-reloaded. +func OnUnload() {} +``` + +`plugin.API` surface: `Name()`, `Subscribe(hook, order, handler)`, `Log(...)`, +`Logf(fmt, ...)`, `GetSetting(key)`, `SetSetting(key, value)`. Plugins may also +import the Go standard library (yaegi blocks `unsafe`, `os/exec`, fork, `os.Exit` +by default). **Trust model**: plugins are admin-installed, trusted code — yaegi is +defense-in-depth, not a hostile-code sandbox. + +### Hook points (fired by the host) + +| Constant | Fired at | Payload | +|----------|----------|---------| +| `HookStorageCreated` / `Updated` / `Deleted` | storage lifecycle (bridged from `op` StorageHook) | `mount_path`, `driver` | +| `HookFsObjsUpdated` | dir contents change (bridged from `op` ObjsUpdateHook) | `path`, `count` | +| `HookFsListAfter` | after a directory is listed (`handles.FsList`) | `path`, `count` | +| `HookFsLinkAfter` | after a link is generated (`handles.Link`) | `path` | +| `HookUserLoginAfter` | after successful login (`handles.loginHash`) | `username` | + +`FireHook` is a no-op when no plugin subscribes, so hot paths gate payload +construction with `HasSubscribers(hook)`. + +### API endpoints + +| Route | Auth | Purpose | +|-------|------|---------| +| `GET /api/plugin/manifest` | Public | Frontend JS asset manifest | +| `GET /api/plugin/asset/:name` | Public | Serve a frontend JS asset (traversal-guarded) | +| `GET /api/admin/plugin/list` | Admin | Backend Go plugins (+ load status/errors) and frontend plugins | +| `GET /api/admin/plugin/get?name=` | Admin | A backend plugin's source | +| `POST /api/admin/plugin/save` | Admin | Create/overwrite a backend plugin (`{name, source}`), loads immediately | +| `POST /api/admin/plugin/delete` | Admin | Delete a backend plugin (`{name}`) | +| `POST /api/admin/plugin/enable` | Admin | Enable/disable (`{name, enabled}`) | + +The management UI is `OpenList-Frontend/src/pages/manage/plugins/Plugins.tsx` +(backend editor + frontend toggles), with the API client in `plugins/api.ts`. + +### Dependency + +`github.com/traefik/yaegi v0.16.1` — builds on the Go 1.25 toolchain; its +interpreter accepts up to **Go 1.22** language level, so plugin source must avoid +1.23+ features (iterators, generic type aliases). Replaces the former +`github.com/tetratelabs/wazero`. diff --git a/docs/streaming-and-caching.md b/docs/streaming-and-caching.md new file mode 100644 index 0000000000..3383929282 --- /dev/null +++ b/docs/streaming-and-caching.md @@ -0,0 +1,146 @@ +# Streaming and Caching — Link Types, Cache System, RangeReader, SeekableStream, HybridCache + +Deep-dive reference for OpenList's streaming pipeline and caching layer. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +--- + +## Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF** — Core interface for range-based reading: + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader** — Wraps RangeReader with automatic link refresh: + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel; assembles final stream + +--- + +## Stream Types and Reader Management + +> **CRITICAL**: `SeekableStream.Reader` must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** — use lazy initialization via `generateReader()` + +**Common Pitfall — Early Reader Creation**: +```go +// WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +--- + +## Hash Calculation for Uploads + +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + +--- + +## Link Refresh Pattern + +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +--- + +## Proxy Function (`server/common/proxy.go`) + +Handles multiple scenarios in priority order: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) — wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +--- + +## HybridCache — Replacing the Old MaxBufferLimit Truncation + +**Background**: Historically `CacheFullAndHash` → `CacheFullAndWriter` → `cache()` capped buffering at `MaxBufferLimit` (~48MB), so non-seekable streams larger than that produced a truncated SHA1 (only the first 48MB hashed) and providers like 115 rejected the upload. The 115_open driver carried a `utils.CreateTempFile` workaround. + +**Current (2026-05-17)**: upstream PR #2460 unified caching via `internal/mem.HybridCache` — three tiers (heap memory → `LinearMemory` → temp file fallback) with 16MB blocks (`MaxBlockLimit`). `CacheFullAndWriter` now writes the *entire* stream regardless of size, so the truncation bug is gone for **all drivers**, not just 115_open. The 115_open workaround was removed; the standard `stream.CacheFullAndHash` path is back. + +**Unaffected paths**: +- `SeekableStream` (copy tasks): uses `RangeRead` directly, never goes through `cache()` +- Form uploads: `c.FormFile()` already stores to a temp file, so `GetFile()` returns non-nil and `CacheFullAndWriter` reads the entire file + +--- + +## Pass 2 Prefetch on `hybridSectionReader` + +For sequential multipart uploads (115_open, baidu_netdisk, aliyundrive_open, etc.), `hybridSectionReader.GetSectionReader` launches a background goroutine to pre-read the next chunk while the caller uploads the current one. The next call picks up the prefetched block; mismatched offsets fall back to synchronous read. Prefetch is clamped to remaining file size, drains on `DiscardSection`, and surfaces errors on the next `GetSectionReader`. Cleanup is registered via `file.Add` so the goroutine is drained before `HybridCache` is freed. + +Drivers that already use `errgroup.Lifecycle` (e.g. 123/upload.go) with `Before` (List/GetSectionReader) and `Do` (upload) get overlap from the lifecycle pattern itself; the section-reader prefetch helps drivers that loop sequentially without Before/Do split (e.g. 115_open, which requires `oss.Sequential()`). diff --git a/docs/uploads.md b/docs/uploads.md new file mode 100644 index 0000000000..2990eb2555 --- /dev/null +++ b/docs/uploads.md @@ -0,0 +1,56 @@ +# Uploads — OSS Callback Validation, Put vs PutResult, Merge-Task Tolerance + +Deep-dive reference for OpenList's upload pipeline. See [INDEX.MD](../../INDEX.MD) for the subsystem map. + +--- + +## OSS Callback Validation + +Drivers that upload via Aliyun OSS (e.g., `115`, `115_open`) use a callback mechanism: after OSS stores the file, it POSTs to the storage provider's callback URL. The provider returns a JSON response indicating whether the file was registered. + +**Critical**: Always capture and validate the callback response: +```go +var bodyBytes []byte +_, err = bucket.CompleteMultipartUpload(imur, parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(callbackVar))), + oss.CallbackResult(&bodyBytes), // ← MUST capture this +) +// Check both OSS error AND callback response +if err != nil { return err } +// Parse bodyBytes to verify {"state": true} +``` + +Without `oss.CallbackResult`, the upload appears successful (OSS returns 200) but the file is never registered on the provider's side. The local cache shows the file temporarily, but it vanishes on refresh. + +--- + +## `Put` vs `PutResult` + +Drivers can implement either: +- `driver.Put` (returns `error`): When `Put` returns nil, `op.Put` creates a temporary object in the directory cache. +- `driver.PutResult` (returns `model.Obj, error`): When `PutResult` returns an actual object, that object is used in the cache instead. + +Use `PutResult` when the storage provider returns metadata about the uploaded file (ID, size, timestamps), so the cache entry is accurate without requiring a re-list. + +--- + +## Merge-Task ObjectNotFound Tolerance + +Merge-mode `FileTransferTask.RunWithNextTaskCallback` calls `op.List(dst)` to build the `existedObjs` skip set. A non-existent dst must be treated as "empty" rather than fatal (otherwise resuming an interrupted merge to a fresh dst fails immediately). The logic is extracted into `existingDstFilesFn` in `internal/fs/copy_move.go`: + +```go +dstObjs, err := listDst(ctx, dstPath) +if err != nil && !errors.Is(err, errs.ObjectNotFound) { + return nil, errors.WithMessagef(err, "failed list dst [%s] objs", dstPath) +} +// non-existent dst → empty map, merge proceeds and creates dst on demand +``` + +A previous BFS-style "precreate one level of subdirectories" optimization was removed (2026-05-17, commit `6b3ce577`); directory creation now happens on demand via `op.Put`'s internal `MakeDir(parent)` and `op.MakeDir`'s recursive parent walk. The top-level dst `MakeDir` at `copy_move.go:198` is kept for early failure detection. 12 unit tests on `existingDstFilesFn` (including raw + wrapped `ObjectNotFound`) lock in the contract. + +--- + +## HybridCache and Prefetch + +See [streaming-and-caching.md](streaming-and-caching.md) for the full HybridCache description and `hybridSectionReader` prefetch details, which directly affect upload throughput for multipart drivers. diff --git a/drivers/115_open/video.go b/drivers/115_open/video.go new file mode 100644 index 0000000000..04b5a6f9b5 --- /dev/null +++ b/drivers/115_open/video.go @@ -0,0 +1,56 @@ +package _115_open + +import ( + "context" + "sort" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// toVideoPlayInfos maps the SDK video-play URLs to driver.VideoPlayInfo, +// sorted by Definition descending (highest quality first). +func toVideoPlayInfos(urls []sdk.VideoPlayURL) []driver.VideoPlayInfo { + infos := make([]driver.VideoPlayInfo, 0, len(urls)) + for _, u := range urls { + infos = append(infos, driver.VideoPlayInfo{ + Resolution: u.Desc, + Definition: u.Definition, + URL: u.URL, + }) + } + sort.SliceStable(infos, func(i, j int) bool { + return infos[i].Definition > infos[j].Definition + }) + return infos +} + +// VideoPlay returns the official online-play (transcoded streaming) sources +// for a video at multiple resolutions. +func (d *Open115) VideoPlay(ctx context.Context, file model.Obj) ([]driver.VideoPlayInfo, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + obj, ok := file.(*Obj) + if !ok { + return nil, errors.New("can't convert obj") + } + pc := obj.Pc + if pc == "" { + return nil, errors.New("can't get pick code") + } + + resp, err := d.client.VideoPlay(ctx, pc) + if err != nil { + log.Errorf("[115] VideoPlay API failed: %v", err) + return nil, errors.WithStack(err) + } + if resp == nil || len(resp.VideoURL) == 0 { + return nil, errors.New("no official play sources") + } + + return toVideoPlayInfos(resp.VideoURL), nil +} diff --git a/drivers/115_open/video_test.go b/drivers/115_open/video_test.go new file mode 100644 index 0000000000..f63bec0ef9 --- /dev/null +++ b/drivers/115_open/video_test.go @@ -0,0 +1,48 @@ +package _115_open + +import ( + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" +) + +func TestToVideoPlayInfos_SortAndMap(t *testing.T) { + urls := []sdk.VideoPlayURL{ + {URL: "http://example.com/720", Definition: 2, Desc: "720P"}, + {URL: "http://example.com/1080", Definition: 4, Desc: "1080P"}, + {URL: "http://example.com/360", Definition: 1, Desc: "360P"}, + } + + got := toVideoPlayInfos(urls) + + if len(got) != 3 { + t.Fatalf("expected 3 infos, got %d", len(got)) + } + + // Sorted by Definition descending (highest quality first). + wantDefinitions := []int{4, 2, 1} + wantResolutions := []string{"1080P", "720P", "360P"} + wantURLs := []string{"http://example.com/1080", "http://example.com/720", "http://example.com/360"} + + for i := range got { + if got[i].Definition != wantDefinitions[i] { + t.Errorf("index %d: Definition = %d, want %d", i, got[i].Definition, wantDefinitions[i]) + } + if got[i].Resolution != wantResolutions[i] { + t.Errorf("index %d: Resolution = %q, want %q", i, got[i].Resolution, wantResolutions[i]) + } + if got[i].URL != wantURLs[i] { + t.Errorf("index %d: URL = %q, want %q", i, got[i].URL, wantURLs[i]) + } + } +} + +func TestToVideoPlayInfos_Empty(t *testing.T) { + got := toVideoPlayInfos(nil) + if got == nil { + t.Fatalf("expected non-nil empty slice, got nil") + } + if len(got) != 0 { + t.Fatalf("expected empty slice, got %d elements", len(got)) + } +} diff --git a/go.mod b/go.mod index 04bda07d6b..28aac1961d 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.24.0 - -toolchain go1.24.13 +go 1.25.0 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 @@ -72,6 +70,7 @@ require ( github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 github.com/tchap/go-patricia/v2 v2.3.3 + github.com/traefik/yaegi v0.16.1 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 @@ -295,7 +294,7 @@ require ( go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/sync v0.19.0 - golang.org/x/sys v0.40.0 + golang.org/x/sys v0.44.0 golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 golang.org/x/tools v0.39.0 // indirect diff --git a/go.sum b/go.sum index 895fbcece9..907b590f21 100644 --- a/go.sum +++ b/go.sum @@ -645,6 +645,8 @@ github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8O github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/traefik/yaegi v0.16.1 h1:f1De3DVJqIDKmnasUF6MwmWv1dSEEat0wcpXhD2On3E= +github.com/traefik/yaegi v0.16.1/go.mod h1:4eVhbPb3LnD2VigQjhYbEJ69vDRFdT2HQNrXx8eEwUY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= @@ -772,8 +774,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 7b91769f9c..751f9b832f 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -50,7 +50,10 @@ func InitDB() { if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) { log.Fatalf("db name error.") } - dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", + // _busy_timeout makes concurrent writers wait for the WAL lock + // instead of failing with SQLITE_BUSY (e.g. parallel storage + // status persistence during startup). + dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental&_busy_timeout=5000", database.DBFile)), gormConfig) } case "mysql": diff --git a/internal/bootstrap/plugin.go b/internal/bootstrap/plugin.go new file mode 100644 index 0000000000..9a453e3b9b --- /dev/null +++ b/internal/bootstrap/plugin.go @@ -0,0 +1,81 @@ +package bootstrap + +import ( + "context" + "path/filepath" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// pluginScanInterval is how often the manager polls the plugins dir for changes +// (hot-reload). +const pluginScanInterval = 30 * time.Second + +// InitPlugins boots the plugin manager: it watches /plugins for backend +// Go-source plugins and frontend JS plugins and keeps them loaded, wires the +// plugin capability functions, and bridges the host's lifecycle hooks into the +// plugin registry so plugins receive real events. Failure is non-fatal — the +// server runs fine without plugins. +func InitPlugins() { + root := filepath.Join(flags.DataDir, "plugins") + mgr, err := plugin.NewManager(context.Background(), root, "/api/plugin/asset") + if err != nil { + utils.Log.Warnf("[plugin] init failed, plugin support disabled: %v", err) + return + } + plugin.Default = mgr + + // Capability wiring. These let plugins read/write settings without the + // plugin package importing internal/op (which would create an import cycle). + plugin.SettingGetter = func(key string) string { + return op.GetSettingsMap()[key] + } + plugin.SettingSetter = func(key, value string) error { + item, err := op.GetSettingItemByKey(key) + if err != nil { + // Unknown key → create a private setting owned by the plugin. + item = &model.SettingItem{ + Key: key, + Type: "string", + Group: model.SINGLE, + Flag: model.PRIVATE, + } + } + item.Value = value + return op.SaveSettingItem(item) + } + + // Bridge the host's existing lifecycle hooks into the plugin registry, so a + // plugin can react to storage and directory changes without us having to + // scatter Fire() calls through op. FireHook is a no-op when nobody listens. + op.RegisterStorageHook(func(typ string, storageDriver driver.Driver) { + hook := map[string]string{ + "add": plugin.HookStorageCreated, + "del": plugin.HookStorageDeleted, + "update": plugin.HookStorageUpdated, + }[typ] + if hook == "" { + return + } + st := storageDriver.GetStorage() + plugin.FireHook(hook, map[string]any{ + "mount_path": st.MountPath, + "driver": st.Driver, + }) + }) + op.RegisterObjsUpdateHook(func(_ context.Context, parent string, objs []model.Obj) { + plugin.FireHook(plugin.HookFsObjsUpdated, map[string]any{ + "path": parent, + "count": len(objs), + }) + }) + + mgr.Start(pluginScanInterval) + utils.Log.Infof("[plugin] manager started, watching %s", root) +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index ff02509baa..853cf997d7 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/frontend" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -38,9 +39,13 @@ func Init() { InitStreamLimit() InitIndex() InitUpgradePatch() + InitPlugins() } func Release() { + if plugin.Default != nil { + _ = plugin.Default.Close() + } db.Close() } diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index 1389095bf5..031b2f5000 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -15,16 +16,51 @@ func LoadStorages() { if err != nil { utils.Log.Fatalf("failed get enabled storages: %+v", err) } - go func(storages []model.Storage) { - for i := range storages { - err := op.LoadStorage(context.Background(), storages[i]) - if err != nil { - utils.Log.Errorf("failed get enabled storages: %+v", err) - } else { - utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", - storages[i].MountPath, storages[i].Driver, storages[i].Order) - } + + // Seed the progress tracker so the status API reports work from the very + // first poll, before any storage has finished loading. + targets := make([]op.LoadTarget, len(storages)) + for i := range storages { + targets[i] = op.LoadTarget{MountPath: storages[i].MountPath, Driver: storages[i].Driver} + } + op.StorageLoadProgress.Begin(targets) + + // Group by driver, preserving order. Storages of the SAME driver must load + // sequentially — they often share a rate limiter, token-refresh mutex or + // login endpoint, so concurrent init of the same driver type races. Different + // driver types are independent and load in parallel. + groupOrder := make([]string, 0) + groups := make(map[string][]model.Storage) + for i := range storages { + d := storages[i].Driver + if _, ok := groups[d]; !ok { + groupOrder = append(groupOrder, d) + } + groups[d] = append(groups[d], storages[i]) + } + + go func() { + var wg sync.WaitGroup + for _, d := range groupOrder { + wg.Add(1) + go func(group []model.Storage) { + defer wg.Done() + for _, storage := range group { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadLoading, "") + err := op.LoadStorage(context.Background(), storage) + if err != nil { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadFailed, err.Error()) + utils.Log.Errorf("failed load storage: [%s], driver: [%s]: %+v", + storage.MountPath, storage.Driver, err) + } else { + op.StorageLoadProgress.SetState(storage.MountPath, op.LoadLoaded, "") + utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", + storage.MountPath, storage.Driver, storage.Order) + } + } + }(groups[d]) } + wg.Wait() conf.SendStoragesLoadedSignal() - }(storages) + }() } diff --git a/internal/db/db.go b/internal/db/db.go index 96529c15d3..a6bda52b3c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.Favorite)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/favorite.go b/internal/db/favorite.go new file mode 100644 index 0000000000..1cbeb6d5ff --- /dev/null +++ b/internal/db/favorite.go @@ -0,0 +1,44 @@ +package db + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func CreateFavorite(f *model.Favorite) error { + return errors.WithStack(db.Create(f).Error) +} + +func UpdateFavorite(f *model.Favorite) error { + return errors.WithStack(db.Save(f).Error) +} + +// DeleteFavorite removes a favorite scoped to the owning user so users can't +// delete favorites belonging to others. +func DeleteFavorite(userID, id uint) error { + return errors.WithStack( + db.Where("user_id = ? AND id = ?", userID, id).Delete(&model.Favorite{}).Error, + ) +} + +func GetFavoritesByUser(userID uint) ([]model.Favorite, error) { + var favorites []model.Favorite + if err := db.Where("user_id = ?", userID).Order(columnName("created_at") + " DESC").Find(&favorites).Error; err != nil { + return nil, errors.Wrapf(err, "failed get favorites") + } + return favorites, nil +} + +// GetFavoriteByUserAndPath looks up a single favorite by (userID, path) for +// dedup/toggle. Returns (nil, nil) when no matching favorite exists. +func GetFavoriteByUserAndPath(userID uint, path string) (*model.Favorite, error) { + var f model.Favorite + if err := db.Where("user_id = ? AND path = ?", userID, path).First(&f).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, errors.Wrapf(err, "failed get favorite") + } + return &f, nil +} diff --git a/internal/driver/video.go b/internal/driver/video.go new file mode 100644 index 0000000000..b6b7e0d884 --- /dev/null +++ b/internal/driver/video.go @@ -0,0 +1,20 @@ +package driver + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// VideoPlayInfo describes one official transcoded online-play source for a video. +type VideoPlayInfo struct { + Resolution string `json:"resolution"` // human readable, e.g. "1080P" + Definition int `json:"definition"` + URL string `json:"url"` +} + +// VideoPlayer is an optional driver capability that exposes the storage +// provider's official online-play (transcoded streaming) sources for a video. +type VideoPlayer interface { + VideoPlay(ctx context.Context, file model.Obj) ([]VideoPlayInfo, error) +} diff --git a/internal/model/favorite.go b/internal/model/favorite.go new file mode 100644 index 0000000000..304db0b140 --- /dev/null +++ b/internal/model/favorite.go @@ -0,0 +1,11 @@ +package model + +type Favorite struct { + ID uint `json:"id" gorm:"primaryKey"` + UserID uint `json:"user_id" gorm:"index"` + Path string `json:"path"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Tag string `json:"tag"` // optional free-text label + CreatedAt int64 `json:"created_at"` +} diff --git a/internal/model/storage.go b/internal/model/storage.go index a6b4745a60..264d7b2a64 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -77,6 +77,18 @@ func (d DiskUsage) MarshalJSON() ([]byte, error) { type StorageDetails struct { DiskUsage + // DriverName is the storage's driver/type (e.g. "Local", "BaiduNetdisk"), + // surfaced to the UI so it can label which storage a mount belongs to. + DriverName string +} + +func (s StorageDetails) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "total_space": s.TotalSpace, + "used_space": s.UsedSpace, + "free_space": s.FreeSpace(), + "driver_name": s.DriverName, + }) } type ObjWithStorageDetails interface { diff --git a/internal/model/storage_details_test.go b/internal/model/storage_details_test.go new file mode 100644 index 0000000000..eec33efde4 --- /dev/null +++ b/internal/model/storage_details_test.go @@ -0,0 +1,27 @@ +package model + +import ( + "encoding/json" + "testing" +) + +func TestStorageDetailsMarshalIncludesDriverName(t *testing.T) { + d := StorageDetails{ + DiskUsage: DiskUsage{TotalSpace: 100, UsedSpace: 40}, + DriverName: "BaiduNetdisk", + } + b, err := json.Marshal(d) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + if m["driver_name"] != "BaiduNetdisk" { + t.Fatalf("expected driver_name in JSON, got %v", m["driver_name"]) + } + if m["total_space"].(float64) != 100 || m["free_space"].(float64) != 60 { + t.Fatalf("disk fields wrong: %v", m) + } +} diff --git a/internal/model/user.go b/internal/model/user.go index 2dacd752eb..8043ba822f 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -64,6 +64,7 @@ type User struct { // 13: can decompress archives // 14: can share // 15: can customize share id + // 16: can manage other users' profiles (delegated by admin: edit username/password only) Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -228,6 +229,17 @@ func (u *User) CanCustomizeShareID() bool { return CanCustomizeShareID(u.Permission) } +func CanManageUserInfo(permission int32) bool { + return (permission>>16)&1 == 1 +} + +// CanManageUserInfo reports whether this user may edit other users' profiles +// (username/password). Admins always can; non-admins only when delegated this +// permission bit by an admin. +func (u *User) CanManageUserInfo() bool { + return u.IsAdmin() || CanManageUserInfo(u.Permission) +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/model/user_test.go b/internal/model/user_test.go new file mode 100644 index 0000000000..023e937e70 --- /dev/null +++ b/internal/model/user_test.go @@ -0,0 +1,34 @@ +package model + +import "testing" + +func TestCanManageUserInfoBit(t *testing.T) { + // Bit 16 is the delegated "manage user info" permission. + const bit int32 = 1 << 16 + + if CanManageUserInfo(0) { + t.Fatal("no permission bits should mean no delegated user management") + } + if !CanManageUserInfo(bit) { + t.Fatal("bit 16 set should grant delegated user management") + } + // Lower bits must not be mistaken for bit 16. + if CanManageUserInfo(0xFFFF) { + t.Fatal("bits 0-15 must not imply user management") + } +} + +func TestUserCanManageUserInfoAdminAlways(t *testing.T) { + admin := &User{Role: ADMIN, Permission: 0} + if !admin.CanManageUserInfo() { + t.Fatal("admin must always be able to manage user info") + } + normal := &User{Role: GENERAL, Permission: 0} + if normal.CanManageUserInfo() { + t.Fatal("a normal user without the bit must not manage user info") + } + delegated := &User{Role: GENERAL, Permission: 1 << 16} + if !delegated.CanManageUserInfo() { + t.Fatal("a normal user with bit 16 must be able to manage user info") + } +} diff --git a/internal/op/loading.go b/internal/op/loading.go new file mode 100644 index 0000000000..e05b5611af --- /dev/null +++ b/internal/op/loading.go @@ -0,0 +1,125 @@ +package op + +import "sync" + +// StorageLoadState is the lifecycle state of a single storage during startup load. +type StorageLoadState string + +const ( + LoadPending StorageLoadState = "pending" + LoadLoading StorageLoadState = "loading" + LoadLoaded StorageLoadState = "loaded" + LoadFailed StorageLoadState = "failed" +) + +func (s StorageLoadState) terminal() bool { + return s == LoadLoaded || s == LoadFailed +} + +// LoadTarget is the minimal description of a storage to be loaded. Kept free of +// model.Storage so the tracker stays trivially testable. +type LoadTarget struct { + MountPath string + Driver string +} + +// StorageLoadInfo is the public, snapshot-safe view of one storage's load state. +type StorageLoadInfo struct { + MountPath string `json:"mount_path"` + Driver string `json:"driver"` + State StorageLoadState `json:"state"` + Error string `json:"error,omitempty"` +} + +// ProgressSnapshot is an immutable view of the overall loading progress, safe to +// serialise to JSON and hand to the frontend status bar. +type ProgressSnapshot struct { + Total int `json:"total"` + Pending int `json:"pending"` + Loading int `json:"loading"` + Loaded int `json:"loaded"` + Failed int `json:"failed"` + Finished bool `json:"finished"` + Items []StorageLoadInfo `json:"items"` +} + +// LoadingProgress tracks per-storage load state during startup. All methods are +// safe for concurrent use, so storages may be loaded in parallel while the HTTP +// API polls Snapshot(). +type LoadingProgress struct { + mu sync.RWMutex + items map[string]*StorageLoadInfo + order []string + started bool +} + +func NewLoadingProgress() *LoadingProgress { + return &LoadingProgress{items: map[string]*StorageLoadInfo{}} +} + +// Begin (re)initialises the tracker with all targets in the pending state. +func (p *LoadingProgress) Begin(targets []LoadTarget) { + p.mu.Lock() + defer p.mu.Unlock() + p.items = make(map[string]*StorageLoadInfo, len(targets)) + p.order = make([]string, 0, len(targets)) + p.started = true + for _, t := range targets { + if _, ok := p.items[t.MountPath]; ok { + continue + } + p.items[t.MountPath] = &StorageLoadInfo{ + MountPath: t.MountPath, + Driver: t.Driver, + State: LoadPending, + } + p.order = append(p.order, t.MountPath) + } +} + +// SetState updates the state (and optional error) of one storage. Unknown mount +// paths are ignored so a late rename can't corrupt the counts. +func (p *LoadingProgress) SetState(mountPath string, state StorageLoadState, errMsg string) { + p.mu.Lock() + defer p.mu.Unlock() + info, ok := p.items[mountPath] + if !ok { + return + } + info.State = state + info.Error = errMsg +} + +// Snapshot returns a deep copy of the current progress. +func (p *LoadingProgress) Snapshot() ProgressSnapshot { + p.mu.RLock() + defer p.mu.RUnlock() + snap := ProgressSnapshot{ + Total: len(p.order), + Items: make([]StorageLoadInfo, 0, len(p.order)), + } + terminal := 0 + for _, mp := range p.order { + info := p.items[mp] + switch info.State { + case LoadPending: + snap.Pending++ + case LoadLoading: + snap.Loading++ + case LoadLoaded: + snap.Loaded++ + case LoadFailed: + snap.Failed++ + } + if info.State.terminal() { + terminal++ + } + snap.Items = append(snap.Items, *info) + } + snap.Finished = p.started && terminal == len(p.order) + return snap +} + +// StorageLoadProgress is the global tracker populated by bootstrap.LoadStorages +// and read by the storage loading-status API. +var StorageLoadProgress = NewLoadingProgress() diff --git a/internal/op/loading_test.go b/internal/op/loading_test.go new file mode 100644 index 0000000000..940afd1497 --- /dev/null +++ b/internal/op/loading_test.go @@ -0,0 +1,113 @@ +package op + +import "testing" + +func items() []LoadTarget { + return []LoadTarget{ + {MountPath: "/a", Driver: "Local"}, + {MountPath: "/b", Driver: "OneDrive"}, + {MountPath: "/c", Driver: "Local"}, + } +} + +func TestLoadingProgressBeginInitialisesPending(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + s := p.Snapshot() + if s.Total != 3 { + t.Fatalf("expected total 3, got %d", s.Total) + } + if s.Pending != 3 { + t.Fatalf("expected 3 pending, got %d", s.Pending) + } + if s.Finished { + t.Fatal("should not be finished right after Begin") + } + if len(s.Items) != 3 || s.Items[0].MountPath != "/a" { + t.Fatalf("expected ordered items, got %+v", s.Items) + } + if s.Items[0].State != LoadPending { + t.Fatalf("expected pending state, got %s", s.Items[0].State) + } +} + +func TestLoadingProgressTransitions(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + + p.SetState("/a", LoadLoading, "") + if got := p.Snapshot().Loading; got != 1 { + t.Fatalf("expected 1 loading, got %d", got) + } + + p.SetState("/a", LoadLoaded, "") + p.SetState("/b", LoadLoaded, "") + s := p.Snapshot() + if s.Loaded != 2 { + t.Fatalf("expected 2 loaded, got %d", s.Loaded) + } + if s.Finished { + t.Fatal("should not be finished with one storage left") + } +} + +func TestLoadingProgressFailedRecordsError(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/a", LoadFailed, "boom") + var found *StorageLoadInfo + for i := range p.Snapshot().Items { + if p.Snapshot().Items[i].MountPath == "/a" { + found = &p.Snapshot().Items[i] + } + } + if found == nil || found.State != LoadFailed || found.Error != "boom" { + t.Fatalf("expected failed with error, got %+v", found) + } + if p.Snapshot().Failed != 1 { + t.Fatalf("expected 1 failed, got %d", p.Snapshot().Failed) + } +} + +func TestLoadingProgressFinishedWhenAllTerminal(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/a", LoadLoaded, "") + p.SetState("/b", LoadFailed, "x") + p.SetState("/c", LoadLoaded, "") + s := p.Snapshot() + if !s.Finished { + t.Fatal("expected finished when all storages reached a terminal state") + } + if s.Loaded != 2 || s.Failed != 1 || s.Pending != 0 || s.Loading != 0 { + t.Fatalf("unexpected counts: %+v", s) + } +} + +func TestLoadingProgressEmptyIsFinished(t *testing.T) { + p := NewLoadingProgress() + p.Begin(nil) + s := p.Snapshot() + if s.Total != 0 || !s.Finished { + t.Fatalf("empty progress should be finished, got %+v", s) + } +} + +func TestLoadingProgressSetStateUnknownMountIsNoop(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + p.SetState("/does-not-exist", LoadLoaded, "") + if p.Snapshot().Loaded != 0 { + t.Fatal("setting state on an unknown mount must not change counts") + } +} + +func TestLoadingProgressSnapshotIsCopy(t *testing.T) { + p := NewLoadingProgress() + p.Begin(items()) + s := p.Snapshot() + s.Items[0].State = LoadLoaded // mutate the copy + if p.Snapshot().Items[0].State != LoadPending { + t.Fatal("Snapshot must return a copy; internal state was mutated") + } +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 2e93bf5693..2390069c9b 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -486,6 +486,9 @@ func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bo if err != nil { return nil, err } + // Label the details with the storage's driver type so the UI can show + // which kind of storage the current mount is. + ret.DriverName = storage.Config().Name Cache.SetStorageDetails(storage, ret) return ret, nil }) diff --git a/internal/plugin/hooks.go b/internal/plugin/hooks.go new file mode 100644 index 0000000000..7a4af93def --- /dev/null +++ b/internal/plugin/hooks.go @@ -0,0 +1,60 @@ +package plugin + +// Well-known hook points the host fires. Plugins Subscribe to these by name. +// Handlers may read and mutate ctx.Payload; mutations are visible to later +// handlers and to the host. +const ( + // Storage lifecycle. Payload: {"mount_path": string, "driver": string}. + HookStorageCreated = "storage.created" + HookStorageUpdated = "storage.updated" + HookStorageDeleted = "storage.deleted" + + // HookFsObjsUpdated fires after a directory's contents change (upload, move, + // rename, copy, delete, mkdir). Payload: {"path": string, "count": int}. + HookFsObjsUpdated = "fs.objs.updated" + + // HookFsListAfter fires after a directory is listed. + // Payload: {"path": string, "count": int, "provider": string}. + HookFsListAfter = "fs.list.after" + + // HookFsLinkAfter fires after a download link is generated. + // Payload: {"path": string}. + HookFsLinkAfter = "fs.link.after" + + // HookUserLoginAfter fires after a successful login. + // Payload: {"username": string}. + HookUserLoginAfter = "user.login.after" +) + +// Capability functions injected by bootstrap. They live here (rather than the +// plugin package importing internal/op directly) to avoid an import cycle: op +// fires plugin hooks, so plugin must not import op. Nil-safe — an unwired +// capability is simply unavailable to plugins. +var ( + SettingGetter func(key string) string + SettingSetter func(key, value string) error +) + +// FireHook fires a hook on the process-wide plugin registry, if one exists and +// actually has subscribers. It is a cheap no-op otherwise, so callers on hot +// paths (list, link) can invoke it unconditionally without building a payload +// when nothing is listening — guard the payload construction with HasSubscribers. +func FireHook(hook string, payload map[string]any) { + if Default == nil { + return + } + reg := Default.Registry() + if !reg.HasHandlers(hook) { + return + } + _, _ = reg.Fire(hook, payload) +} + +// HasSubscribers reports whether any plugin is listening on a hook. Callers on +// hot paths use it to skip building a payload when there are no subscribers. +func HasSubscribers(hook string) bool { + if Default == nil { + return false + } + return Default.Registry().HasHandlers(hook) +} diff --git a/internal/plugin/interp.go b/internal/plugin/interp.go new file mode 100644 index 0000000000..0caf3cd254 --- /dev/null +++ b/internal/plugin/interp.go @@ -0,0 +1,206 @@ +package plugin + +import ( + "fmt" + "reflect" + "sync" + + log "github.com/sirupsen/logrus" + "github.com/traefik/yaegi/interp" + "github.com/traefik/yaegi/stdlib" +) + +// Runtime is a yaegi-backed host for Go-source plugins. Plugins are plain `.go` +// files interpreted in-process — no compile step, no WASM toolchain. Reloading a +// plugin under a name that already exists atomically replaces it (hot-reload). +// +// Coupling boundary: interpreted plugins may import ONLY this package (for the +// API/HookContext types) plus the Go standard library. The host's internal/* +// packages are never registered with the interpreter, so a plugin literally +// cannot reach into backend internals — keeping plugins decoupled from churn. +// +// Trust model: yaegi blocks `unsafe`, `os/exec`, fork and `os.Exit` by default, +// but it is NOT a hostile-code sandbox. Plugins are admin-installed, trusted code. +type Runtime struct { + mu sync.Mutex + reg *Registry + plugins map[string]*loadedPlugin +} + +// loadedPlugin tracks a live interpreter and its optional teardown callback. +type loadedPlugin struct { + interp *interp.Interpreter + onUnload func() +} + +// API is the surface a backend plugin uses to talk to the host. It is the single +// coupling point between plugins and the server — deliberately small and stable. +type API interface { + // Name returns the plugin's name (its file name without extension). + Name() string + // Subscribe registers handler for a named hook. Lower order runs first. + // Subscribe to the well-known Hook* constants exported by this package. + Subscribe(hook string, order int, handler Handler) + // Log writes a line to the server log, tagged with the plugin's name. + Log(args ...any) + // Logf is the printf-style variant of Log. + Logf(format string, args ...any) + // GetSetting reads a server setting by key ("" if unset/unavailable). + GetSetting(key string) string + // SetSetting persists a server setting. Use a plugin-namespaced key. + SetSetting(key, value string) error +} + +// pluginAPI is the per-plugin API implementation, pre-bound to the plugin's name +// and the shared registry so Subscribe attributes hooks to the right plugin +// (which is what makes UnsubscribeAll-on-reload correct). +type pluginAPI struct { + name string + reg *Registry +} + +func (a *pluginAPI) Name() string { return a.name } + +func (a *pluginAPI) Subscribe(hook string, order int, handler Handler) { + a.reg.Subscribe(a.name, hook, order, handler) +} + +func (a *pluginAPI) Log(args ...any) { + log.Infof("[go-plugin %s] %s", a.name, fmt.Sprint(args...)) +} + +func (a *pluginAPI) Logf(format string, args ...any) { + log.Infof("[go-plugin %s] %s", a.name, fmt.Sprintf(format, args...)) +} + +func (a *pluginAPI) GetSetting(key string) string { + if SettingGetter == nil { + return "" + } + return SettingGetter(key) +} + +func (a *pluginAPI) SetSetting(key, value string) error { + if SettingSetter == nil { + return fmt.Errorf("setting persistence is not available") + } + return SettingSetter(key, value) +} + +// hostSymbols exposes this package's plugin-facing types to interpreted plugins +// under its real import path, so a plugin can: +// +// import plugin "github.com/OpenListTeam/OpenList/v4/internal/plugin" +// +// Only API, Handler and HookContext are exported — nothing else of the host. +var hostSymbols = interp.Exports{ + "github.com/OpenListTeam/OpenList/v4/internal/plugin/plugin": map[string]reflect.Value{ + "API": reflect.ValueOf((*API)(nil)), + "Handler": reflect.ValueOf((*Handler)(nil)), + "HookContext": reflect.ValueOf((*HookContext)(nil)), + // Well-known hook names, so plugins can use the constants instead of + // hard-coding strings. + "HookStorageCreated": reflect.ValueOf(HookStorageCreated), + "HookStorageUpdated": reflect.ValueOf(HookStorageUpdated), + "HookStorageDeleted": reflect.ValueOf(HookStorageDeleted), + "HookFsObjsUpdated": reflect.ValueOf(HookFsObjsUpdated), + "HookFsListAfter": reflect.ValueOf(HookFsListAfter), + "HookFsLinkAfter": reflect.ValueOf(HookFsLinkAfter), + "HookUserLoginAfter": reflect.ValueOf(HookUserLoginAfter), + }, +} + +// NewRuntime builds a runtime that subscribes plugins into reg. +func NewRuntime(reg *Registry) *Runtime { + return &Runtime{reg: reg, plugins: map[string]*loadedPlugin{}} +} + +// Load interprets a Go-source plugin under name and runs its entry point. The +// plugin must declare `package main` and export `func OnLoad(api plugin.API)`. +// It may optionally export `func OnUnload()`, called before the plugin is +// dropped or hot-reloaded. If a plugin with the same name is already loaded its +// teardown runs and its hooks are dropped first, so Load doubles as hot-reload. +func (r *Runtime) Load(name string, source []byte) error { + i := interp.New(interp.Options{}) + if err := i.Use(stdlib.Symbols); err != nil { + return fmt.Errorf("use stdlib: %w", err) + } + if err := i.Use(hostSymbols); err != nil { + return fmt.Errorf("use host symbols: %w", err) + } + if _, err := i.Eval(string(source)); err != nil { + return fmt.Errorf("eval %q: %w", name, err) + } + v, err := i.Eval("main.OnLoad") + if err != nil { + return fmt.Errorf("plugin %q must export func OnLoad(plugin.API): %w", name, err) + } + onLoad, ok := v.Interface().(func(API)) + if !ok { + return fmt.Errorf("plugin %q OnLoad has wrong signature, want func(plugin.API)", name) + } + // Optional teardown hook. + var onUnload func() + if uv, uerr := i.Eval("main.OnUnload"); uerr == nil { + onUnload, _ = uv.Interface().(func()) + } + + // Tear down any previous version before the new one takes over. + r.teardown(name) + + r.mu.Lock() + r.plugins[name] = &loadedPlugin{interp: i, onUnload: onUnload} + r.mu.Unlock() + + onLoad(&pluginAPI{name: name, reg: r.reg}) + return nil +} + +// teardown runs a plugin's OnUnload (if any), drops its hook subscriptions, and +// removes it from the live set. Safe to call for an unknown name. +func (r *Runtime) teardown(name string) { + r.mu.Lock() + lp, ok := r.plugins[name] + delete(r.plugins, name) + r.mu.Unlock() + if !ok { + return + } + if lp.onUnload != nil { + func() { + // A misbehaving teardown must not take down the server. + defer func() { + if rec := recover(); rec != nil { + log.Warnf("[plugin] %q OnUnload panicked: %v", name, rec) + } + }() + lp.onUnload() + }() + } + r.reg.UnsubscribeAll(name) +} + +// Unload removes a plugin: runs its teardown and drops all of its hooks. +func (r *Runtime) Unload(name string) { r.teardown(name) } + +// Loaded reports whether a plugin is currently loaded. +func (r *Runtime) Loaded(name string) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.plugins[name] + return ok +} + +// Close tears down all loaded plugins. +func (r *Runtime) Close() error { + r.mu.Lock() + names := make([]string, 0, len(r.plugins)) + for name := range r.plugins { + names = append(names, name) + } + r.mu.Unlock() + for _, name := range names { + r.teardown(name) + } + return nil +} diff --git a/internal/plugin/interp_test.go b/internal/plugin/interp_test.go new file mode 100644 index 0000000000..fd391e854c --- /dev/null +++ b/internal/plugin/interp_test.go @@ -0,0 +1,140 @@ +package plugin + +import "testing" + +// A minimal, valid plugin: subscribes to a hook and mutates the payload. +const demoPlugin = ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Log("demo loaded") + api.Subscribe("demo", 0, func(c *plug.HookContext) error { + c.Payload["touched"] = "v1" + return nil + }) +} +` + +// Same hook, different payload value — used to verify hot-reload replaces the +// old handler instead of stacking a second one. +const demoPluginV2 = ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Subscribe("demo", 0, func(c *plug.HookContext) error { + c.Payload["touched"] = "v2" + return nil + }) +} +` + +func TestRuntimeLoadAndFire(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("demo", []byte(demoPlugin)); err != nil { + t.Fatalf("Load: %v", err) + } + if !rt.Loaded("demo") { + t.Fatal("expected demo loaded") + } + ctx, errs := reg.Fire("demo", map[string]any{}) + if len(errs) != 0 { + t.Fatalf("unexpected handler errors: %v", errs) + } + if ctx.Payload["touched"] != "v1" { + t.Fatalf("expected payload touched=v1, got %v", ctx.Payload["touched"]) + } +} + +func TestRuntimeHotReloadReplacesHandlers(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("demo", []byte(demoPlugin)); err != nil { + t.Fatalf("first load: %v", err) + } + if err := rt.Load("demo", []byte(demoPluginV2)); err != nil { + t.Fatalf("reload: %v", err) + } + // Exactly one handler must remain, producing v2 (not v1, and not both). + if got := len(reg.Handlers("demo")); got != 1 { + t.Fatalf("expected 1 handler after reload, got %d", got) + } + ctx, _ := reg.Fire("demo", map[string]any{}) + if ctx.Payload["touched"] != "v2" { + t.Fatalf("expected v2 after reload, got %v", ctx.Payload["touched"]) + } +} + +func TestRuntimeUnloadDropsHooks(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + _ = rt.Load("demo", []byte(demoPlugin)) + rt.Unload("demo") + if rt.Loaded("demo") { + t.Fatal("expected demo unloaded") + } + if got := len(reg.Handlers("demo")); got != 0 { + t.Fatalf("expected 0 handlers after unload, got %d", got) + } +} + +func TestRuntimeBadSourceErrors(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + if err := rt.Load("bad", []byte("package main\nthis is not go")); err == nil { + t.Fatal("expected error for invalid plugin source") + } + if rt.Loaded("bad") { + t.Fatal("a plugin that failed to eval must not be considered loaded") + } +} + +// Verifies the exported Hook* constants are usable from interpreted plugins. +func TestRuntimeHookConstantsExposed(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + src := ` +package main + +import plug "github.com/OpenListTeam/OpenList/v4/internal/plugin" + +func OnLoad(api plug.API) { + api.Subscribe(plug.HookFsListAfter, 0, func(c *plug.HookContext) error { + c.Payload["seen"] = true + return nil + }) +} +` + if err := rt.Load("c", []byte(src)); err != nil { + t.Fatalf("Load using Hook constant: %v", err) + } + ctx, _ := reg.Fire(HookFsListAfter, map[string]any{}) + if ctx.Payload["seen"] != true { + t.Fatal("expected handler subscribed via Hook constant to fire") + } +} + +func TestRuntimeMissingOnLoadErrors(t *testing.T) { + reg := NewRegistry() + rt := NewRuntime(reg) + defer rt.Close() + + src := "package main\nvar X = 1\n" + if err := rt.Load("noentry", []byte(src)); err == nil { + t.Fatal("expected error when plugin has no OnLoad") + } +} diff --git a/internal/plugin/manage.go b/internal/plugin/manage.go new file mode 100644 index 0000000000..600df59fb8 --- /dev/null +++ b/internal/plugin/manage.go @@ -0,0 +1,159 @@ +package plugin + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// A disabled plugin keeps its source on disk under this suffix so Sync skips it +// (only *.go is loaded). Enabling renames it back to .go. +const disabledSuffix = ".go.disabled" + +// pluginNameRe constrains plugin names to a safe, path-traversal-proof charset. +var pluginNameRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// ValidPluginName reports whether name is an acceptable backend plugin name. +func ValidPluginName(name string) bool { return pluginNameRe.MatchString(name) } + +// PluginInfo describes a backend Go plugin for the management UI. +type PluginInfo struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Loaded bool `json:"loaded"` + Error string `json:"error,omitempty"` +} + +func (m *Manager) enabledPath(name string) string { + return filepath.Join(m.goDir(), name+".go") +} + +func (m *Manager) disabledPath(name string) string { + return filepath.Join(m.goDir(), name+disabledSuffix) +} + +// ListGoPlugins returns every backend plugin on disk (enabled or disabled) with +// its current load status and last load error, sorted by name. +func (m *Manager) ListGoPlugins() []PluginInfo { + entries, _ := os.ReadDir(m.goDir()) + seen := map[string]*PluginInfo{} + for _, e := range entries { + if e.IsDir() { + continue + } + n := e.Name() + switch { + case strings.HasSuffix(n, disabledSuffix): + name := strings.TrimSuffix(n, disabledSuffix) + seen[name] = &PluginInfo{Name: name, Enabled: false} + case strings.HasSuffix(n, ".go"): + name := strings.TrimSuffix(n, ".go") + seen[name] = &PluginInfo{Name: name, Enabled: true} + } + } + m.mu.Lock() + errs := make(map[string]string, len(m.loadErrors)) + for k, v := range m.loadErrors { + errs[k] = v + } + m.mu.Unlock() + + out := make([]PluginInfo, 0, len(seen)) + for name, info := range seen { + info.Loaded = m.runtime.Loaded(name) + info.Error = errs[name] + out = append(out, *info) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// GetGoPluginSource returns a plugin's source (enabled or disabled variant). +func (m *Manager) GetGoPluginSource(name string) (string, error) { + if !ValidPluginName(name) { + return "", fmt.Errorf("invalid plugin name %q", name) + } + if b, err := os.ReadFile(m.enabledPath(name)); err == nil { + return string(b), nil + } + b, err := os.ReadFile(m.disabledPath(name)) + if err != nil { + return "", fmt.Errorf("plugin %q not found", name) + } + return string(b), nil +} + +// SaveGoPlugin writes (creates or overwrites) a plugin's source and loads it +// immediately. Writing to a currently-disabled plugin keeps it disabled. A +// returned error means the write failed; a successful write whose code fails to +// load still returns nil — inspect the load error via ListGoPlugins. +func (m *Manager) SaveGoPlugin(name, source string) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q (use letters, digits, '-' or '_')", name) + } + if err := os.MkdirAll(m.goDir(), 0o755); err != nil { + return err + } + // Preserve enabled/disabled state when overwriting an existing plugin. + target := m.enabledPath(name) + if _, err := os.Stat(m.disabledPath(name)); err == nil { + if _, err := os.Stat(target); err != nil { + target = m.disabledPath(name) + } + } + if err := os.WriteFile(target, []byte(source), 0o644); err != nil { + return err + } + // Reconcile immediately so the change takes effect without waiting for the + // next watcher tick. + _, _ = m.Sync() + return nil +} + +// DeleteGoPlugin removes a plugin (both variants) and unloads it. +func (m *Manager) DeleteGoPlugin(name string) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q", name) + } + removed := false + for _, p := range []string{m.enabledPath(name), m.disabledPath(name)} { + if err := os.Remove(p); err == nil { + removed = true + } else if !os.IsNotExist(err) { + return err + } + } + if !removed { + return fmt.Errorf("plugin %q not found", name) + } + _, _ = m.Sync() + return nil +} + +// SetGoPluginEnabled enables or disables a plugin by renaming its file between +// .go and .go.disabled, then reconciles. +func (m *Manager) SetGoPluginEnabled(name string, enabled bool) error { + if !ValidPluginName(name) { + return fmt.Errorf("invalid plugin name %q", name) + } + from, to := m.disabledPath(name), m.enabledPath(name) + if !enabled { + from, to = m.enabledPath(name), m.disabledPath(name) + } + if _, err := os.Stat(from); err != nil { + // Already in the desired state (or missing) — treat as a no-op success + // if the target exists, else report not found. + if _, err2 := os.Stat(to); err2 == nil { + return nil + } + return fmt.Errorf("plugin %q not found", name) + } + if err := os.Rename(from, to); err != nil { + return err + } + _, _ = m.Sync() + return nil +} diff --git a/internal/plugin/manage_test.go b/internal/plugin/manage_test.go new file mode 100644 index 0000000000..97f90ac71c --- /dev/null +++ b/internal/plugin/manage_test.go @@ -0,0 +1,115 @@ +package plugin + +import "testing" + +func findInfo(infos []PluginInfo, name string) (PluginInfo, bool) { + for _, i := range infos { + if i.Name == name { + return i, true + } + } + return PluginInfo{}, false +} + +func TestSaveListAndLoad(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatalf("SaveGoPlugin: %v", err) + } + info, ok := findInfo(m.ListGoPlugins(), "demo") + if !ok { + t.Fatal("expected demo in plugin list") + } + if !info.Enabled || !info.Loaded || info.Error != "" { + t.Fatalf("expected demo enabled+loaded+no-error, got %+v", info) + } + // Source round-trips. + src, err := m.GetGoPluginSource("demo") + if err != nil || src != demoPlugin { + t.Fatalf("GetGoPluginSource mismatch: err=%v", err) + } +} + +func TestSaveInvalidName(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("../evil", demoPlugin); err == nil { + t.Fatal("expected invalid name to be rejected") + } +} + +func TestSaveBrokenSourceReportsError(t *testing.T) { + m := newTestManager(t) + // Write succeeds, but the code fails to load → recorded as an error, not loaded. + if err := m.SaveGoPlugin("broken", "package main\nnot valid go"); err != nil { + t.Fatalf("SaveGoPlugin (write) should succeed: %v", err) + } + info, ok := findInfo(m.ListGoPlugins(), "broken") + if !ok { + t.Fatal("expected broken plugin listed") + } + if info.Loaded { + t.Fatal("broken plugin must not be loaded") + } + if info.Error == "" { + t.Fatal("expected a load error to be reported") + } +} + +func TestEnableDisable(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatal(err) + } + if err := m.SetGoPluginEnabled("demo", false); err != nil { + t.Fatalf("disable: %v", err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after disable") + } + info, _ := findInfo(m.ListGoPlugins(), "demo") + if info.Enabled { + t.Fatal("expected demo to be marked disabled") + } + if err := m.SetGoPluginEnabled("demo", true); err != nil { + t.Fatalf("enable: %v", err) + } + if !m.runtime.Loaded("demo") { + t.Fatal("expected demo loaded after re-enable") + } +} + +func TestDeleteUnloads(t *testing.T) { + m := newTestManager(t) + if err := m.SaveGoPlugin("demo", demoPlugin); err != nil { + t.Fatal(err) + } + if err := m.DeleteGoPlugin("demo"); err != nil { + t.Fatalf("delete: %v", err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after delete") + } + if _, ok := findInfo(m.ListGoPlugins(), "demo"); ok { + t.Fatal("expected demo gone from list") + } +} + +func TestGetSettingCapability(t *testing.T) { + // The API exposes injected capabilities; verify the nil-safe defaults and a + // wired getter. + api := &pluginAPI{name: "x", reg: NewRegistry()} + SettingGetter = nil + if api.GetSetting("k") != "" { + t.Fatal("expected empty string when no getter is wired") + } + SettingGetter = func(key string) string { + if key == "k" { + return "v" + } + return "" + } + defer func() { SettingGetter = nil }() + if api.GetSetting("k") != "v" { + t.Fatal("expected wired getter to return v") + } +} diff --git a/internal/plugin/manager.go b/internal/plugin/manager.go new file mode 100644 index 0000000000..2aa0303d54 --- /dev/null +++ b/internal/plugin/manager.go @@ -0,0 +1,225 @@ +package plugin + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// ManifestEntry describes a frontend (JS) plugin the backend serves to the web UI. +type ManifestEntry struct { + ID string `json:"id"` + URL string `json:"url"` +} + +// Manifest is the payload the frontend fetches to hot-load JS plugins. +type Manifest struct { + Plugins []ManifestEntry `json:"plugins"` +} + +type fileState struct { + mod time.Time + size int64 +} + +// Manager watches a plugins directory and keeps the yaegi Runtime in sync with +// the .go source files on disk, and builds the frontend manifest from the JS files. +// +// Layout under Root: +// +// /go/*.go backend Go-source plugins (yaegi) +// /frontend/*.js frontend JS plugins (served via AssetURLBase) +type Manager struct { + Root string + AssetURLBase string // e.g. "/api/plugin/asset" + runtime *Runtime + registry *Registry + mu sync.Mutex + known map[string]fileState // go-plugin path -> state + loadErrors map[string]string // plugin name -> last load error (for the UI) + stop chan struct{} + stopOnce sync.Once +} + +// Default is the process-wide manager, set during bootstrap. HTTP handlers read +// it; it may be nil if plugin support failed to initialise. +var Default *Manager + +func NewManager(ctx context.Context, root, assetURLBase string) (*Manager, error) { + _ = ctx // kept for signature/back-compat; the yaegi runtime needs no context + reg := NewRegistry() + return &Manager{ + Root: root, + AssetURLBase: strings.TrimRight(assetURLBase, "/"), + runtime: NewRuntime(reg), + registry: reg, + known: map[string]fileState{}, + loadErrors: map[string]string{}, + stop: make(chan struct{}), + }, nil +} + +func (m *Manager) Registry() *Registry { return m.registry } +func (m *Manager) Runtime() *Runtime { return m.runtime } + +func (m *Manager) goDir() string { return filepath.Join(m.Root, "go") } +func (m *Manager) frontendDir() string { return filepath.Join(m.Root, "frontend") } + +func nameFromPath(path, ext string) string { + return strings.TrimSuffix(filepath.Base(path), ext) +} + +// Sync reconciles loaded plugins with the .go files on disk: new or changed +// files are (re)loaded, deleted files are unloaded. Returns the names that were +// loaded/reloaded this pass. A missing directory is treated as empty. +func (m *Manager) Sync() ([]string, error) { + entries, err := os.ReadDir(m.goDir()) + if err != nil { + if os.IsNotExist(err) { + m.unloadMissing(map[string]bool{}) + return nil, nil + } + return nil, err + } + + present := map[string]bool{} + var changed []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") { + continue + } + path := filepath.Join(m.goDir(), e.Name()) + info, statErr := e.Info() + if statErr != nil { + continue + } + present[path] = true + + m.mu.Lock() + prev, ok := m.known[path] + unchanged := ok && prev.mod.Equal(info.ModTime()) && prev.size == info.Size() + m.mu.Unlock() + if unchanged { + continue + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + log.Warnf("[plugin] read %s: %v", path, readErr) + continue + } + name := nameFromPath(path, ".go") + // Runtime.Load drops the old version's hooks before re-subscribing, so it + // is safe to call repeatedly for hot-reload. + if err := m.runtime.Load(name, data); err != nil { + log.Warnf("[plugin] load %s: %v", name, err) + // Surface the error to the management UI, but still remember the + // file state so a syntactically-broken plugin isn't retried every + // tick — it retries when the file is edited (mtime/size changes). + m.mu.Lock() + m.loadErrors[name] = err.Error() + m.known[path] = fileState{mod: info.ModTime(), size: info.Size()} + m.mu.Unlock() + continue + } + m.mu.Lock() + delete(m.loadErrors, name) + m.known[path] = fileState{mod: info.ModTime(), size: info.Size()} + m.mu.Unlock() + changed = append(changed, name) + log.Infof("[plugin] loaded go plugin %q", name) + } + + m.unloadMissing(present) + return changed, nil +} + +func (m *Manager) unloadMissing(present map[string]bool) { + m.mu.Lock() + var gone []string + for path := range m.known { + if !present[path] { + gone = append(gone, path) + } + } + for _, path := range gone { + delete(m.known, path) + } + m.mu.Unlock() + + for _, path := range gone { + name := nameFromPath(path, ".go") + m.runtime.Unload(name) // also drops the plugin's hook subscriptions + m.mu.Lock() + delete(m.loadErrors, name) + m.mu.Unlock() + log.Infof("[plugin] unloaded go plugin %q", name) + } +} + +// FrontendManifest scans the frontend dir for .js plugins and returns the +// manifest the web UI fetches. A missing dir yields an empty manifest. +func (m *Manager) FrontendManifest() Manifest { + manifest := Manifest{Plugins: []ManifestEntry{}} + entries, err := os.ReadDir(m.frontendDir()) + if err != nil { + return manifest + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".js") { + continue + } + id := nameFromPath(e.Name(), ".js") + manifest.Plugins = append(manifest.Plugins, ManifestEntry{ + ID: id, + URL: m.AssetURLBase + "/" + e.Name(), + }) + } + return manifest +} + +// AssetPath resolves a requested asset filename to a path inside the frontend +// dir, rejecting any traversal outside it. Returns ("", false) when unsafe. +func (m *Manager) AssetPath(name string) (string, bool) { + if name == "" || strings.Contains(name, "..") || filepath.IsAbs(name) { + return "", false + } + full := filepath.Join(m.frontendDir(), filepath.Clean(name)) + rel, err := filepath.Rel(m.frontendDir(), full) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + return full, true +} + +// Start runs Sync immediately and then every interval until Close, for hot-reload. +func (m *Manager) Start(interval time.Duration) { + if _, err := m.Sync(); err != nil { + log.Warnf("[plugin] initial sync: %v", err) + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-m.stop: + return + case <-ticker.C: + if _, err := m.Sync(); err != nil { + log.Warnf("[plugin] sync: %v", err) + } + } + } + }() +} + +// Close stops the watcher and tears down the runtime. +func (m *Manager) Close() error { + m.stopOnce.Do(func() { close(m.stop) }) + return m.runtime.Close() +} diff --git a/internal/plugin/manager_test.go b/internal/plugin/manager_test.go new file mode 100644 index 0000000000..773a460732 --- /dev/null +++ b/internal/plugin/manager_test.go @@ -0,0 +1,157 @@ +package plugin + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +func writeGo(t *testing.T, dir, name, source string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(source), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func newTestManager(t *testing.T) *Manager { + t.Helper() + m, err := NewManager(context.Background(), t.TempDir(), "/api/plugin/asset") + if err != nil { + t.Fatalf("NewManager: %v", err) + } + t.Cleanup(func() { _ = m.Close() }) + return m +} + +func TestManagerSyncLoadsGo(t *testing.T) { + m := newTestManager(t) + writeGo(t, m.goDir(), "demo.go", demoPlugin) + + changed, err := m.Sync() + if err != nil { + t.Fatalf("Sync: %v", err) + } + if len(changed) != 1 || changed[0] != "demo" { + t.Fatalf("expected [demo] loaded, got %v", changed) + } + if !m.runtime.Loaded("demo") { + t.Fatal("expected demo loaded in runtime") + } + // The loaded plugin's hook must actually fire through the manager's registry. + ctx, errs := m.registry.Fire("demo", map[string]any{}) + if len(errs) != 0 || ctx.Payload["touched"] != "v1" { + t.Fatalf("expected touched=v1, got %v errs=%v", ctx.Payload["touched"], errs) + } +} + +func TestManagerSyncMissingDirIsEmpty(t *testing.T) { + m := newTestManager(t) + changed, err := m.Sync() + if err != nil { + t.Fatalf("expected nil error for missing dir, got %v", err) + } + if len(changed) != 0 { + t.Fatalf("expected no changes, got %v", changed) + } +} + +func TestManagerSyncUnchangedNotReloaded(t *testing.T) { + m := newTestManager(t) + writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + changed, err := m.Sync() + if err != nil { + t.Fatal(err) + } + if len(changed) != 0 { + t.Fatalf("expected no reload on unchanged file, got %v", changed) + } +} + +func TestManagerSyncUnloadsDeleted(t *testing.T) { + m := newTestManager(t) + path := writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + if m.runtime.Loaded("demo") { + t.Fatal("expected demo unloaded after file removed") + } + if got := len(m.registry.Handlers("demo")); got != 0 { + t.Fatalf("expected hooks dropped after unload, got %d", got) + } +} + +func TestManagerSyncReloadsChanged(t *testing.T) { + m := newTestManager(t) + path := writeGo(t, m.goDir(), "demo.go", demoPlugin) + if _, err := m.Sync(); err != nil { + t.Fatal(err) + } + // Bump mtime forward so the change is detected deterministically. + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(path, future, future); err != nil { + t.Fatal(err) + } + changed, err := m.Sync() + if err != nil { + t.Fatal(err) + } + if len(changed) != 1 || changed[0] != "demo" { + t.Fatalf("expected demo reloaded, got %v", changed) + } +} + +func TestManagerFrontendManifest(t *testing.T) { + m := newTestManager(t) + dir := m.frontendDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "hello.js"), []byte("export default {}"), 0o644); err != nil { + t.Fatal(err) + } + mf := m.FrontendManifest() + if len(mf.Plugins) != 1 { + t.Fatalf("expected 1 manifest entry, got %d", len(mf.Plugins)) + } + if mf.Plugins[0].ID != "hello" { + t.Fatalf("expected id hello, got %s", mf.Plugins[0].ID) + } + if mf.Plugins[0].URL != "/api/plugin/asset/hello.js" { + t.Fatalf("unexpected url %s", mf.Plugins[0].URL) + } +} + +func TestManagerFrontendManifestEmptyWhenNoDir(t *testing.T) { + m := newTestManager(t) + mf := m.FrontendManifest() + if mf.Plugins == nil || len(mf.Plugins) != 0 { + t.Fatalf("expected empty (non-nil) manifest, got %v", mf.Plugins) + } +} + +func TestManagerAssetPathRejectsTraversal(t *testing.T) { + m := newTestManager(t) + if _, ok := m.AssetPath("../../etc/passwd"); ok { + t.Fatal("expected traversal to be rejected") + } + if _, ok := m.AssetPath("ok.js"); !ok { + t.Fatal("expected normal filename to be accepted") + } +} diff --git a/internal/plugin/registry.go b/internal/plugin/registry.go new file mode 100644 index 0000000000..04d58c88aa --- /dev/null +++ b/internal/plugin/registry.go @@ -0,0 +1,120 @@ +// Package plugin provides a cross-platform, hot-reloadable plugin system. +// +// The architecture has two halves: +// - A hook Registry (this file): host code fires named hooks at well-known +// points; plugins subscribe to the hooks they care about. The set of hook +// points is host-provided, but which hooks a plugin subscribes to — and the +// plugin's logic — is fully dynamic and reloadable. +// - A wazero-based WASM Runtime (wasm.go): runs plugin logic in-process, +// sandboxed, on every platform, and can be reloaded without restarting. +package plugin + +import ( + "sort" + "sync" +) + +// HookContext carries the data for a fired hook. Handlers may read and mutate +// Payload; mutations are visible to later handlers and to the host. +type HookContext struct { + Hook string + Payload map[string]any +} + +// Handler reacts to a fired hook. Returning an error is logged but does not stop +// other handlers from running. +type Handler func(ctx *HookContext) error + +type subscription struct { + pluginID string + hook string + order int + handler Handler +} + +// Registry is a concurrency-safe hook bus. The zero value is not usable; call +// NewRegistry. +type Registry struct { + mu sync.RWMutex + subs []subscription +} + +func NewRegistry() *Registry { + return &Registry{} +} + +// Subscribe registers handler for hook on behalf of pluginID. Lower order runs +// first. A plugin may subscribe to the same hook multiple times. +func (r *Registry) Subscribe(pluginID, hook string, order int, handler Handler) { + r.mu.Lock() + defer r.mu.Unlock() + r.subs = append(r.subs, subscription{pluginID, hook, order, handler}) +} + +// UnsubscribeAll removes every subscription owned by pluginID. This is what makes +// hot-reload safe: drop a plugin's hooks before loading its new version. +func (r *Registry) UnsubscribeAll(pluginID string) { + r.mu.Lock() + defer r.mu.Unlock() + kept := r.subs[:0] + for _, s := range r.subs { + if s.pluginID != pluginID { + kept = append(kept, s) + } + } + // Avoid retaining handlers of removed plugins in the backing array. + for i := len(kept); i < len(r.subs); i++ { + r.subs[i] = subscription{} + } + r.subs = kept +} + +// HasHandlers reports whether at least one handler is subscribed to a hook. It +// is cheap and used on hot paths to avoid building a payload when nothing listens. +func (r *Registry) HasHandlers(hook string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + for _, s := range r.subs { + if s.hook == hook { + return true + } + } + return false +} + +// Handlers returns the ordered handlers subscribed to a hook (mainly for tests). +func (r *Registry) Handlers(hook string) []Handler { + r.mu.RLock() + defer r.mu.RUnlock() + var matched []subscription + for _, s := range r.subs { + if s.hook == hook { + matched = append(matched, s) + } + } + sort.SliceStable(matched, func(i, j int) bool { + return matched[i].order < matched[j].order + }) + out := make([]Handler, len(matched)) + for i, s := range matched { + out[i] = s.handler + } + return out +} + +// Fire invokes every handler subscribed to hook, in order, passing a shared +// context. It returns the (possibly mutated) context and a slice of any errors +// handlers returned. One failing handler never blocks the others. +func (r *Registry) Fire(hook string, payload map[string]any) (*HookContext, []error) { + if payload == nil { + payload = map[string]any{} + } + ctx := &HookContext{Hook: hook, Payload: payload} + var errs []error + for _, h := range r.Handlers(hook) { + if err := h(ctx); err != nil { + errs = append(errs, err) + } + } + return ctx, errs +} diff --git a/internal/plugin/registry_test.go b/internal/plugin/registry_test.go new file mode 100644 index 0000000000..a3452ab1e4 --- /dev/null +++ b/internal/plugin/registry_test.go @@ -0,0 +1,101 @@ +package plugin + +import ( + "errors" + "testing" +) + +func TestRegistrySubscribeAndFire(t *testing.T) { + r := NewRegistry() + var calls []string + r.Subscribe("p1", "fs.list.after", 0, func(ctx *HookContext) error { + calls = append(calls, "p1") + return nil + }) + r.Subscribe("p2", "fs.list.after", 0, func(ctx *HookContext) error { + calls = append(calls, "p2") + return nil + }) + r.Subscribe("p3", "other.hook", 0, func(ctx *HookContext) error { + calls = append(calls, "p3") + return nil + }) + + _, errs := r.Fire("fs.list.after", nil) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(calls) != 2 { + t.Fatalf("expected 2 handlers fired, got %v", calls) + } +} + +func TestRegistryOrder(t *testing.T) { + r := NewRegistry() + var calls []string + r.Subscribe("late", "h", 10, func(*HookContext) error { + calls = append(calls, "late") + return nil + }) + r.Subscribe("early", "h", -5, func(*HookContext) error { + calls = append(calls, "early") + return nil + }) + r.Fire("h", nil) + if len(calls) != 2 || calls[0] != "early" || calls[1] != "late" { + t.Fatalf("expected [early late], got %v", calls) + } +} + +func TestRegistryPayloadMutationVisibleToLaterHandlers(t *testing.T) { + r := NewRegistry() + r.Subscribe("a", "h", 0, func(ctx *HookContext) error { + ctx.Payload["n"] = 1 + return nil + }) + r.Subscribe("b", "h", 1, func(ctx *HookContext) error { + if ctx.Payload["n"] != 1 { + t.Errorf("expected payload from earlier handler, got %v", ctx.Payload["n"]) + } + ctx.Payload["n"] = 2 + return nil + }) + ctx, _ := r.Fire("h", nil) + if ctx.Payload["n"] != 2 { + t.Fatalf("expected final payload 2, got %v", ctx.Payload["n"]) + } +} + +func TestRegistryErrorsDoNotStopOthers(t *testing.T) { + r := NewRegistry() + ran := false + r.Subscribe("bad", "h", 0, func(*HookContext) error { + return errors.New("boom") + }) + r.Subscribe("good", "h", 1, func(*HookContext) error { + ran = true + return nil + }) + _, errs := r.Fire("h", nil) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d", len(errs)) + } + if !ran { + t.Fatal("later handler should still run after an earlier error") + } +} + +func TestRegistryUnsubscribeAll(t *testing.T) { + r := NewRegistry() + r.Subscribe("p1", "h", 0, func(*HookContext) error { return nil }) + r.Subscribe("p1", "h2", 0, func(*HookContext) error { return nil }) + r.Subscribe("p2", "h", 0, func(*HookContext) error { return nil }) + + r.UnsubscribeAll("p1") + if got := len(r.Handlers("h")); got != 1 { + t.Fatalf("expected 1 handler on h after unsubscribe, got %d", got) + } + if got := len(r.Handlers("h2")); got != 0 { + t.Fatalf("expected 0 handlers on h2 after unsubscribe, got %d", got) + } +} diff --git a/server/handles/auth.go b/server/handles/auth.go index 7800690918..dcb8a05ee6 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -8,6 +8,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -79,6 +80,7 @@ func loginHash(c *gin.Context, req *LoginReq) { } common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) + plugin.FireHook(plugin.HookUserLoginAfter, map[string]any{"username": user.Username}) } type UserResp struct { diff --git a/server/handles/favorite.go b/server/handles/favorite.go new file mode 100644 index 0000000000..68f6a6f765 --- /dev/null +++ b/server/handles/favorite.go @@ -0,0 +1,117 @@ +package handles + +import ( + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// normalizeFavoritePath cleans a favorite path so that dedup/toggle compares +// equal paths consistently (backslashes, missing leading slash, ".."/"." are +// all normalized away). +func normalizeFavoritePath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "/" + } + return utils.FixAndCleanPath(path) +} + +type FavoriteAddReq struct { + Path string `json:"path" binding:"required"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Tag string `json:"tag"` +} + +func currentUser(c *gin.Context) (*model.User, bool) { + userObj, ok := c.Request.Context().Value(conf.UserKey).(*model.User) + if !ok || userObj == nil { + return nil, false + } + return userObj, true +} + +func ListFavorites(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + favorites, err := db.GetFavoritesByUser(userObj.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, favorites) +} + +func AddFavorite(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var req FavoriteAddReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorStrResp(c, "request invalid", 400) + return + } + path := normalizeFavoritePath(req.Path) + // dedup/toggle: if a favorite with the same (userID, path) exists, just + // update its tag instead of creating a duplicate. + existing, err := db.GetFavoriteByUserAndPath(userObj.ID, path) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if existing != nil { + existing.Tag = req.Tag + existing.Name = req.Name + existing.IsDir = req.IsDir + if err := db.UpdateFavorite(existing); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, existing) + return + } + favorite := &model.Favorite{ + UserID: userObj.ID, + Path: path, + Name: req.Name, + IsDir: req.IsDir, + Tag: req.Tag, + CreatedAt: time.Now().Unix(), + } + if err := db.CreateFavorite(favorite); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, favorite) +} + +func DeleteFavorite(c *gin.Context) { + userObj, ok := currentUser(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + id, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + if err := db.DeleteFavorite(userObj.ID, uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/favorite_test.go b/server/handles/favorite_test.go new file mode 100644 index 0000000000..f85a21e413 --- /dev/null +++ b/server/handles/favorite_test.go @@ -0,0 +1,38 @@ +package handles + +import "testing" + +func TestNormalizeFavoritePath(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty", "", "/"}, + {"whitespace only", " ", "/"}, + {"already clean", "/a/b", "/a/b"}, + {"missing leading slash", "a/b", "/a/b"}, + {"backslashes", "\\a\\b", "/a/b"}, + {"trailing slash", "/a/b/", "/a/b"}, + {"dot segments", "/a/../b", "/b"}, + {"root", "/", "/"}, + {"trim spaces", " /a/b ", "/a/b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeFavoritePath(tc.in); got != tc.want { + t.Fatalf("normalizeFavoritePath(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// TestNormalizeFavoritePathDedup verifies that two inputs that should refer to +// the same favorite normalize to the same key (so toggle/dedup matches them). +func TestNormalizeFavoritePathDedup(t *testing.T) { + a := normalizeFavoritePath("a/b/") + b := normalizeFavoritePath("\\a\\b") + if a != b { + t.Fatalf("expected equal normalized paths, got %q and %q", a, b) + } +} diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index d97d36d24a..b3c3581780 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -10,6 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" @@ -528,4 +529,5 @@ func Link(c *gin.Context) { } defer link.Close() common.SuccessResp(c, link) + plugin.FireHook(plugin.HookFsLinkAfter, map[string]any{"path": rawPath}) } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 8fd731af7b..531131e1e5 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -7,10 +7,12 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/plugin" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -55,6 +57,11 @@ type FsListResp struct { WriteContentBypass bool `json:"write_content_bypass"` Provider string `json:"provider"` DirectUploadTools []string `json:"direct_upload_tools,omitempty"` + // MountDetails is the disk usage of the storage that the *current directory* + // belongs to (nil at the virtual storages-root or when hidden). It lets the + // web header show the current mount's usage dynamically on folder navigation + // without firing an extra fs/get per click. + MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } func FsListSplit(c *gin.Context) { @@ -109,10 +116,19 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string - if canWriteContentAtPath { - if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + var mountDetails *model.StorageDetails + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + if canWriteContentAtPath { directUploadTools = op.GetDirectUploadTools(storage) } + // Current directory's storage usage for the web header. Cache-backed + + // singleflight in op.GetStorageDetails, so concurrent multi-user access + // collapses to at most one provider call per storage per TTL — no 429. + if !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails) { + if d, e := op.GetStorageDetails(c.Request.Context(), storage); e == nil { + mountDetails = d + } + } } common.SuccessResp(c, FsListResp{ Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), @@ -123,7 +139,14 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { WriteContentBypass: common.CanWriteContentBypassUserPerms(meta, reqPath), Provider: provider, DirectUploadTools: directUploadTools, + MountDetails: mountDetails, }) + if plugin.HasSubscribers(plugin.HookFsListAfter) { + plugin.FireHook(plugin.HookFsListAfter, map[string]any{ + "path": reqPath, + "count": total, + }) + } } func FsDirs(c *gin.Context) { @@ -382,6 +405,63 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { }) } +type FsVideoPlayReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` +} + +// FsVideoPlay exposes the storage provider's official online-play (transcoded +// streaming) sources for a video at multiple resolutions, for drivers that +// implement driver.VideoPlayer (e.g. 115_open). +func FsVideoPlay(c *gin.Context) { + var req FsVideoPlayReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if user.IsGuest() && user.Disabled { + common.ErrorStrResp(c, "Guest user is disabled, login please", 401) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + common.GinAppendValues(c, conf.MetaKey, meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + vp, ok := storage.(driver.VideoPlayer) + if !ok { + common.ErrorStrResp(c, "driver does not support official play sources", 400) + return + } + obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + sources, err := vp.VideoPlay(c.Request.Context(), obj) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, sources) +} + func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { var related []model.Obj nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) diff --git a/server/handles/plugin.go b/server/handles/plugin.go new file mode 100644 index 0000000000..ea3d2d4497 --- /dev/null +++ b/server/handles/plugin.go @@ -0,0 +1,148 @@ +package handles + +import ( + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/plugin" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// PluginManifest serves the list of frontend (JS) plugins for the web UI to +// hot-load. Always returns a (possibly empty) manifest. +func PluginManifest(c *gin.Context) { + if plugin.Default == nil { + common.SuccessResp(c, plugin.Manifest{Plugins: []plugin.ManifestEntry{}}) + return + } + common.SuccessResp(c, plugin.Default.FrontendManifest()) +} + +// PluginAsset serves a frontend plugin's JS file by name, guarding against path +// traversal. +func PluginAsset(c *gin.Context) { + if plugin.Default == nil { + c.Status(http.StatusNotFound) + return + } + path, ok := plugin.Default.AssetPath(c.Param("name")) + if !ok { + c.Status(http.StatusBadRequest) + return + } + c.File(path) +} + +// ---- Admin management of backend (Go-source) plugins ---- + +// PluginListResp is the payload for the management UI: backend Go plugins plus +// the frontend JS plugins served to the web UI. +type PluginListResp struct { + Go []plugin.PluginInfo `json:"go"` + Frontend []plugin.ManifestEntry `json:"frontend"` +} + +func pluginMgr(c *gin.Context) *plugin.Manager { + if plugin.Default == nil { + common.ErrorStrResp(c, "plugin support is disabled", 500) + return nil + } + return plugin.Default +} + +// PluginList returns all backend Go plugins (with load status) and frontend JS +// plugins. Admin only. +func PluginList(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + common.SuccessResp(c, PluginListResp{ + Go: mgr.ListGoPlugins(), + Frontend: mgr.FrontendManifest().Plugins, + }) +} + +// PluginGet returns a backend plugin's source for editing. Admin only. +func PluginGet(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + src, err := mgr.GetGoPluginSource(c.Query("name")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + common.SuccessResp(c, gin.H{"name": c.Query("name"), "source": src}) +} + +type PluginSaveReq struct { + Name string `json:"name" binding:"required"` + Source string `json:"source" binding:"required"` +} + +// PluginSave creates or overwrites a backend plugin and loads it immediately. +// A successful write whose code fails to load still returns 200 — the load +// error is reported by PluginList. Admin only. +func PluginSave(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginSaveReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.SaveGoPlugin(req.Name, req.Source); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} + +type PluginNameReq struct { + Name string `json:"name" binding:"required"` +} + +// PluginDelete removes a backend plugin and unloads it. Admin only. +func PluginDelete(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginNameReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.DeleteGoPlugin(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} + +type PluginEnableReq struct { + Name string `json:"name" binding:"required"` + Enabled bool `json:"enabled"` +} + +// PluginSetEnabled enables/disables a backend plugin. Admin only. +func PluginSetEnabled(c *gin.Context) { + mgr := pluginMgr(c) + if mgr == nil { + return + } + var req PluginEnableReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := mgr.SetGoPluginEnabled(req.Name, req.Enabled); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, mgr.ListGoPlugins()) +} diff --git a/server/handles/storage.go b/server/handles/storage.go index 687c93551b..0342b54a0b 100644 --- a/server/handles/storage.go +++ b/server/handles/storage.go @@ -90,6 +90,12 @@ func ListStorages(c *gin.Context) { }) } +// StorageLoadingStatus reports the per-storage startup load progress so the +// frontend can render an async loading status bar. +func StorageLoadingStatus(c *gin.Context) { + common.SuccessResp(c, op.StorageLoadProgress.Snapshot()) +} + func CreateStorage(c *gin.Context) { var req model.Storage if err := c.ShouldBind(&req); err != nil { diff --git a/server/handles/user.go b/server/handles/user.go index 095089d707..4ebbde1686 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,8 +1,10 @@ package handles import ( + "errors" "strconv" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" @@ -10,6 +12,29 @@ import ( log "github.com/sirupsen/logrus" ) +// applyUserUpdatePolicy enforces who may change what during a user update. +// Role can never change here. A non-admin editor (a user delegated the +// "manage user info" permission) may only edit a normal user's username and +// password; every privileged field (permission, role, base path, disabled, +// otp) is forced back to the existing value, and admins can't be edited by them. +func applyUserUpdatePolicy(existing, req *model.User, byAdmin bool) error { + if existing.Role != req.Role { + return errors.New("role can not be changed") + } + if byAdmin { + return nil + } + if existing.IsAdmin() { + return errors.New("you are not allowed to edit an admin user") + } + // Profile-only edit: keep all privileged fields, allow username/password. + username, password := req.Username, req.Password + *req = *existing + req.Username = username + req.Password = password + return nil +} + func ListUsers(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { @@ -60,8 +85,10 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) + editor, _ := c.Request.Context().Value(conf.UserKey).(*model.User) + byAdmin := editor != nil && editor.IsAdmin() + if err := applyUserUpdatePolicy(user, &req, byAdmin); err != nil { + common.ErrorStrResp(c, err.Error(), 403) return } if req.Password == "" { diff --git a/server/handles/user_test.go b/server/handles/user_test.go new file mode 100644 index 0000000000..0cce111e3d --- /dev/null +++ b/server/handles/user_test.go @@ -0,0 +1,80 @@ +package handles + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestApplyUserUpdatePolicyRejectsRoleChange(t *testing.T) { + existing := &model.User{ID: 2, Role: model.GENERAL, Username: "bob"} + req := &model.User{ID: 2, Role: model.ADMIN, Username: "bob"} + if err := applyUserUpdatePolicy(existing, req, true); err == nil { + t.Fatal("expected role change to be rejected even for admin") + } +} + +func TestApplyUserUpdatePolicyAdminCanEditPrivilegedFields(t *testing.T) { + existing := &model.User{ID: 2, Role: model.GENERAL, Username: "bob", Permission: 0, BasePath: "/old"} + req := &model.User{ID: 2, Role: model.GENERAL, Username: "bob2", Permission: 0xFF, BasePath: "/new", Disabled: true} + if err := applyUserUpdatePolicy(existing, req, true); err != nil { + t.Fatalf("admin edit should succeed: %v", err) + } + if req.Permission != 0xFF || req.BasePath != "/new" || !req.Disabled { + t.Fatalf("admin's privileged changes must be preserved, got %+v", req) + } +} + +func TestApplyUserUpdatePolicyDelegatedCannotEditAdmin(t *testing.T) { + existing := &model.User{ID: 1, Role: model.ADMIN, Username: "root"} + req := &model.User{ID: 1, Role: model.ADMIN, Username: "root2"} + if err := applyUserUpdatePolicy(existing, req, false); err == nil { + t.Fatal("a delegated (non-admin) editor must not edit an admin user") + } +} + +func TestApplyUserUpdatePolicyDelegatedProfileOnly(t *testing.T) { + existing := &model.User{ + ID: 2, + Role: model.GENERAL, + Username: "bob", + Permission: 0b101, + BasePath: "/restricted", + Disabled: false, + OtpSecret: "secret", + } + // A delegated editor tries to escalate permission, change base path, disable. + req := &model.User{ + ID: 2, + Role: model.GENERAL, + Username: "bobby", + Password: "newpass", + Permission: 0xFFFF, + BasePath: "/", + Disabled: true, + OtpSecret: "", + } + if err := applyUserUpdatePolicy(existing, req, false); err != nil { + t.Fatalf("delegated profile edit should succeed: %v", err) + } + // Username + password allowed through. + if req.Username != "bobby" { + t.Errorf("username should be editable, got %q", req.Username) + } + if req.Password != "newpass" { + t.Errorf("password should be editable, got %q", req.Password) + } + // Every privileged field must be forced back to the existing value. + if req.Permission != 0b101 { + t.Errorf("permission must not change, got %d", req.Permission) + } + if req.BasePath != "/restricted" { + t.Errorf("base path must not change, got %q", req.BasePath) + } + if req.Disabled { + t.Errorf("disabled must not change") + } + if req.OtpSecret != "secret" { + t.Errorf("otp secret must be preserved, got %q", req.OtpSecret) + } +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index ca67e4a6dd..965994d5f8 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -148,3 +148,15 @@ func AuthAdmin(c *gin.Context) { c.Next() } } + +// AuthUserInfoManage allows admins, or non-admins to whom an admin has delegated +// the "manage user info" permission, to access user-profile management endpoints. +func AuthUserInfoManage(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if !user.CanManageUserInfo() { + common.ErrorStrResp(c, "You are not allowed to manage users", 403) + c.Abort() + } else { + c.Next() + } +} diff --git a/server/router.go b/server/router.go index 3d44a9e3da..2aacbbf3c2 100644 --- a/server/router.go +++ b/server/router.go @@ -78,6 +78,9 @@ func Init(e *gin.Engine) { auth.GET("/me/sshkey/list", handles.ListMyPublicKey) auth.POST("/me/sshkey/add", handles.AddMyPublicKey) auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) + auth.GET("/me/favorites", handles.ListFavorites) + auth.POST("/me/favorites/add", handles.AddFavorite) + auth.POST("/me/favorites/delete", handles.DeleteFavorite) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) @@ -102,11 +105,18 @@ func Init(e *gin.Engine) { public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + // Plugin system: frontend manifest + hot-loadable JS assets (public so the + // web UI can fetch them before auth). + api.GET("/plugin/manifest", handles.PluginManifest) + api.GET("/plugin/asset/:name", handles.PluginAsset) + _fs(auth.Group("/fs")) fsAndShare(api.Group("/fs", middlewares.Auth(true))) _task(auth.Group("/task", middlewares.AuthNotGuest)) _sharing(auth.Group("/share", middlewares.AuthNotGuest)) admin(auth.Group("/admin", middlewares.AuthAdmin)) + // User-profile management: admins, or non-admins delegated the permission. + userManage(auth.Group("/admin/user", middlewares.AuthUserInfoManage)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) } @@ -115,6 +125,15 @@ func Init(e *gin.Engine) { }) } +// userManage registers the user-profile endpoints that an admin may delegate to +// a non-admin via the "manage user info" permission. The UpdateUser handler +// itself restricts a non-admin editor to username/password only. +func userManage(g *gin.RouterGroup) { + g.GET("/list", handles.ListUsers) + g.GET("/get", handles.GetUser) + g.POST("/update", handles.UpdateUser) +} + func admin(g *gin.RouterGroup) { meta := g.Group("/meta") meta.GET("/list", handles.ListMetas) @@ -123,11 +142,19 @@ func admin(g *gin.RouterGroup) { meta.POST("/update", handles.UpdateMeta) meta.POST("/delete", handles.DeleteMeta) + // Backend (Go-source, yaegi) plugin management. Admin only. + plug := g.Group("/plugin") + plug.GET("/list", handles.PluginList) + plug.GET("/get", handles.PluginGet) + plug.POST("/save", handles.PluginSave) + plug.POST("/delete", handles.PluginDelete) + plug.POST("/enable", handles.PluginSetEnabled) + + // Admin-only user operations (create/delete/role-affecting). The profile-edit + // endpoints (list/get/update) live in a separate group that also accepts a + // non-admin who has been delegated the "manage user info" permission. user := g.Group("/user") - user.GET("/list", handles.ListUsers) - user.GET("/get", handles.GetUser) user.POST("/create", handles.CreateUser) - user.POST("/update", handles.UpdateUser) user.POST("/cancel_2fa", handles.Cancel2FAById) user.POST("/delete", handles.DeleteUser) user.POST("/del_cache", handles.DelUserCache) @@ -136,6 +163,7 @@ func admin(g *gin.RouterGroup) { storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) + storage.GET("/loading", handles.StorageLoadingStatus) storage.GET("/get", handles.GetStorage) storage.POST("/create", handles.CreateStorage) storage.POST("/update", handles.UpdateStorage) @@ -191,6 +219,7 @@ func admin(g *gin.RouterGroup) { func fsAndShare(g *gin.RouterGroup) { g.Any("/list", handles.FsListSplit) g.Any("/get", handles.FsGetSplit) + g.POST("/video_play", handles.FsVideoPlay) a := g.Group("/archive") a.Any("/meta", handles.FsArchiveMetaSplit) a.Any("/list", handles.FsArchiveListSplit) From 8918f8ae1016809e0597d34f5a6c64d125831417 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 02:47:13 +0800 Subject: [PATCH 082/107] chore(deps): bump Ironboxplus/115-sdk-go to v0.2.10 Brings the VideoPlayResp fix: 115's /open/video/play returns file_size/duration/width/height as quoted strings; FlexInt64 now tolerates both string and number, fixing "cannot unmarshal string into Go struct field VideoPlayResp.file_size of type int64". --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 28aac1961d..ad91080433 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.9 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.10 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 907b590f21..f9c77dcf50 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/Ironboxplus/115-sdk-go v0.2.9 h1:Y+LdB5/mytE27Tj3Hm2FOH2nu+CKSALQioA7QOtM5sw= -github.com/Ironboxplus/115-sdk-go v0.2.9/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.10 h1:00cZGt952O3R5gICxYyMwb83YgcvMEP2VrpiecbaPlA= +github.com/Ironboxplus/115-sdk-go v0.2.10/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From e4b88f8f37de57a5c3ebc0c3e648c0ff522576c2 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 02:47:29 +0800 Subject: [PATCH 083/107] feat(cluster): storage parameter sharing plugin (backend) Cryptographically-authenticated, real-time storage-config sync between nodes using a distributed CRDT. Disabled by default; fully inert without config. - crypto: PSK->HKDF->AES-256-GCM AEAD envelope (membership auth + confidentiality for the secrets in storage configs), ed25519 per-node identity, node id == hash(pubkey) so records are self-authenticating and origin can't be forged. - state: per-mount LWW-CRDT keyed by content hash (idempotent: identical config => no churn, 'peers seeing the same token don't change'), Lamport versioning, signed records, tombstones. - config: file-backed (enable/key/peers/share filters/intervals/apply_remote/ share_deletes), admin API + status. - transport: sealed envelope, timestamp+nonce replay protection. - engine: push on healthy token refresh (health-gated, never propagates broken tokens), pull on token-invalid, periodic anti-entropy announce backstop; applies peer configs via op create/update/delete (loop-safe via content hash). - op: fire storage hook on saveDriverStorage (token-refresh path) + add NotifyStorageTokenInvalid signal. - wiring: bootstrap Init/Start + /api/cluster/sync peer endpoint and /admin/cluster config/status routes. --- internal/bootstrap/cluster.go | 29 ++ internal/bootstrap/run.go | 2 + internal/cluster/config.go | 161 ++++++++++ internal/cluster/crypto.go | 128 ++++++++ internal/cluster/engine.go | 543 ++++++++++++++++++++++++++++++++++ internal/cluster/identity.go | 30 ++ internal/cluster/state.go | 346 ++++++++++++++++++++++ internal/cluster/transport.go | 142 +++++++++ internal/op/hook.go | 10 + internal/op/storage.go | 5 + server/handles/cluster.go | 90 ++++++ server/router.go | 10 + 12 files changed, 1496 insertions(+) create mode 100644 internal/bootstrap/cluster.go create mode 100644 internal/cluster/config.go create mode 100644 internal/cluster/crypto.go create mode 100644 internal/cluster/engine.go create mode 100644 internal/cluster/identity.go create mode 100644 internal/cluster/state.go create mode 100644 internal/cluster/transport.go create mode 100644 server/handles/cluster.go diff --git a/internal/bootstrap/cluster.go b/internal/bootstrap/cluster.go new file mode 100644 index 0000000000..1d4fb4e106 --- /dev/null +++ b/internal/bootstrap/cluster.go @@ -0,0 +1,29 @@ +package bootstrap + +import ( + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/cluster" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// InitClusterSync loads the storage-sharing cluster engine: identity, config and +// persisted CRDT state, and registers the storage hook so local credential +// changes propagate to peers. The network loops are started later (after storages +// load) by StartClusterSync. Failure is non-fatal — the server runs fine without +// cluster sync. +func InitClusterSync() { + if _, err := cluster.Init(flags.DataDir); err != nil { + utils.Log.Warnf("[cluster] init failed, storage sync disabled: %v", err) + return + } + utils.Log.Infof("[cluster] node %s ready", cluster.Default.NodeID()) +} + +// StartClusterSync seeds records from the now-loaded local storages and starts +// the anti-entropy loop. No-op when cluster sync was not initialized. +func StartClusterSync() { + if cluster.Default == nil { + return + } + cluster.Default.Start() +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 853cf997d7..83d282e938 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -40,6 +40,7 @@ func Init() { InitIndex() InitUpgradePatch() InitPlugins() + InitClusterSync() } func Release() { @@ -97,6 +98,7 @@ func Start() { } InitOfflineDownloadTools() LoadStorages() + StartClusterSync() InitTaskManager() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) diff --git a/internal/cluster/config.go b/internal/cluster/config.go new file mode 100644 index 0000000000..a5e121e96c --- /dev/null +++ b/internal/cluster/config.go @@ -0,0 +1,161 @@ +package cluster + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" +) + +// Config is the operator-facing configuration of the storage-sharing cluster. +// It is persisted as JSON under /cluster/config.json and edited through the +// admin API (and the plugins page UI). Keeping it in its own file rather than the +// global settings table keeps the feature self-contained and pluggable. +type Config struct { + // Enabled turns the whole cluster sync on/off. + Enabled bool `json:"enabled"` + // Key is the cluster pre-shared key. Every node in a cluster must share the + // exact same value; it is the sole secret that authenticates membership and + // encrypts traffic. Empty key => sync stays disabled. + Key string `json:"key"` + // Peers are the base URLs of the other nodes, e.g. "https://node2.example.com". + // The cluster endpoints are reached at /api/cluster/... + Peers []string `json:"peers"` + // ShareDrivers, if non-empty, limits sharing to storages of these driver + // types (e.g. ["115 Cloud","BaiduNetdisk"]). Empty => share every driver. + ShareDrivers []string `json:"share_drivers"` + // ShareMounts, if non-empty, limits sharing to these exact mount paths. + // Empty => no mount-path restriction. + ShareMounts []string `json:"share_mounts"` + // ShareDeletes, when true, propagates storage deletions to peers as + // tombstones. Default false: deleting a mount on one node must NOT silently + // wipe it cluster-wide — sharing is about credentials/config, not lifecycle. + ShareDeletes bool `json:"share_deletes"` + // ApplyRemote, when true, lets incoming peer configs create/update local + // storages. Default (false) makes a node share-only/observe; set true on + // nodes that should adopt peer credentials. Most deployments want this true. + ApplyRemote bool `json:"apply_remote"` + // AnnounceIntervalSec is how often this node broadcasts its digest so peers + // can pull anything they missed. 0 => default. + AnnounceIntervalSec int `json:"announce_interval_sec"` + // RequestTimeoutSec bounds each outbound peer HTTP request. 0 => default. + RequestTimeoutSec int `json:"request_timeout_sec"` +} + +const ( + defaultAnnounceIntervalSec = 60 + defaultRequestTimeoutSec = 15 +) + +func (c *Config) announceInterval() int { + if c.AnnounceIntervalSec <= 0 { + return defaultAnnounceIntervalSec + } + return c.AnnounceIntervalSec +} + +func (c *Config) requestTimeout() int { + if c.RequestTimeoutSec <= 0 { + return defaultRequestTimeoutSec + } + return c.RequestTimeoutSec +} + +// active reports whether sync should actually run: enabled, with a key and at +// least one peer. +func (c *Config) active() bool { + return c.Enabled && strings.TrimSpace(c.Key) != "" && len(c.peerList()) > 0 +} + +// peerList returns the trimmed, non-empty peer URLs. +func (c *Config) peerList() []string { + out := make([]string, 0, len(c.Peers)) + for _, p := range c.Peers { + if p = strings.TrimRight(strings.TrimSpace(p), "/"); p != "" { + out = append(out, p) + } + } + return out +} + +// shouldShare decides whether a storage of the given driver/mount is in scope. +func (c *Config) shouldShare(driver, mountPath string) bool { + if len(c.ShareDrivers) > 0 && !contains(c.ShareDrivers, driver) { + return false + } + if len(c.ShareMounts) > 0 && !contains(c.ShareMounts, mountPath) { + return false + } + return true +} + +func contains(haystack []string, needle string) bool { + for _, h := range haystack { + if strings.EqualFold(strings.TrimSpace(h), needle) { + return true + } + } + return false +} + +// configStore handles persistence of Config under a directory. +type configStore struct { + mu sync.RWMutex + dir string + cfg Config +} + +func newConfigStore(dir string) *configStore { + return &configStore{dir: dir} +} + +func (cs *configStore) path() string { return filepath.Join(cs.dir, "config.json") } + +// loadOrInit reads config.json, or returns a zero (disabled) config if absent. +func (cs *configStore) loadOrInit() (Config, error) { + cs.mu.Lock() + defer cs.mu.Unlock() + b, err := os.ReadFile(cs.path()) + if err != nil { + if os.IsNotExist(err) { + cs.cfg = Config{} + return cs.cfg, nil + } + return Config{}, err + } + var c Config + if err := json.Unmarshal(b, &c); err != nil { + return Config{}, err + } + cs.cfg = c + return c, nil +} + +func (cs *configStore) get() Config { + cs.mu.RLock() + defer cs.mu.RUnlock() + return cs.cfg +} + +// save writes config.json atomically (write temp + rename). +func (cs *configStore) save(c Config) error { + cs.mu.Lock() + defer cs.mu.Unlock() + if err := os.MkdirAll(cs.dir, 0o700); err != nil { + return err + } + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + tmp := cs.path() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + if err := os.Rename(tmp, cs.path()); err != nil { + return err + } + cs.cfg = c + return nil +} diff --git a/internal/cluster/crypto.go b/internal/cluster/crypto.go new file mode 100644 index 0000000000..37d5722f05 --- /dev/null +++ b/internal/cluster/crypto.go @@ -0,0 +1,128 @@ +package cluster + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/ed25519" + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/base32" + "errors" + "fmt" + "io" +) + +// Cryptographic design (see docs/cluster-sync.md): +// +// - A cluster pre-shared key (PSK) is the root of trust. HKDF-SHA256 expands it +// into an AES-256-GCM session key. Only nodes holding the PSK can seal/open +// envelopes, which both authenticates membership and encrypts the payload — +// essential because storage configs carry secrets (cloud tokens/passwords). +// - Replay is prevented by a per-envelope random nonce plus a timestamp window +// and a recently-seen-nonce cache (enforced by the transport layer). +// - Each node owns an ed25519 identity keypair. Every synced config version is +// signed by its origin node, so a malicious member cannot forge a higher +// version for a mount it doesn't own (CRDT integrity), and nodes can be +// revoked by dropping their public key. +const ( + aeadInfo = "openlist-cluster-aead-v1" + aeadKeyN = 32 // AES-256 + nodeIDLen = 16 // bytes of the pubkey hash encoded into the node id +) + +// deriveAEADKey expands the cluster PSK into a stable AES-256 key. +func deriveAEADKey(psk []byte) ([]byte, error) { + if len(psk) == 0 { + return nil, errors.New("empty cluster key") + } + // Fixed salt: the PSK is the secret; a per-cluster constant salt is fine and + // keeps key derivation deterministic across nodes. + return hkdf.Key(sha256.New, psk, []byte("openlist-cluster"), aeadInfo, aeadKeyN) +} + +// aead builds the AES-256-GCM AEAD from a derived key. +func newAEAD(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// seal encrypts plaintext with the cluster key, binding aad. Output is +// nonce||ciphertext. aad is authenticated but not encrypted. +func seal(key, plaintext, aad []byte) ([]byte, error) { + gcm, err := newAEAD(key) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + ct := gcm.Seal(nil, nonce, plaintext, aad) + out := make([]byte, 0, len(nonce)+len(ct)) + out = append(out, nonce...) + out = append(out, ct...) + return out, nil +} + +// open reverses seal. It fails (returns error) if the key is wrong, the data was +// tampered with, or aad does not match — i.e. it both decrypts and authenticates. +func open(key, blob, aad []byte) ([]byte, error) { + gcm, err := newAEAD(key) + if err != nil { + return nil, err + } + ns := gcm.NonceSize() + if len(blob) < ns { + return nil, errors.New("ciphertext too short") + } + nonce, ct := blob[:ns], blob[ns:] + return gcm.Open(nil, nonce, ct, aad) +} + +// identity is a node's ed25519 keypair plus its derived node id. +type identity struct { + NodeID string + Pub ed25519.PublicKey + priv ed25519.PrivateKey +} + +// newIdentity generates a fresh ed25519 identity. +func newIdentity() (*identity, error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + return &identity{NodeID: nodeIDFromPub(pub), Pub: pub, priv: priv}, nil +} + +// identityFromSeed rebuilds an identity from a persisted private-key seed. +func identityFromSeed(seed []byte) (*identity, error) { + if len(seed) != ed25519.SeedSize { + return nil, fmt.Errorf("bad identity seed length %d", len(seed)) + } + priv := ed25519.NewKeyFromSeed(seed) + pub := priv.Public().(ed25519.PublicKey) + return &identity{NodeID: nodeIDFromPub(pub), Pub: pub, priv: priv}, nil +} + +func (id *identity) seed() []byte { return id.priv.Seed() } + +func (id *identity) sign(msg []byte) []byte { return ed25519.Sign(id.priv, msg) } + +// nodeIDFromPub derives a short, stable, human-friendly node id from a pubkey. +func nodeIDFromPub(pub ed25519.PublicKey) string { + sum := sha256.Sum256(pub) + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(sum[:nodeIDLen]) +} + +// verifySig checks an ed25519 signature against a known public key. +func verifySig(pub ed25519.PublicKey, msg, sig []byte) bool { + if len(pub) != ed25519.PublicKeySize { + return false + } + return ed25519.Verify(pub, msg, sig) +} diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go new file mode 100644 index 0000000000..efe9ca1fe3 --- /dev/null +++ b/internal/cluster/engine.go @@ -0,0 +1,543 @@ +package cluster + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + // replayWindowSec bounds clock skew + in-flight time for envelope freshness. + replayWindowSec = 300 + // syncPath is the single endpoint peers exchange sealed messages on. + syncPath = "/api/cluster/sync" +) + +// Manager is the running cluster-sync engine for this node. +type Manager struct { + dir string + id *identity + cfgStore *configStore + state *store + replay *replayCache + client *http.Client + + persistMu sync.Mutex + + stopCh chan struct{} + once sync.Once +} + +// Default is the process-wide manager, set by Init. It is nil when cluster sync +// was never initialized. +var Default *Manager + +func now() int64 { return time.Now().Unix() } + +// Init constructs the manager: loads identity + config + persisted CRDT state and +// registers the storage hook so local credential changes propagate. It does NOT +// start the network loops — call Start once storages are loaded. Safe to call +// even when the feature is disabled (it simply stays dormant). +func Init(dataDir string) (*Manager, error) { + dir := filepath.Join(dataDir, "cluster") + id, err := loadOrCreateIdentity(dir) + if err != nil { + return nil, err + } + m := &Manager{ + dir: dir, + id: id, + cfgStore: newConfigStore(dir), + state: newStore(), + replay: newReplayCache(replayWindowSec), + stopCh: make(chan struct{}), + } + if _, err := m.cfgStore.loadOrInit(); err != nil { + return nil, err + } + cfg := m.cfgStore.get() + m.client = &http.Client{Timeout: time.Duration(cfg.requestTimeout()) * time.Second} + m.loadState() + + op.RegisterStorageHook(m.onStorageHook) + Default = m + return m, nil +} + +// NodeID returns this node's stable identity string. +func (m *Manager) NodeID() string { return m.id.NodeID } + +// RecordView is a redaction-safe summary of one synced mount for the admin UI. +// It deliberately omits the Addition (which holds secrets/tokens). +type RecordView struct { + MountPath string `json:"mount_path"` + Driver string `json:"driver"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + Tombstone bool `json:"tombstone"` + UpdatedAt int64 `json:"updated_at"` + Self bool `json:"self"` // true if this node authored the current version +} + +// Status is the admin overview of the cluster state. +type Status struct { + NodeID string `json:"node_id"` + Enabled bool `json:"enabled"` + Active bool `json:"active"` + Peers []string `json:"peers"` + Records []RecordView `json:"records"` +} + +// Status returns a redaction-safe snapshot for the admin UI. +func (m *Manager) Status() Status { + cfg := m.cfgStore.get() + recs := m.state.snapshot() + views := make([]RecordView, 0, len(recs)) + for _, r := range recs { + views = append(views, RecordView{ + MountPath: r.MountPath, + Driver: r.Config.Driver, + Version: r.Version, + Origin: r.Origin, + Tombstone: r.Tombstone, + UpdatedAt: r.UpdatedAt, + Self: r.Origin == m.id.NodeID, + }) + } + return Status{ + NodeID: m.id.NodeID, + Enabled: cfg.Enabled, + Active: cfg.active(), + Peers: cfg.peerList(), + Records: views, + } +} + +// ---- persistence ---- + +type persistedState struct { + Lamport uint64 `json:"lamport"` + Records []*record `json:"records"` +} + +func (m *Manager) statePath() string { return filepath.Join(m.dir, "state.json") } + +func (m *Manager) loadState() { + b, err := os.ReadFile(m.statePath()) + if err != nil { + return + } + var ps persistedState + if err := json.Unmarshal(b, &ps); err != nil { + utils.Log.Warnf("[cluster] corrupt state file, ignoring: %v", err) + return + } + m.state.load(ps.Records, ps.Lamport) +} + +func (m *Manager) persist() { + m.persistMu.Lock() + defer m.persistMu.Unlock() + ps := persistedState{Lamport: m.state.lamportNow(), Records: m.state.snapshot()} + b, err := json.MarshalIndent(ps, "", " ") + if err != nil { + return + } + if err := os.MkdirAll(m.dir, 0o700); err != nil { + return + } + tmp := m.statePath() + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + utils.Log.Warnf("[cluster] failed to persist state: %v", err) + return + } + _ = os.Rename(tmp, m.statePath()) +} + +// ---- config access ---- + +// GetConfig returns the current config (with the key redacted for display when +// redact is true). +func (m *Manager) GetConfig(redact bool) Config { + c := m.cfgStore.get() + if redact && c.Key != "" { + c.Key = "********" + } + return c +} + +// SetConfig persists a new config. A blank/"********" key keeps the existing key. +func (m *Manager) SetConfig(c Config) error { + old := m.cfgStore.get() + if c.Key == "" || c.Key == "********" { + c.Key = old.Key + } + if err := m.cfgStore.save(c); err != nil { + return err + } + m.client.Timeout = time.Duration(c.requestTimeout()) * time.Second + // Seed records for any newly in-scope local storages so we announce them. + go m.seedLocalStorages() + return nil +} + +// ---- lifecycle ---- + +// Start seeds records from local storages and launches the anti-entropy loop. +func (m *Manager) Start() { + m.once.Do(func() { + go m.seedLocalStorages() + go m.announceLoop() + }) +} + +// Stop halts background loops. +func (m *Manager) Stop() { + select { + case <-m.stopCh: + default: + close(m.stopCh) + } +} + +// seedLocalStorages records the current healthy, in-scope local storages as CRDT +// records (idempotent: unchanged configs cause no version churn) so this node has +// something to announce/serve. +func (m *Manager) seedLocalStorages() { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + var changed bool + for _, d := range op.GetAllStorages() { + st := d.GetStorage() + if st.Status != op.WORK || !cfg.shouldShare(st.Driver, st.MountPath) { + continue + } + if _, ok := m.state.localChange(m.id, fromModel(st), now()); ok { + changed = true + } + } + if changed { + m.persist() + m.broadcast(&syncMessage{Type: "announce", Digests: m.state.digests()}) + } +} + +// announceLoop periodically broadcasts our digest so peers can pull anything they +// missed (anti-entropy backstop for lost push messages). +func (m *Manager) announceLoop() { + for { + cfg := m.cfgStore.get() + interval := time.Duration(cfg.announceInterval()) * time.Second + select { + case <-m.stopCh: + return + case <-time.After(interval): + } + cfg = m.cfgStore.get() + if !cfg.active() { + continue + } + m.broadcast(&syncMessage{Type: "announce", Digests: m.state.digests()}) + } +} + +// ---- storage hook (local change source) ---- + +func (m *Manager) onStorageHook(typ string, d driver.Driver) { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + st := d.GetStorage() + if !cfg.shouldShare(st.Driver, st.MountPath) { + return + } + switch typ { + case "add", "update": + if st.Status != op.WORK { + // Health gating: never propagate a broken/expired token. Instead try + // to recover a good one from a peer. + m.pullMount(st.MountPath) + return + } + rec, ok := m.state.localChange(m.id, fromModel(st), now()) + if ok { + m.persist() + m.broadcast(&syncMessage{Type: "push", Records: []*record{rec}}) + } + case "del": + if !cfg.ShareDeletes { + return + } + rec, ok := m.state.localDelete(m.id, st.MountPath, now()) + if ok { + m.persist() + m.broadcast(&syncMessage{Type: "push", Records: []*record{rec}}) + } + case "token-invalid": + // A driver reported its token is dead; pull a fresh one from peers. + m.pullMount(st.MountPath) + } +} + +// pullMount asks every peer for the latest record of a single mount and applies +// the best healthy answer. +func (m *Manager) pullMount(mountPath string) { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + msg := &syncMessage{Type: "pull", Wants: []string{mountPath}} + for _, peer := range cfg.peerList() { + m.exchange(peer, msg) + } +} + +// ---- message processing (shared by client replies and server requests) ---- + +// process integrates an incoming message and returns the reply to send back. +// It is the heart of the anti-entropy algorithm. +func (m *Manager) process(in *syncMessage) *syncMessage { + reply := &syncMessage{Type: "reply"} + + // 1. Absorb any full records offered to us. + m.applyRecords(in.Records) + + // 2. Anti-entropy on digests: tell the peer where we differ. + if len(in.Digests) > 0 { + peerHas := make(map[string]digest, len(in.Digests)) + for _, dg := range in.Digests { + peerHas[dg.MountPath] = dg + local, ok := m.state.get(dg.MountPath) + if !ok { + // We lack it entirely — request the full record. + reply.Wants = append(reply.Wants, dg.MountPath) + continue + } + cmp := digestOf(local) + switch { + case digestDominates(cmp, dg): + reply.Records = append(reply.Records, local) // ours is newer + case digestDominates(dg, cmp): + reply.Wants = append(reply.Wants, dg.MountPath) // theirs is newer + } + } + // Records we hold that the peer never mentioned — it is missing them. + for _, local := range m.state.snapshot() { + if _, seen := peerHas[local.MountPath]; !seen { + reply.Records = append(reply.Records, local) + } + } + } + + // 3. Serve explicit wants (pull). + for _, mp := range in.Wants { + if r, ok := m.state.get(mp); ok { + reply.Records = append(reply.Records, r) + } + } + return reply +} + +// applyRecords verifies, merges and (if configured) applies a batch of records. +func (m *Manager) applyRecords(recs []*record) { + if len(recs) == 0 { + return + } + cfg := m.cfgStore.get() + var dirty bool + for _, r := range recs { + if r == nil || !r.verify() { + continue + } + switch m.state.merge(r) { + case mergeApplied: + dirty = true + if cfg.ApplyRemote { + m.applyToLocal(r) + } + case mergeTombstone: + dirty = true + if cfg.ApplyRemote { + m.deleteLocal(r.MountPath) + } + } + } + if dirty { + m.persist() + } +} + +// applyToLocal creates or updates the local storage from a record's config. The +// resulting op hook is neutralized by content-hash idempotency: state already +// holds this exact hash, so onStorageHook's localChange is a no-op. +func (m *Manager) applyToLocal(r *record) { + ctx := context.Background() + if d, err := op.GetStorageByMountPath(r.MountPath); err == nil { + existing := *d.GetStorage() // copy; preserve ID/Status + r.Config.applyTo(&existing) + if err := op.UpdateStorage(ctx, existing); err != nil { + utils.Log.Warnf("[cluster] apply update %s failed: %v", r.MountPath, err) + } else { + utils.Log.Infof("[cluster] applied peer config for %s (v%d from %s)", r.MountPath, r.Version, r.Origin) + } + return + } + var st model.Storage + r.Config.applyTo(&st) + if _, err := op.CreateStorage(ctx, st); err != nil { + utils.Log.Warnf("[cluster] apply create %s failed: %v", r.MountPath, err) + } else { + utils.Log.Infof("[cluster] created storage %s from peer (v%d from %s)", r.MountPath, r.Version, r.Origin) + } +} + +func (m *Manager) deleteLocal(mountPath string) { + d, err := op.GetStorageByMountPath(mountPath) + if err != nil { + return + } + id := d.GetStorage().ID + if err := op.DeleteStorageById(context.Background(), id); err != nil { + utils.Log.Warnf("[cluster] apply delete %s failed: %v", mountPath, err) + } +} + +// ---- networking ---- + +// broadcast sends a message to every peer (fire-and-forget), processing each +// peer's reply. +func (m *Manager) broadcast(msg *syncMessage) { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + for _, peer := range cfg.peerList() { + go m.exchange(peer, msg) + } +} + +// exchange performs one full round-trip with a single peer: send msg, absorb the +// reply's records, and satisfy any records the peer asked for (reply.Wants) with +// an immediate follow-up push to that same peer. +func (m *Manager) exchange(peer string, msg *syncMessage) { + reply, err := m.send(peer, msg) + if err != nil { + utils.Log.Debugf("[cluster] send to %s failed: %v", peer, err) + return + } + m.applyRecords(reply.Records) + if len(reply.Wants) > 0 { + out := &syncMessage{Type: "push"} + for _, mp := range reply.Wants { + if r, ok := m.state.get(mp); ok { + out.Records = append(out.Records, r) + } + } + if len(out.Records) > 0 { + if _, err := m.send(peer, out); err != nil { + utils.Log.Debugf("[cluster] follow-up push to %s failed: %v", peer, err) + } + } + } +} + +// send seals msg, POSTs it to a peer's sync endpoint, and returns the decrypted +// reply. +func (m *Manager) send(peer string, msg *syncMessage) (*syncMessage, error) { + cfg := m.cfgStore.get() + key, err := deriveAEADKey([]byte(cfg.Key)) + if err != nil { + return nil, err + } + body, err := sealEnvelope(key, m.id, msg, now()) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, peer+syncPath, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := m.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, &httpError{code: resp.StatusCode, body: string(raw)} + } + _, reply, err := openEnvelope(key, raw, now(), m.replay, replayWindowSec) + if err != nil { + return nil, err + } + return reply, nil +} + +// HandleSync is the HTTP handler body: it decrypts a peer's request, processes +// it, and writes back a sealed reply. Returns the sealed reply bytes and any +// error (caller maps error to an HTTP status). +func (m *Manager) HandleSync(raw []byte) ([]byte, error) { + cfg := m.cfgStore.get() + if !cfg.active() { + return nil, errDisabled + } + key, err := deriveAEADKey([]byte(cfg.Key)) + if err != nil { + return nil, err + } + _, msg, err := openEnvelope(key, raw, now(), m.replay, replayWindowSec) + if err != nil { + return nil, err + } + reply := m.process(msg) + return sealEnvelope(key, m.id, reply, now()) +} + +type httpError struct { + code int + body string +} + +func (e *httpError) Error() string { return e.body } + +type sentinel string + +func (s sentinel) Error() string { return string(s) } + +const errDisabled = sentinel("cluster sync disabled") + +// ---- digest helpers ---- + +func digestOf(r *record) digest { + return digest{MountPath: r.MountPath, ContentHash: r.ContentHash, Version: r.Version, Origin: r.Origin, Tombstone: r.Tombstone} +} + +// digestDominates reports whether a wins over b under the same LWW ordering as +// records. +func digestDominates(a, b digest) bool { + if a.Version != b.Version { + return a.Version > b.Version + } + if a.Origin != b.Origin { + return a.Origin > b.Origin + } + return a.ContentHash > b.ContentHash +} diff --git a/internal/cluster/identity.go b/internal/cluster/identity.go new file mode 100644 index 0000000000..add3f4aa8c --- /dev/null +++ b/internal/cluster/identity.go @@ -0,0 +1,30 @@ +package cluster + +import ( + "crypto/ed25519" + "os" + "path/filepath" +) + +// loadOrCreateIdentity returns this node's stable ed25519 identity, generating +// and persisting a fresh one (the 32-byte seed) on first run. The seed file is +// 0600 — it is the node's private key and the basis of its node id, which other +// nodes pin (a node id is the hash of the public key). +func loadOrCreateIdentity(dir string) (*identity, error) { + path := filepath.Join(dir, "identity.key") + seed, err := os.ReadFile(path) + if err == nil && len(seed) == ed25519.SeedSize { + return identityFromSeed(seed) + } + id, err := newIdentity() + if err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(path, id.seed(), 0o600); err != nil { + return nil, err + } + return id, nil +} diff --git a/internal/cluster/state.go b/internal/cluster/state.go new file mode 100644 index 0000000000..5c5835bb19 --- /dev/null +++ b/internal/cluster/state.go @@ -0,0 +1,346 @@ +package cluster + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "sync" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// syncableStorage is the canonical, node-independent view of a storage mount that +// we replicate between cluster nodes. Node-local runtime fields (ID, Status, +// Modified) are deliberately excluded so the content hash is identical on every +// node for the same logical config — that idempotency is what lets a node ignore +// a peer that reports "the same token I already have" ("其他节点看到一样的token就不用变"). +type syncableStorage struct { + MountPath string `json:"mount_path"` + Order int `json:"order"` + Driver string `json:"driver"` + CacheExpiration int `json:"cache_expiration"` + CustomCachePolicies string `json:"custom_cache_policies"` + Addition string `json:"addition"` + Remark string `json:"remark"` + Disabled bool `json:"disabled"` + DisableIndex bool `json:"disable_index"` + EnableSign bool `json:"enable_sign"` + // Sort + OrderBy string `json:"order_by"` + OrderDirection string `json:"order_direction"` + ExtractFolder string `json:"extract_folder"` + // Proxy + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` + DownProxyURL string `json:"down_proxy_url"` + DisableProxySign bool `json:"disable_proxy_sign"` +} + +func fromModel(s *model.Storage) syncableStorage { + return syncableStorage{ + MountPath: s.MountPath, + Order: s.Order, + Driver: s.Driver, + CacheExpiration: s.CacheExpiration, + CustomCachePolicies: s.CustomCachePolicies, + Addition: s.Addition, + Remark: s.Remark, + Disabled: s.Disabled, + DisableIndex: s.DisableIndex, + EnableSign: s.EnableSign, + OrderBy: s.OrderBy, + OrderDirection: s.OrderDirection, + ExtractFolder: s.ExtractFolder, + WebProxy: s.WebProxy, + WebdavPolicy: s.WebdavPolicy, + ProxyRange: s.ProxyRange, + DownProxyURL: s.DownProxyURL, + DisableProxySign: s.DisableProxySign, + } +} + +// applyTo writes the synced fields onto an existing storage model, preserving the +// node-local runtime fields (ID/Status/Modified) of dst. +func (sc syncableStorage) applyTo(dst *model.Storage) { + dst.MountPath = sc.MountPath + dst.Order = sc.Order + dst.Driver = sc.Driver + dst.CacheExpiration = sc.CacheExpiration + dst.CustomCachePolicies = sc.CustomCachePolicies + dst.Addition = sc.Addition + dst.Remark = sc.Remark + dst.Disabled = sc.Disabled + dst.DisableIndex = sc.DisableIndex + dst.EnableSign = sc.EnableSign + dst.OrderBy = sc.OrderBy + dst.OrderDirection = sc.OrderDirection + dst.ExtractFolder = sc.ExtractFolder + dst.WebProxy = sc.WebProxy + dst.WebdavPolicy = sc.WebdavPolicy + dst.ProxyRange = sc.ProxyRange + dst.DownProxyURL = sc.DownProxyURL + dst.DisableProxySign = sc.DisableProxySign +} + +// canonicalJSON marshals deterministically. encoding/json already sorts struct +// fields by declaration order (stable), so a plain Marshal is canonical here. +func (sc syncableStorage) canonicalJSON() []byte { + b, _ := json.Marshal(sc) + return b +} + +// contentHash is the idempotency key: identical config → identical hash on every +// node, regardless of who authored it. +func (sc syncableStorage) contentHash() string { + sum := sha256.Sum256(sc.canonicalJSON()) + return hex.EncodeToString(sum[:]) +} + +// record is one replicated mount entry: the config plus CRDT version metadata. +// Ordering is a Lamport clock with the origin node id as a deterministic +// tiebreaker, giving a total order for last-writer-wins convergence. +type record struct { + MountPath string `json:"mount_path"` + ContentHash string `json:"content_hash"` + Version uint64 `json:"version"` // Lamport logical clock + Origin string `json:"origin"` // node id that authored this version + OriginPub []byte `json:"origin_pub"` // origin's ed25519 pubkey; node id == hash(pubkey) + Tombstone bool `json:"tombstone"` // true => mount was deleted + UpdatedAt int64 `json:"updated_at"` // unix seconds, informational only + Config syncableStorage `json:"config"` // empty when Tombstone + Sig []byte `json:"sig"` // origin's ed25519 signature over signingBytes +} + +// verify checks a record is internally authentic: the origin node id is the hash +// of the embedded pubkey (so a member cannot claim another node's id without its +// private key), the signature is valid under that pubkey, and the content hash +// matches the carried config. The cluster PSK (transport seal) gates membership; +// this gates per-record origin integrity for relayed records. +func (r *record) verify() bool { + if nodeIDFromPub(r.OriginPub) != r.Origin { + return false + } + if !r.Tombstone { + if r.Config.contentHash() != r.ContentHash { + return false + } + } else if r.ContentHash != (syncableStorage{MountPath: r.MountPath}).contentHash() { + return false + } + return verifySig(r.OriginPub, r.signingBytes(), r.Sig) +} + +// signingBytes is the stable byte string an origin node signs to authenticate a +// version. It excludes the signature itself and the (informational) timestamp. +func (r *record) signingBytes() []byte { + // length-free, delimiter-joined fields; ContentHash already binds Config. + var b []byte + b = append(b, r.MountPath...) + b = append(b, 0) + b = append(b, r.ContentHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + b = append(b, 0) + if r.Tombstone { + b = append(b, 1) + } else { + b = append(b, 0) + } + return b +} + +// dominates reports whether r should win over other under LWW ordering. +func (r *record) dominates(other *record) bool { + if r.Version != other.Version { + return r.Version > other.Version + } + // Equal Lamport time: break ties deterministically by origin id, then by + // content hash so two distinct concurrent edits still converge identically + // on every node. + if r.Origin != other.Origin { + return r.Origin > other.Origin + } + return r.ContentHash > other.ContentHash +} + +// digest is the compact form announced to peers so they can detect divergence +// without shipping full configs (and secrets) on every heartbeat. +type digest struct { + MountPath string `json:"mount_path"` + ContentHash string `json:"content_hash"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + Tombstone bool `json:"tombstone"` +} + +// store is the in-memory CRDT keyed by mount path, with a monotonically +// non-decreasing Lamport clock shared across all mounts. +type store struct { + mu sync.RWMutex + records map[string]*record + lamport uint64 +} + +func newStore() *store { + return &store{records: make(map[string]*record)} +} + +// tick advances and returns the Lamport clock for a locally-originated change. +func (s *store) tick() uint64 { + s.lamport++ + return s.lamport +} + +// observe bumps the Lamport clock to stay ahead of a value seen from a peer. +func (s *store) observe(v uint64) { + if v > s.lamport { + s.lamport = v + } +} + +func (s *store) get(mountPath string) (*record, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.records[mountPath] + return r, ok +} + +func (s *store) snapshot() []*record { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]*record, 0, len(s.records)) + for _, r := range s.records { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { return out[i].MountPath < out[j].MountPath }) + return out +} + +func (s *store) digests() []digest { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]digest, 0, len(s.records)) + for _, r := range s.records { + out = append(out, digest{ + MountPath: r.MountPath, + ContentHash: r.ContentHash, + Version: r.Version, + Origin: r.Origin, + Tombstone: r.Tombstone, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].MountPath < out[j].MountPath }) + return out +} + +// localChange records a config a node authored itself. It returns the new record +// to broadcast, or (nil,false) when nothing changed (idempotent no-op): same +// content hash and not resurrecting a tombstone. +func (s *store) localChange(id *identity, cfg syncableStorage, now int64) (*record, bool) { + s.mu.Lock() + defer s.mu.Unlock() + h := cfg.contentHash() + if cur, ok := s.records[cfg.MountPath]; ok && !cur.Tombstone && cur.ContentHash == h { + return nil, false // unchanged — do not bump version or churn peers + } + s.lamport++ + r := &record{ + MountPath: cfg.MountPath, + ContentHash: h, + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + Tombstone: false, + UpdatedAt: now, + Config: cfg, + } + r.Sig = id.sign(r.signingBytes()) + s.records[cfg.MountPath] = r + return r, true +} + +// localDelete authors a tombstone for a mount. Returns (nil,false) if already +// tombstoned or unknown. +func (s *store) localDelete(id *identity, mountPath string, now int64) (*record, bool) { + s.mu.Lock() + defer s.mu.Unlock() + cur, ok := s.records[mountPath] + if !ok || cur.Tombstone { + return nil, false + } + s.lamport++ + r := &record{ + MountPath: mountPath, + // hash of an empty config keeps signingBytes well-defined for tombstones + ContentHash: syncableStorage{MountPath: mountPath}.contentHash(), + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + Tombstone: true, + UpdatedAt: now, + } + r.Sig = id.sign(r.signingBytes()) + s.records[mountPath] = r + return r, true +} + +// mergeResult describes what merge did with an incoming record. +type mergeResult int + +const ( + mergeIgnored mergeResult = iota // incoming did not win (older/equal/duplicate) + mergeApplied // incoming won and replaced local (config changed) + mergeTombstone // incoming won and is a delete +) + +// merge integrates a peer's record. The caller must have already verified r.Sig +// against the origin's known public key. merge enforces LWW ordering and the +// idempotency rule, and advances the Lamport clock. +func (s *store) merge(r *record) mergeResult { + s.mu.Lock() + defer s.mu.Unlock() + if r.Version > s.lamport { + s.lamport = r.Version + } + cur, ok := s.records[r.MountPath] + if ok { + // Idempotent: identical live content, ignore regardless of version churn. + if !r.Tombstone && !cur.Tombstone && cur.ContentHash == r.ContentHash { + return mergeIgnored + } + if !r.dominates(cur) { + return mergeIgnored + } + } + cp := *r + s.records[r.MountPath] = &cp + if cp.Tombstone { + return mergeTombstone + } + return mergeApplied +} + +// load replaces the store contents from a persisted snapshot. +func (s *store) load(records []*record, lamport uint64) { + s.mu.Lock() + defer s.mu.Unlock() + s.records = make(map[string]*record, len(records)) + for _, r := range records { + s.records[r.MountPath] = r + } + s.lamport = lamport +} + +func (s *store) lamportNow() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.lamport +} diff --git a/internal/cluster/transport.go b/internal/cluster/transport.go new file mode 100644 index 0000000000..41e1e8ca28 --- /dev/null +++ b/internal/cluster/transport.go @@ -0,0 +1,142 @@ +package cluster + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sync" +) + +// syncMessage is the inner (encrypted) protocol payload exchanged between nodes. +// One message type carries push, anti-entropy announce, and pull in a single +// round trip: +// - Records: full configs the sender is offering (push, or reply to a Want). +// - Digests: compact (no-secrets) advert of everything the sender holds, so the +// receiver can detect what it is missing. +// - Wants: mount paths the sender wants the receiver to send back in full. +type syncMessage struct { + Type string `json:"type"` // "push" | "announce" | "pull" | "reply" + Records []*record `json:"records,omitempty"` + Digests []digest `json:"digests,omitempty"` + Wants []string `json:"wants,omitempty"` +} + +// envelope is the outer, on-the-wire structure. Its Cipher is the syncMessage +// sealed with the cluster key; only PSK holders can open it (membership auth + +// confidentiality for the secrets inside). Sender/PubKey/Timestamp/Nonce are +// authenticated as AEAD additional data so they cannot be tampered with. +type envelope struct { + Sender string `json:"sender"` // sender node id (== hash(PubKey)) + PubKey []byte `json:"pubkey"` // sender ed25519 pubkey + Timestamp int64 `json:"ts"` // unix seconds (replay window) + Nonce string `json:"nonce"` // base64 random, replay cache key + Cipher []byte `json:"cipher"` // sealed syncMessage +} + +// envelopeAAD binds the cleartext header fields to the ciphertext. +func envelopeAAD(sender string, pub []byte, ts int64, nonce string) []byte { + aad := make([]byte, 0, len(sender)+len(pub)+len(nonce)+16) + aad = append(aad, sender...) + aad = append(aad, 0) + aad = append(aad, pub...) + aad = append(aad, 0) + aad = append(aad, nonce...) + aad = append(aad, 0) + t := make([]byte, 8) + for i := 0; i < 8; i++ { + t[i] = byte(ts >> (8 * uint(i))) + } + aad = append(aad, t...) + return aad +} + +// sealEnvelope builds an encrypted envelope around a syncMessage. +func sealEnvelope(key []byte, id *identity, msg *syncMessage, now int64) ([]byte, error) { + plain, err := json.Marshal(msg) + if err != nil { + return nil, err + } + nb := make([]byte, 18) + if _, err := rand.Read(nb); err != nil { + return nil, err + } + nonce := base64.RawStdEncoding.EncodeToString(nb) + aad := envelopeAAD(id.NodeID, id.Pub, now, nonce) + cipher, err := seal(key, plain, aad) + if err != nil { + return nil, err + } + return json.Marshal(envelope{ + Sender: id.NodeID, + PubKey: id.Pub, + Timestamp: now, + Nonce: nonce, + Cipher: cipher, + }) +} + +// openEnvelope authenticates and decrypts an envelope. It enforces: +// - sender node id == hash(pubkey) (self-consistent identity), +// - timestamp within replayWindow of now, +// - nonce not seen before (replay cache), +// - AEAD open with the cluster key (membership + integrity + confidentiality). +func openEnvelope(key []byte, raw []byte, now int64, rc *replayCache, replayWindow int64) (*envelope, *syncMessage, error) { + var env envelope + if err := json.Unmarshal(raw, &env); err != nil { + return nil, nil, err + } + if nodeIDFromPub(env.PubKey) != env.Sender { + return nil, nil, errors.New("sender id does not match pubkey") + } + if d := now - env.Timestamp; d > replayWindow || d < -replayWindow { + return nil, nil, fmt.Errorf("timestamp outside replay window (%ds)", d) + } + if !rc.checkAndAdd(env.Nonce, now) { + return nil, nil, errors.New("replayed nonce") + } + aad := envelopeAAD(env.Sender, env.PubKey, env.Timestamp, env.Nonce) + plain, err := open(key, env.Cipher, aad) + if err != nil { + return nil, nil, fmt.Errorf("decrypt failed (wrong cluster key or tampered): %w", err) + } + var msg syncMessage + if err := json.Unmarshal(plain, &msg); err != nil { + return nil, nil, err + } + return &env, &msg, nil +} + +// replayCache remembers recently-seen nonces to reject replays. Entries expire +// after the replay window so memory stays bounded. +type replayCache struct { + mu sync.Mutex + seen map[string]int64 + window int64 +} + +func newReplayCache(window int64) *replayCache { + return &replayCache{seen: make(map[string]int64), window: window} +} + +// checkAndAdd returns false if the nonce was already seen (a replay); otherwise +// it records the nonce and returns true. It opportunistically evicts expired +// entries. +func (rc *replayCache) checkAndAdd(nonce string, now int64) bool { + rc.mu.Lock() + defer rc.mu.Unlock() + if _, ok := rc.seen[nonce]; ok { + return false + } + // Evict stale entries (cheap amortized sweep). + if len(rc.seen) > 0 { + for k, t := range rc.seen { + if now-t > rc.window { + delete(rc.seen, k) + } + } + } + rc.seen[nonce] = now + return true +} diff --git a/internal/op/hook.go b/internal/op/hook.go index 5cf01730d2..f149ccfce8 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -110,3 +110,13 @@ func callStorageHooks(typ string, storage driver.Driver) { func RegisterStorageHook(hook StorageHook) { storageHooks = append(storageHooks, hook) } + +// NotifyStorageTokenInvalid signals that a storage's credentials were found to be +// invalid/expired while in use (as opposed to successfully refreshed). It fires +// the storage hook with the "token-invalid" type so listeners — notably cluster +// sync — can react by pulling fresh credentials from a healthy peer instead of +// propagating the broken token. Drivers may call this when an API call fails with +// an unrecoverable auth error. +func NotifyStorageTokenInvalid(storage driver.Driver) { + go callStorageHooks("token-invalid", storage) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 2390069c9b..c78f02d155 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -305,6 +305,11 @@ func saveDriverStorage(driver driver.Driver) error { if err != nil { return errors.WithMessage(err, "failed update storage in database") } + // A driver persisting its own state is the canonical "token refreshed / + // rotated" moment. Fire the storage hook so listeners (e.g. cluster sync) + // can propagate the fresh credentials to peer nodes. Cluster sync dedups by + // content hash, so unchanged saves cause no churn. + go callStorageHooks("update", driver) return nil } diff --git a/server/handles/cluster.go b/server/handles/cluster.go new file mode 100644 index 0000000000..63ecc4177f --- /dev/null +++ b/server/handles/cluster.go @@ -0,0 +1,90 @@ +package handles + +import ( + "io" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/cluster" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// ClusterSync is the peer-to-peer endpoint nodes use to exchange sealed, +// encrypted sync messages. It is intentionally unauthenticated at the HTTP layer: +// authentication and confidentiality come from the cluster pre-shared key (only +// PSK holders can seal/open the AEAD envelope). The body and response are raw +// sealed bytes, not JSON. +func ClusterSync(c *gin.Context) { + m := cluster.Default + if m == nil { + c.Status(http.StatusServiceUnavailable) + return + } + raw, err := io.ReadAll(io.LimitReader(c.Request.Body, 16<<20)) + if err != nil { + c.Status(http.StatusBadRequest) + return + } + reply, err := m.HandleSync(raw) + if err != nil { + // Do not leak crypto details; a wrong key / replay / disabled all map to + // a generic rejection. + c.Status(http.StatusForbidden) + return + } + c.Data(http.StatusOK, "application/octet-stream", reply) +} + +// ---- Admin config/status ---- + +func clusterMgr(c *gin.Context) *cluster.Manager { + if cluster.Default == nil { + common.ErrorStrResp(c, "cluster sync is unavailable", 500) + return nil + } + return cluster.Default +} + +// ClusterGetConfig returns the current cluster config (key redacted) plus a live +// status overview. Admin only. +func ClusterGetConfig(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, gin.H{ + "config": m.GetConfig(true), + "status": m.Status(), + }) +} + +// ClusterSetConfig persists a new cluster config. A blank or "********" key keeps +// the existing key. Admin only. +func ClusterSetConfig(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + var cfg cluster.Config + if err := c.ShouldBind(&cfg); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := m.SetConfig(cfg); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, gin.H{ + "config": m.GetConfig(true), + "status": m.Status(), + }) +} + +// ClusterStatus returns just the live status overview. Admin only. +func ClusterStatus(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, m.Status()) +} diff --git a/server/router.go b/server/router.go index 2aacbbf3c2..3d86a5993d 100644 --- a/server/router.go +++ b/server/router.go @@ -110,6 +110,10 @@ func Init(e *gin.Engine) { api.GET("/plugin/manifest", handles.PluginManifest) api.GET("/plugin/asset/:name", handles.PluginAsset) + // Cluster storage-sync peer endpoint. Unauthenticated at the HTTP layer — + // authentication and encryption come from the cluster pre-shared key. + api.POST("/cluster/sync", handles.ClusterSync) + _fs(auth.Group("/fs")) fsAndShare(api.Group("/fs", middlewares.Auth(true))) _task(auth.Group("/task", middlewares.AuthNotGuest)) @@ -150,6 +154,12 @@ func admin(g *gin.RouterGroup) { plug.POST("/delete", handles.PluginDelete) plug.POST("/enable", handles.PluginSetEnabled) + // Cluster storage-sharing config/status. Admin only. + clusterGrp := g.Group("/cluster") + clusterGrp.GET("/config", handles.ClusterGetConfig) + clusterGrp.POST("/config", handles.ClusterSetConfig) + clusterGrp.GET("/status", handles.ClusterStatus) + // Admin-only user operations (create/delete/role-affecting). The profile-edit // endpoints (list/get/update) live in a separate group that also accepts a // non-admin who has been delegated the "manage user info" permission. From d85a14782e563e01ef9db470d7f207414bf5c5ef Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 03:03:33 +0800 Subject: [PATCH 084/107] feat(cluster): persistent WebSocket transport (NAT-friendly) + relay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the stateless HTTP request/reply sync link with a stateful, persistent WebSocket so nodes behind NAT can participate: a NAT'd node dials OUT to its configured reachable peers and keeps the link open (it cannot be dialed itself), while a reachable node accepts inbound links at /api/cluster/ws and RELAYS newly-applied records between everyone it is connected to — so two NAT'd nodes converge through a common reachable peer. Same sealed AEAD envelope per frame (PSK auth + confidentiality), so crypto/CRDT layers are unchanged. Adds a dial supervisor (auto-reconnect/backoff), ping keepalive to hold NAT mappings, and relay-on-change so propagation terminates. Drops the old POST /cluster/sync in favour of GET /cluster/ws. --- internal/cluster/conn.go | 291 +++++++++++++++++++++++++++++++++++++ internal/cluster/engine.go | 216 ++++++++++++++------------- server/handles/cluster.go | 26 +--- server/router.go | 7 +- 4 files changed, 408 insertions(+), 132 deletions(-) create mode 100644 internal/cluster/conn.go diff --git a/internal/cluster/conn.go b/internal/cluster/conn.go new file mode 100644 index 0000000000..65059855cc --- /dev/null +++ b/internal/cluster/conn.go @@ -0,0 +1,291 @@ +package cluster + +import ( + "net/http" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/gorilla/websocket" +) + +// Transport model (NAT-friendly): +// +// The sync link is a STATEFUL, persistent WebSocket, never a stateless +// per-message HTTP request. A node behind NAT cannot be dialed, so it instead +// dials OUT to the reachable peers listed in its config and keeps that +// connection open; frames flow in both directions over it for the connection's +// lifetime. A reachable node also ACCEPTS inbound connections at +// /api/cluster/ws and RELAYS applied records between everyone it is connected +// to — so two NAT'd nodes that both dial the same reachable peer still converge +// through it. Every frame is the same sealed envelope used elsewhere, so the +// crypto/CRDT layers are unchanged and transport-agnostic. +const ( + writeWait = 10 * time.Second + pongWait = 60 * time.Second + pingPeriod = (pongWait * 9) / 10 + dialBackoffMin = 2 * time.Second + dialBackoffMax = 60 * time.Second + sendQueueLen = 128 + maxFrameBytes = 16 << 20 +) + +var wsUpgrader = websocket.Upgrader{ + ReadBufferSize: 4096, + WriteBufferSize: 4096, + // Auth is the cluster PSK (AEAD), not the HTTP origin — accept any origin. + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// peerConn is one authenticated, persistent connection to another node. +type peerConn struct { + mgr *Manager + ws *websocket.Conn + send chan []byte + outbound bool // true if we dialed it + nodeID string // learned from the first authenticated frame + closeOnce sync.Once + closed chan struct{} +} + +func newPeerConn(mgr *Manager, ws *websocket.Conn, outbound bool) *peerConn { + return &peerConn{ + mgr: mgr, + ws: ws, + send: make(chan []byte, sendQueueLen), + outbound: outbound, + closed: make(chan struct{}), + } +} + +// enqueue queues a frame for sending; drops it (and closes the conn) if the peer +// is too slow, so a stuck peer can't block the whole node. +func (c *peerConn) enqueue(frame []byte) { + select { + case c.send <- frame: + case <-c.closed: + default: + utils.Log.Warnf("[cluster] peer %s send queue full, dropping connection", c.nodeID) + c.close() + } +} + +func (c *peerConn) close() { + c.closeOnce.Do(func() { + close(c.closed) + _ = c.ws.Close() + c.mgr.conns.remove(c) + }) +} + +// readPump authenticates and dispatches every incoming frame for the connection's +// lifetime. +func (c *peerConn) readPump() { + defer c.close() + c.ws.SetReadLimit(maxFrameBytes) + _ = c.ws.SetReadDeadline(time.Now().Add(pongWait)) + c.ws.SetPongHandler(func(string) error { + return c.ws.SetReadDeadline(time.Now().Add(pongWait)) + }) + for { + mt, data, err := c.ws.ReadMessage() + if err != nil { + return + } + if mt != websocket.BinaryMessage { + continue + } + c.mgr.handleFrame(c, data) + } +} + +// writePump serializes all writes for the connection and keeps it alive with +// periodic pings (essential to hold a NAT mapping open). +func (c *peerConn) writePump() { + ticker := time.NewTicker(pingPeriod) + defer func() { + ticker.Stop() + c.close() + }() + for { + select { + case <-c.closed: + return + case frame := <-c.send: + _ = c.ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.ws.WriteMessage(websocket.BinaryMessage, frame); err != nil { + return + } + case <-ticker.C: + _ = c.ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.ws.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// connRegistry tracks all live connections and indexes the most recent one per +// node id. +type connRegistry struct { + mu sync.RWMutex + conns map[*peerConn]struct{} + byNode map[string]*peerConn +} + +func newConnRegistry() *connRegistry { + return &connRegistry{ + conns: make(map[*peerConn]struct{}), + byNode: make(map[string]*peerConn), + } +} + +func (r *connRegistry) add(c *peerConn) { + r.mu.Lock() + defer r.mu.Unlock() + r.conns[c] = struct{}{} +} + +// bind records the authenticated node id for a connection (idempotent). +func (r *connRegistry) bind(c *peerConn, nodeID string) { + r.mu.Lock() + defer r.mu.Unlock() + c.nodeID = nodeID + r.byNode[nodeID] = c +} + +func (r *connRegistry) remove(c *peerConn) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.conns, c) + if c.nodeID != "" && r.byNode[c.nodeID] == c { + delete(r.byNode, c.nodeID) + } +} + +// all returns a snapshot of live connections. +func (r *connRegistry) all() []*peerConn { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*peerConn, 0, len(r.conns)) + for c := range r.conns { + out = append(out, c) + } + return out +} + +// hasNode reports whether we already have a live connection to a node id (used +// to avoid duplicate dialed+accepted links to the same peer). +func (r *connRegistry) hasNode(nodeID string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.byNode[nodeID] + return ok +} + +func (r *connRegistry) closeAll() { + for _, c := range r.all() { + c.close() + } +} + +// ---- connection establishment ---- + +// wsURL converts a peer base URL (http/https) into its cluster WebSocket URL. +func wsURL(peer string) string { + peer = strings.TrimRight(strings.TrimSpace(peer), "/") + switch { + case strings.HasPrefix(peer, "https://"): + peer = "wss://" + strings.TrimPrefix(peer, "https://") + case strings.HasPrefix(peer, "http://"): + peer = "ws://" + strings.TrimPrefix(peer, "http://") + case strings.HasPrefix(peer, "wss://"), strings.HasPrefix(peer, "ws://"): + // already a ws URL + default: + peer = "ws://" + peer + } + return peer + wsPath +} + +// startConn registers a connection, starts its pumps, and greets the peer with +// our digests so anti-entropy begins immediately. +func (m *Manager) startConn(c *peerConn) { + m.conns.add(c) + go c.writePump() + go c.readPump() + m.sendTo(c, &syncMessage{Type: "announce", Digests: m.state.digests()}) +} + +// dialPeer opens an outbound persistent connection to a peer and blocks until it +// closes. Dialing OUT is what lets a NAT'd node participate without being +// reachable itself. +func (m *Manager) dialPeer(peer string) error { + ws, _, err := websocket.DefaultDialer.Dial(wsURL(peer), nil) + if err != nil { + return err + } + c := newPeerConn(m, ws, true) + m.startConn(c) + <-c.closed + return nil +} + +// ServeWS upgrades an inbound HTTP request to a persistent connection. Auth is +// deferred to the first sealed frame (PSK), so the upgrade itself is open. +func (m *Manager) ServeWS(w http.ResponseWriter, r *http.Request) { + cfg := m.cfgStore.get() + if !cfg.active() { + http.Error(w, "cluster sync disabled", http.StatusServiceUnavailable) + return + } + ws, err := wsUpgrader.Upgrade(w, r, nil) + if err != nil { + return + } + m.startConn(newPeerConn(m, ws, false)) +} + +// dialSupervisor keeps one live outbound connection to every configured peer, +// re-dialing as connections drop or config changes. +func (m *Manager) dialSupervisor() { + for { + if cfg := m.cfgStore.get(); cfg.active() { + for _, peer := range cfg.peerList() { + m.ensureDial(peer) + } + } + select { + case <-m.stopCh: + return + case <-time.After(dialReconcile): + } + } +} + +// ensureDial starts (at most one) outbound dial loop for a peer URL. dialPeer +// blocks for the connection's lifetime, so the dialing flag also prevents a +// duplicate link while connected. +func (m *Manager) ensureDial(peer string) { + m.dialMu.Lock() + if m.dialing[peer] { + m.dialMu.Unlock() + return + } + m.dialing[peer] = true + m.dialMu.Unlock() + go func() { + defer func() { + m.dialMu.Lock() + delete(m.dialing, peer) + m.dialMu.Unlock() + }() + if err := m.dialPeer(peer); err != nil { + utils.Log.Debugf("[cluster] dial %s failed: %v", peer, err) + select { + case <-m.stopCh: + case <-time.After(dialBackoffMin): + } + } + }() +} diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index efe9ca1fe3..9f1d7514c8 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -1,11 +1,8 @@ package cluster import ( - "bytes" "context" "encoding/json" - "io" - "net/http" "os" "path/filepath" "sync" @@ -20,8 +17,11 @@ import ( const ( // replayWindowSec bounds clock skew + in-flight time for envelope freshness. replayWindowSec = 300 - // syncPath is the single endpoint peers exchange sealed messages on. - syncPath = "/api/cluster/sync" + // wsPath is the persistent-connection endpoint peers dial. + wsPath = "/api/cluster/ws" + // dialReconcile is how often the dial supervisor re-checks that every + // configured peer has a live outbound connection. + dialReconcile = 5 * time.Second ) // Manager is the running cluster-sync engine for this node. @@ -31,10 +31,13 @@ type Manager struct { cfgStore *configStore state *store replay *replayCache - client *http.Client + conns *connRegistry persistMu sync.Mutex + dialMu sync.Mutex + dialing map[string]bool // peer URLs with an in-flight/live outbound dial + stopCh chan struct{} once sync.Once } @@ -61,13 +64,13 @@ func Init(dataDir string) (*Manager, error) { cfgStore: newConfigStore(dir), state: newStore(), replay: newReplayCache(replayWindowSec), + conns: newConnRegistry(), + dialing: make(map[string]bool), stopCh: make(chan struct{}), } if _, err := m.cfgStore.loadOrInit(); err != nil { return nil, err } - cfg := m.cfgStore.get() - m.client = &http.Client{Timeout: time.Duration(cfg.requestTimeout()) * time.Second} m.loadState() op.RegisterStorageHook(m.onStorageHook) @@ -186,7 +189,9 @@ func (m *Manager) SetConfig(c Config) error { if err := m.cfgStore.save(c); err != nil { return err } - m.client.Timeout = time.Duration(c.requestTimeout()) * time.Second + // Drop all live connections so they reconnect with the new settings (key, + // peers, ...); the dial supervisor re-dials the configured peers. + m.conns.closeAll() // Seed records for any newly in-scope local storages so we announce them. go m.seedLocalStorages() return nil @@ -194,21 +199,24 @@ func (m *Manager) SetConfig(c Config) error { // ---- lifecycle ---- -// Start seeds records from local storages and launches the anti-entropy loop. +// Start seeds records from local storages and launches the connection dialer and +// anti-entropy loop. func (m *Manager) Start() { m.once.Do(func() { go m.seedLocalStorages() go m.announceLoop() + go m.dialSupervisor() }) } -// Stop halts background loops. +// Stop halts background loops and closes all connections. func (m *Manager) Stop() { select { case <-m.stopCh: default: close(m.stopCh) } + m.conns.closeAll() } // seedLocalStorages records the current healthy, in-scope local storages as CRDT @@ -293,38 +301,32 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { } } -// pullMount asks every peer for the latest record of a single mount and applies -// the best healthy answer. +// pullMount asks every connected peer for the latest record of a single mount. +// Answers arrive asynchronously and are applied by handleFrame. func (m *Manager) pullMount(mountPath string) { cfg := m.cfgStore.get() if !cfg.active() { return } - msg := &syncMessage{Type: "pull", Wants: []string{mountPath}} - for _, peer := range cfg.peerList() { - m.exchange(peer, msg) - } + m.broadcast(&syncMessage{Type: "pull", Wants: []string{mountPath}}) } -// ---- message processing (shared by client replies and server requests) ---- +// ---- message processing ---- -// process integrates an incoming message and returns the reply to send back. -// It is the heart of the anti-entropy algorithm. -func (m *Manager) process(in *syncMessage) *syncMessage { +// buildReply produces the anti-entropy answer to a peer's digests/wants. It does +// NOT absorb the incoming records (handleFrame does that first, separately, so it +// can relay newly-applied ones). Returns nil when there is nothing to send back. +func (m *Manager) buildReply(in *syncMessage) *syncMessage { reply := &syncMessage{Type: "reply"} - // 1. Absorb any full records offered to us. - m.applyRecords(in.Records) - - // 2. Anti-entropy on digests: tell the peer where we differ. + // Anti-entropy on digests: tell the peer where we differ. if len(in.Digests) > 0 { peerHas := make(map[string]digest, len(in.Digests)) for _, dg := range in.Digests { peerHas[dg.MountPath] = dg local, ok := m.state.get(dg.MountPath) if !ok { - // We lack it entirely — request the full record. - reply.Wants = append(reply.Wants, dg.MountPath) + reply.Wants = append(reply.Wants, dg.MountPath) // we lack it continue } cmp := digestOf(local) @@ -343,21 +345,29 @@ func (m *Manager) process(in *syncMessage) *syncMessage { } } - // 3. Serve explicit wants (pull). + // Serve explicit wants (pull). for _, mp := range in.Wants { if r, ok := m.state.get(mp); ok { reply.Records = append(reply.Records, r) } } + + if len(reply.Records) == 0 && len(reply.Wants) == 0 { + return nil + } return reply } -// applyRecords verifies, merges and (if configured) applies a batch of records. -func (m *Manager) applyRecords(recs []*record) { +// applyRecords verifies, merges and (if configured) applies a batch of records, +// returning the records that were newly applied/tombstoned so the caller can +// relay them onward (hub behaviour). Only genuine changes are returned, so relay +// naturally terminates. +func (m *Manager) applyRecords(recs []*record) []*record { if len(recs) == 0 { - return + return nil } cfg := m.cfgStore.get() + var applied []*record var dirty bool for _, r := range recs { if r == nil || !r.verify() { @@ -366,11 +376,13 @@ func (m *Manager) applyRecords(recs []*record) { switch m.state.merge(r) { case mergeApplied: dirty = true + applied = append(applied, r) if cfg.ApplyRemote { m.applyToLocal(r) } case mergeTombstone: dirty = true + applied = append(applied, r) if cfg.ApplyRemote { m.deleteLocal(r.MountPath) } @@ -379,6 +391,7 @@ func (m *Manager) applyRecords(recs []*record) { if dirty { m.persist() } + return applied } // applyToLocal creates or updates the local storage from a record's config. The @@ -416,114 +429,97 @@ func (m *Manager) deleteLocal(mountPath string) { } } -// ---- networking ---- +// ---- networking (persistent connections) ---- -// broadcast sends a message to every peer (fire-and-forget), processing each -// peer's reply. -func (m *Manager) broadcast(msg *syncMessage) { - cfg := m.cfgStore.get() - if !cfg.active() { - return - } - for _, peer := range cfg.peerList() { - go m.exchange(peer, msg) +// seal wraps a message in an encrypted envelope addressed from this node. +func (m *Manager) seal(msg *syncMessage) ([]byte, error) { + key, err := deriveAEADKey([]byte(m.cfgStore.get().Key)) + if err != nil { + return nil, err } + return sealEnvelope(key, m.id, msg, now()) } -// exchange performs one full round-trip with a single peer: send msg, absorb the -// reply's records, and satisfy any records the peer asked for (reply.Wants) with -// an immediate follow-up push to that same peer. -func (m *Manager) exchange(peer string, msg *syncMessage) { - reply, err := m.send(peer, msg) +// sendTo enqueues a sealed message on a single connection. +func (m *Manager) sendTo(c *peerConn, msg *syncMessage) { + frame, err := m.seal(msg) if err != nil { - utils.Log.Debugf("[cluster] send to %s failed: %v", peer, err) return } - m.applyRecords(reply.Records) - if len(reply.Wants) > 0 { - out := &syncMessage{Type: "push"} - for _, mp := range reply.Wants { - if r, ok := m.state.get(mp); ok { - out.Records = append(out.Records, r) - } - } - if len(out.Records) > 0 { - if _, err := m.send(peer, out); err != nil { - utils.Log.Debugf("[cluster] follow-up push to %s failed: %v", peer, err) - } - } - } + c.enqueue(frame) } -// send seals msg, POSTs it to a peer's sync endpoint, and returns the decrypted -// reply. -func (m *Manager) send(peer string, msg *syncMessage) (*syncMessage, error) { - cfg := m.cfgStore.get() - key, err := deriveAEADKey([]byte(cfg.Key)) - if err != nil { - return nil, err +// broadcast enqueues a sealed message on every live connection. The same sealed +// frame is reused for all peers (each peer keeps its own replay cache, so a +// shared nonce is fine). +func (m *Manager) broadcast(msg *syncMessage) { + conns := m.conns.all() + if len(conns) == 0 { + return } - body, err := sealEnvelope(key, m.id, msg, now()) + frame, err := m.seal(msg) if err != nil { - return nil, err + return } - req, err := http.NewRequest(http.MethodPost, peer+syncPath, bytes.NewReader(body)) - if err != nil { - return nil, err + for _, c := range conns { + c.enqueue(frame) } - req.Header.Set("Content-Type", "application/octet-stream") - resp, err := m.client.Do(req) - if err != nil { - return nil, err +} + +// relay forwards newly-applied records to every connection except the one they +// arrived on — this is what lets two NAT'd nodes converge through a common +// reachable peer. Records carry their own origin signature, so re-sealing them in +// our envelope does not weaken authenticity. +func (m *Manager) relay(recs []*record, except *peerConn) { + conns := m.conns.all() + if len(conns) <= 1 { + return } - defer resp.Body.Close() - raw, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + frame, err := m.seal(&syncMessage{Type: "push", Records: recs}) if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, &httpError{code: resp.StatusCode, body: string(raw)} + return } - _, reply, err := openEnvelope(key, raw, now(), m.replay, replayWindowSec) - if err != nil { - return nil, err + for _, c := range conns { + if c == except { + continue + } + c.enqueue(frame) } - return reply, nil } -// HandleSync is the HTTP handler body: it decrypts a peer's request, processes -// it, and writes back a sealed reply. Returns the sealed reply bytes and any -// error (caller maps error to an HTTP status). -func (m *Manager) HandleSync(raw []byte) ([]byte, error) { +// handleFrame authenticates and processes one inbound frame from a connection. +// A frame that fails to open (wrong cluster key / replay / stale) drops the +// connection — only PSK holders are admitted. +func (m *Manager) handleFrame(c *peerConn, data []byte) { cfg := m.cfgStore.get() if !cfg.active() { - return nil, errDisabled + c.close() + return } key, err := deriveAEADKey([]byte(cfg.Key)) if err != nil { - return nil, err + c.close() + return } - _, msg, err := openEnvelope(key, raw, now(), m.replay, replayWindowSec) + env, msg, err := openEnvelope(key, data, now(), m.replay, replayWindowSec) if err != nil { - return nil, err + utils.Log.Debugf("[cluster] frame rejected from %s: %v", c.nodeID, err) + c.close() + return + } + if c.nodeID == "" && env.Sender != m.id.NodeID { + m.conns.bind(c, env.Sender) + } + // Absorb offered records first, then relay the ones that were new. + if applied := m.applyRecords(msg.Records); len(applied) > 0 { + m.relay(applied, c) + } + // Answer the peer's anti-entropy digests / pull wants. + if reply := m.buildReply(msg); reply != nil { + m.sendTo(c, reply) } - reply := m.process(msg) - return sealEnvelope(key, m.id, reply, now()) -} - -type httpError struct { - code int - body string } -func (e *httpError) Error() string { return e.body } - -type sentinel string - -func (s sentinel) Error() string { return string(s) } - -const errDisabled = sentinel("cluster sync disabled") - // ---- digest helpers ---- func digestOf(r *record) digest { diff --git a/server/handles/cluster.go b/server/handles/cluster.go index 63ecc4177f..dbb195eee9 100644 --- a/server/handles/cluster.go +++ b/server/handles/cluster.go @@ -1,7 +1,6 @@ package handles import ( - "io" "net/http" "github.com/OpenListTeam/OpenList/v4/internal/cluster" @@ -9,30 +8,19 @@ import ( "github.com/gin-gonic/gin" ) -// ClusterSync is the peer-to-peer endpoint nodes use to exchange sealed, -// encrypted sync messages. It is intentionally unauthenticated at the HTTP layer: +// ClusterWS is the peer-to-peer endpoint nodes use to establish a persistent, +// stateful sync connection (WebSocket). A persistent connection is required so a +// node behind NAT can participate: it dials OUT to this endpoint and keeps the +// link open, since it cannot be dialed itself. The HTTP upgrade is open; // authentication and confidentiality come from the cluster pre-shared key (only -// PSK holders can seal/open the AEAD envelope). The body and response are raw -// sealed bytes, not JSON. -func ClusterSync(c *gin.Context) { +// PSK holders can seal/open the per-frame AEAD envelope). +func ClusterWS(c *gin.Context) { m := cluster.Default if m == nil { c.Status(http.StatusServiceUnavailable) return } - raw, err := io.ReadAll(io.LimitReader(c.Request.Body, 16<<20)) - if err != nil { - c.Status(http.StatusBadRequest) - return - } - reply, err := m.HandleSync(raw) - if err != nil { - // Do not leak crypto details; a wrong key / replay / disabled all map to - // a generic rejection. - c.Status(http.StatusForbidden) - return - } - c.Data(http.StatusOK, "application/octet-stream", reply) + m.ServeWS(c.Writer, c.Request) } // ---- Admin config/status ---- diff --git a/server/router.go b/server/router.go index 3d86a5993d..f919672ec4 100644 --- a/server/router.go +++ b/server/router.go @@ -110,9 +110,10 @@ func Init(e *gin.Engine) { api.GET("/plugin/manifest", handles.PluginManifest) api.GET("/plugin/asset/:name", handles.PluginAsset) - // Cluster storage-sync peer endpoint. Unauthenticated at the HTTP layer — - // authentication and encryption come from the cluster pre-shared key. - api.POST("/cluster/sync", handles.ClusterSync) + // Cluster storage-sync peer endpoint: a persistent WebSocket so NAT'd nodes + // can dial out and stay connected. Authentication/encryption come from the + // cluster pre-shared key (per-frame AEAD), not the HTTP layer. + api.GET("/cluster/ws", handles.ClusterWS) _fs(auth.Group("/fs")) fsAndShare(api.Group("/fs", middlewares.Auth(true))) From 2c8a9827adecf9faba41136196357bde0749a5f1 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 03:03:34 +0800 Subject: [PATCH 085/107] test(cluster): unit tests for crypto, CRDT merge, transport, config, registry Covers AEAD seal/open round-trip + wrong-key/tampered-aad rejection, ed25519 identity binding (node id == hash(pubkey)) and record self-authentication, LWW merge + content-hash idempotency + tombstones, envelope replay/stale rejection, config active/share-filter gating, and wsURL/conn registry. --- internal/cluster/config_test.go | 51 +++++++++++ internal/cluster/conn_test.go | 41 +++++++++ internal/cluster/crypto_test.go | 95 ++++++++++++++++++++ internal/cluster/state_test.go | 138 +++++++++++++++++++++++++++++ internal/cluster/transport_test.go | 62 +++++++++++++ 5 files changed, 387 insertions(+) create mode 100644 internal/cluster/config_test.go create mode 100644 internal/cluster/conn_test.go create mode 100644 internal/cluster/crypto_test.go create mode 100644 internal/cluster/state_test.go create mode 100644 internal/cluster/transport_test.go diff --git a/internal/cluster/config_test.go b/internal/cluster/config_test.go new file mode 100644 index 0000000000..13327f64bd --- /dev/null +++ b/internal/cluster/config_test.go @@ -0,0 +1,51 @@ +package cluster + +import "testing" + +func TestConfigActive(t *testing.T) { + cases := []struct { + name string + cfg Config + want bool + }{ + {"disabled", Config{Enabled: false, Key: "k", Peers: []string{"http://p"}}, false}, + {"no key", Config{Enabled: true, Peers: []string{"http://p"}}, false}, + {"no peers", Config{Enabled: true, Key: "k"}, false}, + {"blank peers only", Config{Enabled: true, Key: "k", Peers: []string{" ", "/"}}, false}, + {"active", Config{Enabled: true, Key: "k", Peers: []string{"http://p"}}, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.cfg.active(); got != c.want { + t.Fatalf("active() = %v, want %v", got, c.want) + } + }) + } +} + +func TestConfigShouldShare(t *testing.T) { + cfg := Config{ShareDrivers: []string{"115 Open"}, ShareMounts: []string{"/115"}} + if !cfg.shouldShare("115 Open", "/115") { + t.Fatal("in-scope storage should be shared") + } + if cfg.shouldShare("Local", "/115") { + t.Fatal("driver out of filter must not be shared") + } + if cfg.shouldShare("115 Open", "/other") { + t.Fatal("mount out of filter must not be shared") + } + + // Empty filters => share everything. + open := Config{} + if !open.shouldShare("AnyDriver", "/anywhere") { + t.Fatal("empty filters should share everything") + } +} + +func TestConfigPeerListNormalizes(t *testing.T) { + cfg := Config{Peers: []string{" http://a/ ", "http://b", "", " "}} + got := cfg.peerList() + if len(got) != 2 || got[0] != "http://a" || got[1] != "http://b" { + t.Fatalf("peerList normalize failed: %#v", got) + } +} diff --git a/internal/cluster/conn_test.go b/internal/cluster/conn_test.go new file mode 100644 index 0000000000..d966da55cf --- /dev/null +++ b/internal/cluster/conn_test.go @@ -0,0 +1,41 @@ +package cluster + +import "testing" + +func TestWSURL(t *testing.T) { + cases := map[string]string{ + "https://node.example.com": "wss://node.example.com" + wsPath, + "http://10.0.0.2:5244": "ws://10.0.0.2:5244" + wsPath, + "https://node.example.com/": "wss://node.example.com" + wsPath, + " http://h:1/ ": "ws://h:1" + wsPath, + "node.example.com": "ws://node.example.com" + wsPath, + "wss://already.example.com": "wss://already.example.com" + wsPath, + } + for in, want := range cases { + if got := wsURL(in); got != want { + t.Errorf("wsURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestConnRegistry(t *testing.T) { + r := newConnRegistry() + c1 := &peerConn{closed: make(chan struct{})} + c2 := &peerConn{closed: make(chan struct{})} + r.add(c1) + r.add(c2) + if len(r.all()) != 2 { + t.Fatalf("expected 2 conns, got %d", len(r.all())) + } + r.bind(c1, "NODE1") + if !r.hasNode("NODE1") { + t.Fatal("hasNode should find a bound node") + } + r.remove(c1) + if r.hasNode("NODE1") { + t.Fatal("removed conn's node id should be gone") + } + if len(r.all()) != 1 { + t.Fatalf("expected 1 conn after remove, got %d", len(r.all())) + } +} diff --git a/internal/cluster/crypto_test.go b/internal/cluster/crypto_test.go new file mode 100644 index 0000000000..85bf81002e --- /dev/null +++ b/internal/cluster/crypto_test.go @@ -0,0 +1,95 @@ +package cluster + +import ( + "bytes" + "testing" +) + +func TestSealOpenRoundTrip(t *testing.T) { + key, err := deriveAEADKey([]byte("super-secret-cluster-key")) + if err != nil { + t.Fatal(err) + } + plain := []byte(`{"token":"abc123","secret":"xyz"}`) + aad := []byte("aad-context") + blob, err := seal(key, plain, aad) + if err != nil { + t.Fatal(err) + } + got, err := open(key, blob, aad) + if err != nil { + t.Fatalf("open failed: %v", err) + } + if !bytes.Equal(got, plain) { + t.Fatalf("round-trip mismatch: %q != %q", got, plain) + } +} + +func TestOpenFailsWithWrongKey(t *testing.T) { + k1, _ := deriveAEADKey([]byte("key-one")) + k2, _ := deriveAEADKey([]byte("key-two")) + blob, _ := seal(k1, []byte("hello"), nil) + if _, err := open(k2, blob, nil); err == nil { + t.Fatal("expected open to fail with a different cluster key") + } +} + +func TestOpenFailsWithTamperedAAD(t *testing.T) { + key, _ := deriveAEADKey([]byte("k")) + blob, _ := seal(key, []byte("hello"), []byte("aad-1")) + if _, err := open(key, blob, []byte("aad-2")); err == nil { + t.Fatal("expected open to fail when aad differs") + } +} + +func TestDeriveAEADKeyDeterministicAndRejectsEmpty(t *testing.T) { + a, err := deriveAEADKey([]byte("same")) + if err != nil { + t.Fatal(err) + } + b, _ := deriveAEADKey([]byte("same")) + if !bytes.Equal(a, b) { + t.Fatal("key derivation must be deterministic across nodes") + } + if len(a) != aeadKeyN { + t.Fatalf("derived key length = %d, want %d", len(a), aeadKeyN) + } + if _, err := deriveAEADKey(nil); err == nil { + t.Fatal("empty PSK must be rejected") + } +} + +func TestIdentityNodeIDBoundToPubKey(t *testing.T) { + id, err := newIdentity() + if err != nil { + t.Fatal(err) + } + // node id is the hash of the pubkey — re-deriving must match. + if nodeIDFromPub(id.Pub) != id.NodeID { + t.Fatal("node id is not bound to pubkey") + } + // persistence round-trip via seed. + id2, err := identityFromSeed(id.seed()) + if err != nil { + t.Fatal(err) + } + if id2.NodeID != id.NodeID { + t.Fatal("identity not stable across seed reload") + } +} + +func TestSignVerify(t *testing.T) { + id, _ := newIdentity() + msg := []byte("authentic message") + sig := id.sign(msg) + if !verifySig(id.Pub, msg, sig) { + t.Fatal("valid signature rejected") + } + if verifySig(id.Pub, []byte("tampered"), sig) { + t.Fatal("signature verified against the wrong message") + } + other, _ := newIdentity() + if verifySig(other.Pub, msg, sig) { + t.Fatal("signature verified under the wrong pubkey") + } +} diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go new file mode 100644 index 0000000000..bf9f36b228 --- /dev/null +++ b/internal/cluster/state_test.go @@ -0,0 +1,138 @@ +package cluster + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func mkCfg(mount, addition string) syncableStorage { + return fromModel(&model.Storage{MountPath: mount, Driver: "115 Open", Addition: addition}) +} + +func TestLocalChangeIdempotent(t *testing.T) { + s := newStore() + id, _ := newIdentity() + cfg := mkCfg("/115", `{"token":"v1"}`) + + r1, ok := s.localChange(id, cfg, 1) + if !ok || r1 == nil { + t.Fatal("first change should produce a record") + } + v1 := r1.Version + + // Same content again => no-op, no version bump ("same token, don't change"). + if r2, ok := s.localChange(id, cfg, 2); ok || r2 != nil { + t.Fatal("identical config must be a no-op") + } + if cur, _ := s.get("/115"); cur.Version != v1 { + t.Fatalf("version churned on identical config: %d != %d", cur.Version, v1) + } + + // Changed content => new version. + r3, ok := s.localChange(id, mkCfg("/115", `{"token":"v2"}`), 3) + if !ok || r3.Version <= v1 { + t.Fatalf("changed config must bump version (%d should be > %d)", r3.Version, v1) + } +} + +func TestRecordVerify(t *testing.T) { + s := newStore() + id, _ := newIdentity() + r, _ := s.localChange(id, mkCfg("/a", `{"t":"1"}`), 1) + if !r.verify() { + t.Fatal("self-produced record must verify") + } + // Tamper with the content hash. + bad := *r + bad.ContentHash = "deadbeef" + if bad.verify() { + t.Fatal("record with mismatched content hash must not verify") + } + // Forge origin (claim a different node id without its key). + forged := *r + forged.Origin = "SOMEONEELSE" + if forged.verify() { + t.Fatal("record whose origin != hash(pubkey) must not verify") + } +} + +func TestMergeLWWAndIdempotency(t *testing.T) { + // Two nodes author the same mount; higher Lamport version wins. + nodeA, _ := newIdentity() + nodeB, _ := newIdentity() + + sa := newStore() + rA, _ := sa.localChange(nodeA, mkCfg("/m", `{"t":"A"}`), 1) + + sb := newStore() + // B is ahead on the Lamport clock. + sb.lamport = 5 + rB, _ := sb.localChange(nodeB, mkCfg("/m", `{"t":"B"}`), 1) + + // On a third node, apply A then B: B (higher version) must win. + s := newStore() + if got := s.merge(rA); got != mergeApplied { + t.Fatalf("first merge = %v, want applied", got) + } + if got := s.merge(rB); got != mergeApplied { + t.Fatalf("higher-version merge = %v, want applied", got) + } + cur, _ := s.get("/m") + if cur.Origin != nodeB.NodeID { + t.Fatal("LWW: higher Lamport version (B) should win") + } + + // Re-merging A (lower version) is ignored. + if got := s.merge(rA); got != mergeIgnored { + t.Fatalf("stale merge = %v, want ignored", got) + } + + // Merging an identical-content record is ignored (idempotent). + dupB := *rB + if got := s.merge(&dupB); got != mergeIgnored { + t.Fatalf("duplicate-content merge = %v, want ignored", got) + } +} + +func TestMergeTombstone(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.localChange(id, mkCfg("/d", `{"t":"1"}`), 1) + del, ok := s.localDelete(id, "/d", 2) + if !ok { + t.Fatal("delete should produce a tombstone") + } + + other := newStore() + other.merge(mustRecord(t, id, "/d", `{"t":"1"}`, 1)) + if got := other.merge(del); got != mergeTombstone { + t.Fatalf("tombstone merge = %v, want tombstone", got) + } + cur, _ := other.get("/d") + if !cur.Tombstone { + t.Fatal("record should be tombstoned after merge") + } +} + +func mustRecord(t *testing.T, id *identity, mount, addition string, now int64) *record { + t.Helper() + s := newStore() + r, ok := s.localChange(id, mkCfg(mount, addition), now) + if !ok { + t.Fatal("failed to build record") + } + return r +} + +func TestApplyToPreservesLocalRuntimeFields(t *testing.T) { + cfg := mkCfg("/x", `{"t":"shared"}`) + dst := &model.Storage{ID: 42, Status: "work", Addition: `{"t":"old"}`} + cfg.applyTo(dst) + if dst.ID != 42 || dst.Status != "work" { + t.Fatal("applyTo must preserve local ID/Status") + } + if dst.Addition != `{"t":"shared"}` { + t.Fatalf("applyTo must overwrite Addition, got %q", dst.Addition) + } +} diff --git a/internal/cluster/transport_test.go b/internal/cluster/transport_test.go new file mode 100644 index 0000000000..2ebb7c13df --- /dev/null +++ b/internal/cluster/transport_test.go @@ -0,0 +1,62 @@ +package cluster + +import "testing" + +func TestEnvelopeSealOpenRoundTrip(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + msg := &syncMessage{Type: "push", Wants: []string{"/a", "/b"}} + + raw, err := sealEnvelope(key, id, msg, 1000) + if err != nil { + t.Fatal(err) + } + env, got, err := openEnvelope(key, raw, 1000, rc, replayWindowSec) + if err != nil { + t.Fatalf("open envelope failed: %v", err) + } + if env.Sender != id.NodeID { + t.Fatal("sender id mismatch") + } + if got.Type != "push" || len(got.Wants) != 2 { + t.Fatalf("payload not preserved: %+v", got) + } +} + +func TestEnvelopeReplayRejected(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(key, id, &syncMessage{Type: "announce"}, 1000) + + if _, _, err := openEnvelope(key, raw, 1000, rc, replayWindowSec); err != nil { + t.Fatalf("first open should succeed: %v", err) + } + if _, _, err := openEnvelope(key, raw, 1001, rc, replayWindowSec); err == nil { + t.Fatal("replayed nonce must be rejected") + } +} + +func TestEnvelopeStaleTimestampRejected(t *testing.T) { + key, _ := deriveAEADKey([]byte("psk")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(key, id, &syncMessage{Type: "announce"}, 1000) + + // now is far beyond the replay window from the envelope timestamp. + if _, _, err := openEnvelope(key, raw, 1000+replayWindowSec+10, rc, replayWindowSec); err == nil { + t.Fatal("stale timestamp must be rejected") + } +} + +func TestEnvelopeWrongKeyRejected(t *testing.T) { + k1, _ := deriveAEADKey([]byte("psk-1")) + k2, _ := deriveAEADKey([]byte("psk-2")) + id, _ := newIdentity() + rc := newReplayCache(replayWindowSec) + raw, _ := sealEnvelope(k1, id, &syncMessage{Type: "announce"}, 1000) + if _, _, err := openEnvelope(k2, raw, 1000, rc, replayWindowSec); err == nil { + t.Fatal("wrong cluster key must fail to open (membership auth)") + } +} From b0deac1ce12526206b8289378289388a792fbcd7 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 04:19:54 +0800 Subject: [PATCH 086/107] feat(cluster): credential-only sync via manual multipartite groups + auto-discovery Redesign the cluster plugin around three user requirements: - Sync ONLY credentials. Instead of replicating whole storage configs, extract just the credential fields (token/cookie/secret/refresh/access/...) from a driver's Addition by name heuristic and overlay them onto peers, preserving every node-local field (mount path, root folder, cache, order, proxy). New creds.go isolates extraction/apply/hash; only changed fields are written, so identical credentials cause no churn or re-init. - Manual multipartite sync groups. A group links storages across nodes that share one account; the group set is cluster-shared (LWW doc, signed). Creds flow per-group, idempotent by cred-hash, health-gated (never push a broken token; pull from peers on token-invalid/non-WORK). - Auto-discovery. Any node that can open our AEAD envelope is admitted; public nodes advertise an addr and peers learn+dial the rest via inventory exchange (PEX). Manual peer lists are gone; an optional seed only bootstraps the first join. Node inventory (per-node storages) is gossiped to power the UI. state.go now holds groups + credential records + inventory CRDTs; status.go exposes a redaction-safe overview (nodes/liveness, per-group sync state, live connections, activity log, rollup stats) for the admin UI. Config drops the driver/mount filters and peers list in favour of key/label/addr/seeds. New POST /admin/cluster/groups endpoint. Unit tests cover credential extraction/overlay/idempotency, group + cred LWW, forgery rejection and inventory merge. --- docs/cluster-sync.md | 401 ++++++++++++++++++++ internal/cluster/config.go | 111 +++--- internal/cluster/config_test.go | 43 +-- internal/cluster/conn.go | 80 +++- internal/cluster/creds.go | 180 +++++++++ internal/cluster/engine.go | 643 ++++++++++++++++++++------------ internal/cluster/state.go | 640 +++++++++++++++++++------------ internal/cluster/state_test.go | 258 ++++++++----- internal/cluster/status.go | 222 +++++++++++ internal/cluster/transport.go | 30 +- server/handles/cluster.go | 20 + server/router.go | 1 + 12 files changed, 1944 insertions(+), 685 deletions(-) create mode 100644 docs/cluster-sync.md create mode 100644 internal/cluster/creds.go create mode 100644 internal/cluster/status.go diff --git a/docs/cluster-sync.md b/docs/cluster-sync.md new file mode 100644 index 0000000000..bff539ec73 --- /dev/null +++ b/docs/cluster-sync.md @@ -0,0 +1,401 @@ +# Cluster Storage Config Sync — Deep Dive + +Package: `internal/cluster` +Status: implemented, integration test in progress (2026-06-21) +Journal entry: [2026-06-21 — Cluster Storage Config Sync Plugin](../../JOURNAL.md) + +--- + +## Overview + +The cluster sync plugin lets multiple OpenList nodes securely share storage driver +configurations in real time. Key properties: + +- **Cryptographically authenticated** — pre-shared key (PSK) + per-node ed25519 identity +- **NAT-friendly** — nodes behind NAT dial out; a persistent WebSocket holds the NAT mapping +- **CRDT-consistent** — Last-Writer-Wins, idempotent, tombstone deletes, no churn on identical content +- **Health-gated** — broken / expired tokens never leave the originating node + +--- + +## Package Layout + +```text +internal/cluster/ + crypto.go HKDF key derivation, AES-256-GCM seal/open, ed25519 identity + identity.go Seed persistence (loadOrCreateIdentity) + state.go CRDT store: syncableStorage, record, digest, merge + config.go Config struct, configStore (atomic save), active/peerList/shouldShare + transport.go syncMessage, envelope, sealEnvelope/openEnvelope, replayCache + conn.go peerConn, connRegistry, dialPeer, ServeWS, dialSupervisor, ensureDial + engine.go Manager (Default singleton), Init/Start/Stop, hook wiring, relay logic + *_test.go Unit tests — all passing +``` + +--- + +## Cryptographic Design + +### Key Derivation + +```text +PSK (user-supplied string) + └─► HKDF-SHA256 + salt = "openlist-cluster" + info = "openlist-cluster-aead-v1" + length = 32 bytes + └─► AES-256-GCM AEAD key +``` + +HKDF provides domain separation — the same PSK could be re-used with different `info` +strings for future sub-keys without collisions. + +### Wire Frame + +Every message is wrapped in a sealed envelope: + +```text +[nonce: 12 bytes][GCM ciphertext + tag][AAD: frame type byte] +``` + +- `sealEnvelope(key, plaintext, aad)` → `envelope{Nonce, Ciphertext}` +- `openEnvelope(key, env, aad)` → plaintext or error +- Only PSK holders can seal or open — unauthenticated nodes see only opaque bytes. + +### Replay Protection + +`replayCache` tracks seen nonces within a sliding time window. A frame is rejected if: + +- its nonce has been seen before (replay), or +- its embedded timestamp falls outside the acceptance window (stale / future frame). + +### Node Identity + +Each node generates an ed25519 keypair on first start. The 32-byte seed is persisted at +`/cluster/identity.key` (mode 0600). + +```text +seed (32 B, persisted) + └─► ed25519 keypair (newIdentity / identityFromSeed) + +node_id = base32( sha256(pubkey)[:16] ) — 16 uppercase characters +``` + +Sync records are self-authenticating: + +```text +record.OriginPub — sender's public key (32 bytes) +record.Sig — ed25519 signature over content hash +record.verify() checks: + 1. nodeIDFromPub(OriginPub) == record.Origin + 2. sha256(canonical JSON) matches embedded ContentHash + 3. ed25519.Verify(OriginPub, ContentHash, Sig) +``` + +A node that does not know the PSK cannot open envelopes. A node that knows the PSK but +forges a record will fail `record.verify()` because it cannot sign with the origin node's +private key. + +--- + +## CRDT State Machine + +### Keying Strategy + +The CRDT is keyed by **content hash** — a SHA-256 of the storage configuration's canonical +JSON with identity-mutable fields excluded: + +```text +excluded: ID, Status, Modified +included: driver name, mount path, all driver-specific config fields +``` + +Consequences: + +- Two storages with identical config but different IDs share the same hash — idempotent, + no churn. +- A token refresh changes the config → new hash → a real LWW update is generated and + propagated. +- Renaming a mount path changes the hash — treated as delete + create (tombstone + new + record). + +### Record Lifecycle + +```text +localChange(storage) + ├─ health check: skip if storage.Status == StatusBroken or token appears invalid + ├─ compute contentHash + ├─ if store already has same hash at same-or-higher Lamport clock → no-op (idempotent) + └─ create signed record, bump Lamport clock, insert into store + +localDelete(storage) + └─ create tombstone record (Deleted=true), signed, bump clock, insert + +merge(remote record) + ├─ verify signature + ├─ if local has same contentHash at higher-or-equal clock → discard (dominates) + ├─ if tombstone + local has live record at lower clock → apply tombstone + └─ otherwise apply: insert/update store entry +``` + +`record.dominates(other)` implements the LWW tiebreak: higher Lamport clock wins; equal +clocks tiebreak by node_id lexicographic order (deterministic, no coin flip needed). + +### Persistence + +`state.json` in `/cluster/` is written atomically (tmp file + rename) on every +change. `snapshot()` serialises the full store. `load()` re-hydrates on startup, so +nodes survive restarts without losing sync state. + +--- + +## Transport Layer + +### Design Choice: Persistent WebSocket over Stateless HTTP + +Stateless HTTP POST was considered and rejected: it cannot reach nodes behind NAT because +the caller must initiate — a node behind NAT has no publicly reachable address for peers +to POST to. + +**Persistent WebSocket** solves this: the NAT'd node dials out, the TCP connection is +held open, and the remote peer can push frames back at any time over the same connection. +Ping keepalives prevent the NAT mapping from timing out. + +### Connection Model + +```text +┌──────────────┐ ┌──────────────┐ +│ NAT'd node │ ── WS dial ──────────► │ Reachable B │ +│ │ ◄─────────── push/relay ─ │ +└──────────────┘ └──────┬───────┘ + │ relay + ┌──────▼───────┐ + │ Reachable C │ + └──────────────┘ +``` + +- `dialPeer(url)` — dials and blocks until the connection closes, then returns so + `dialSupervisor` can retry with exponential backoff. +- `ServeWS(c *gin.Context)` — upgrades an HTTP request to WebSocket and registers the + connection in `connRegistry`. +- `ensureDial(peer)` — idempotent: if a dial goroutine is already running for `peer`, + does nothing. +- `relay(records, excludeConn)` — after `applyRecords` returns the set of newly applied + records, `relay` forwards them to all other connected peers (excluding the sender to + prevent echo). + +### Keepalive Constants + +| Constant | Value | Purpose | +|---|---|---| +| `pingPeriod` | 54 s | How often the server sends a WS ping frame | +| `pongWait` | 60 s | How long to wait for a pong before closing | +| `writeWait` | 10 s | Deadline for a single write operation | +| `dialBackoffMin` | 2 s | Initial retry delay after dial failure | +| `dialBackoffMax` | 5 min | Maximum retry delay (exponential cap) | +| `sendQueueLen` | 128 | Per-connection outbound channel depth | +| `maxFrameBytes` | 16 MB | Maximum inbound frame size | + +### Connection Registry + +`connRegistry` is a goroutine-safe map of active `*peerConn` entries, keyed by connection +ID. A node_id is bound to a connection once identity is established during the WS +handshake (`bind`). `hasNode(nodeID)` lets the engine avoid duplicate dials. + +### URL Normalisation + +`wsURL(rawURL, path)` converts HTTP base URLs to WebSocket URLs: + +```text +http://host/... → ws://host/api/cluster/ws +https://host/... → wss://host/api/cluster/ws +``` + +--- + +## Engine (Manager) + +`Manager` is the central coordinator. `Default` is the package-level singleton initialised +by `InitClusterSync()` in `internal/bootstrap/cluster.go`. + +### Lifecycle + +```text +InitClusterSync() called during Init(), after InitPlugins() + └─ Manager.Init(dataDir) + ├─ loadOrCreateIdentity + ├─ load configStore + └─ load state.json (if present) + +StartClusterSync() called during Start(), after LoadStorages() + └─ Manager.Start(ctx) + ├─ seedLocalStorages() — feed local storages into CRDT on first start + ├─ announceLoop() — periodic full-state announce to all peers + ├─ dialSupervisor() — maintain outbound WS connections to configured peers + └─ register storage hooks +``` + +### Storage Hooks + +Two hooks wire the engine into the storage lifecycle: + +| Hook event | Source | Engine action | +|---|---|---| +| `"update"` | `saveDriverStorage` (after token refresh) | `onStorageHook` → `localChange` → broadcast | +| `"token-invalid"` | `NotifyStorageTokenInvalid` | `onStorageHook` → skip propagation (health gate) | + +The hook fires as a goroutine (`go callStorageHooks(...)`) so it never blocks the caller's +hot path. + +### Announce Loop + +Every `AnnounceIntervalSec` (default 30 s) the engine broadcasts a digest summary of its +CRDT store to all peers. Peers that have diverged request missing records; peers that are +up to date discard the digest without generating traffic. This provides eventual consistency +for nodes that were offline during a change. + +### Apply + Relay + +```go +func (m *Manager) applyRecords(records []record) []record +``` + +- Verifies each record's signature. +- Calls `store.merge()` for each. +- Returns the subset of records that were actually applied (i.e., advanced the CRDT state). +- The caller (`handleFrame`) passes the applied set to `relay()`. + +--- + +## API Endpoints + +Registered in `server/router.go`: + +| Method | Path | Handler | Auth | +|---|---|---|---| +| `GET` | `/api/cluster/ws` | `ClusterWS` | API key (peer-to-peer) | +| `GET` | `/api/admin/cluster/config` | `ClusterGetConfig` | Admin | +| `POST` | `/api/admin/cluster/config` | `ClusterSetConfig` | Admin | +| `GET` | `/api/admin/cluster/status` | `ClusterStatus` | Admin | + +`ClusterSetConfig` calls `Manager.SetConfig` which atomically saves the new config and +calls `connRegistry.closeAll()` — existing connections drop and `dialSupervisor` reconnects +with the new peer list. + +--- + +## Configuration Reference + +Persisted at `/cluster/config.json`. Editable via the admin UI or directly as JSON. + +```json +{ + "enabled": true, + "key": "", + "peers": ["https://peer-b.example.com"], + "share_drivers": [], + "share_mounts": ["/cluster-test"], + "share_deletes": false, + "apply_remote": true, + "announce_interval_sec": 30, + "request_timeout_sec": 10 +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `enabled` | bool | Master switch — `false` disables all sync activity | +| `key` | string | Pre-shared key; all cluster members must use the same value | +| `peers` | `[]string` | Base URLs of peer nodes this node dials out to | +| `share_drivers` | `[]string` | Only share storages using these driver names (empty = all) | +| `share_mounts` | `[]string` | Only share storages whose mount path matches a prefix (empty = all) | +| `share_deletes` | bool | Whether to propagate tombstone (delete) records | +| `apply_remote` | bool | Whether to apply incoming records to the local database | +| `announce_interval_sec` | int | Seconds between periodic full-state announce broadcasts | +| `request_timeout_sec` | int | HTTP / WS dial timeout | + +`active()`, `peerList()`, and `shouldShare()` are **pointer receivers** on `*Config` to +ensure callers always read the live configuration after a `SetConfig` call. + +--- + +## Frontend UI (`ClusterConfig.tsx`) + +Located at `src/pages/manage/plugins/ClusterConfig.tsx`, rendered inside the Plugins +management page. + +Sections: + +1. **Enable switch** — master on/off toggle +2. **Pre-shared Key** — password field (masked) +3. **Peers** — textarea, one URL per line +4. **Scope** — `share_drivers` (comma-separated), `share_mounts` (comma-separated) +5. **Behaviour** — `apply_remote`, `share_deletes` switches; interval number inputs +6. **Live Status** — polls `/api/admin/cluster/status`; shows `node_id`, connected peer + count, and number of synced records + +--- + +## Operational Notes + +### Data Files + +```text +/cluster/ + identity.key 32-byte ed25519 seed (mode 0600, never share) + config.json cluster configuration + state.json CRDT store snapshot (rebuilt on startup if missing) +``` + +### Safe Scoping for Testing + +Set `share_mounts` to a single test path (e.g. `/cluster-test`) so that production +storages are never touched by the sync mechanism. + +### NAT'd Node Setup + +A NAT'd node only needs to list reachable peers in `peers`. It dials out on startup and +after each disconnect. It does not need an open inbound port. + +### Token Refresh Propagation + +When `saveDriverStorage` is called after a token refresh, the `"update"` hook fires +asynchronously. The engine calls `localChange` (health-checked) and, if the content hash +changed, broadcasts to all connected peers. Peers receiving the update call `applyToLocal` +→ `op.UpdateStorage`, which persists the fresh token without triggering another hook cycle +(the content hash will be identical on the receiving side after apply). + +### Avoiding Churn + +The idempotency guarantee means that if node A and node B both store the same token +(same content hash), neither will generate a CRDT update for the other. The announce +digest handshake confirms they agree; no record is transmitted. + +### Broken Token Gate + +`localChange` checks storage health before accepting a record into the CRDT store. If the +storage status is `StatusBroken`, or if the token fields appear invalid, the record is +silently dropped. This prevents a node that has entered a broken-token state from +poisoning healthy peers. + +--- + +## Testing + +| File | Coverage | +|---|---| +| `crypto_test.go` | AEAD round-trip, wrong key, tampered AAD, identity binding | +| `state_test.go` | LWW merge, idempotency (same content hash), tombstone, Lamport ordering | +| `config_test.go` | `active()` / `peerList()` pointer-receiver correctness, `shouldShare()` filter | +| `transport_test.go` | Envelope replay rejection, stale-timestamp rejection | +| `conn_test.go` | `wsURL` conversion (http→ws, https→wss), `connRegistry` add/bind/remove/closeAll | + +All tests pass as of rev `2c8a9827`. + +--- + +## Related Docs + +- [streaming-and-caching.md](streaming-and-caching.md) — RangeReader, SeekableStream pitfalls +- [new-apis-2026-06-21.md](new-apis-2026-06-21.md) — storage loading progress, permission bit 16, video_play, favorites +- [architecture.md](architecture.md) — driver system, request flow, packages, startup sequence diff --git a/internal/cluster/config.go b/internal/cluster/config.go index a5e121e96c..df755da135 100644 --- a/internal/cluster/config.go +++ b/internal/cluster/config.go @@ -8,10 +8,17 @@ import ( "sync" ) -// Config is the operator-facing configuration of the storage-sharing cluster. +// Config is the operator-facing configuration of the credential-sharing cluster. // It is persisted as JSON under /cluster/config.json and edited through the // admin API (and the plugins page UI). Keeping it in its own file rather than the // global settings table keeps the feature self-contained and pluggable. +// +// The redesign narrows sharing to CREDENTIALS only (tokens/cookies/secrets) and +// routes them through explicit, manually-defined sync groups (see groups.go). +// Peer membership is auto-discovered: any node that can open our AEAD envelope is +// admitted, public nodes advertise their address, and the rest is learned via +// peer exchange. The only manual knobs left are the shared key, an optional +// self-advertised address, and optional bootstrap seeds for the first join. type Config struct { // Enabled turns the whole cluster sync on/off. Enabled bool `json:"enabled"` @@ -19,84 +26,65 @@ type Config struct { // exact same value; it is the sole secret that authenticates membership and // encrypts traffic. Empty key => sync stays disabled. Key string `json:"key"` - // Peers are the base URLs of the other nodes, e.g. "https://node2.example.com". - // The cluster endpoints are reached at /api/cluster/... - Peers []string `json:"peers"` - // ShareDrivers, if non-empty, limits sharing to storages of these driver - // types (e.g. ["115 Cloud","BaiduNetdisk"]). Empty => share every driver. - ShareDrivers []string `json:"share_drivers"` - // ShareMounts, if non-empty, limits sharing to these exact mount paths. - // Empty => no mount-path restriction. - ShareMounts []string `json:"share_mounts"` - // ShareDeletes, when true, propagates storage deletions to peers as - // tombstones. Default false: deleting a mount on one node must NOT silently - // wipe it cluster-wide — sharing is about credentials/config, not lifecycle. - ShareDeletes bool `json:"share_deletes"` - // ApplyRemote, when true, lets incoming peer configs create/update local - // storages. Default (false) makes a node share-only/observe; set true on - // nodes that should adopt peer credentials. Most deployments want this true. + // Label is a human-friendly name for THIS node shown in the cluster UI of all + // nodes (e.g. "cfscan", "home-nas"). Falls back to the node id when empty. + Label string `json:"label"` + // Addr is this node's externally reachable base URL (e.g. + // "https://node1.example.com"). Advertising it lets other nodes auto-discover + // and dial this node. Leave empty for nodes behind NAT — they dial out and are + // reached through a connected public node's relay instead. + Addr string `json:"addr"` + // Seeds are optional bootstrap base URLs to dial when joining an existing + // cluster. Only needed once: after the first connection, every reachable + // node's address is learned automatically via peer exchange. + Seeds []string `json:"seeds"` + // ApplyRemote, when true (the default for a credential cluster), lets incoming + // peer credentials update local storages in a shared group. Set false to make + // a node observe/share-only. ApplyRemote bool `json:"apply_remote"` - // AnnounceIntervalSec is how often this node broadcasts its digest so peers - // can pull anything they missed. 0 => default. + // AnnounceIntervalSec is how often this node re-broadcasts its inventory + + // credential digests so peers converge after any lost message. 0 => default. AnnounceIntervalSec int `json:"announce_interval_sec"` - // RequestTimeoutSec bounds each outbound peer HTTP request. 0 => default. - RequestTimeoutSec int `json:"request_timeout_sec"` } const ( - defaultAnnounceIntervalSec = 60 - defaultRequestTimeoutSec = 15 + defaultAnnounceIntervalSec = 45 ) -func (c *Config) announceInterval() int { +func (c Config) announceInterval() int { if c.AnnounceIntervalSec <= 0 { return defaultAnnounceIntervalSec } return c.AnnounceIntervalSec } -func (c *Config) requestTimeout() int { - if c.RequestTimeoutSec <= 0 { - return defaultRequestTimeoutSec - } - return c.RequestTimeoutSec +// active reports whether sync should actually run: enabled with a key. Unlike the +// previous design it does NOT require a configured peer — a reachable node can run +// purely accept-only and still serve a whole cluster. +func (c Config) active() bool { + return c.Enabled && strings.TrimSpace(c.Key) != "" } -// active reports whether sync should actually run: enabled, with a key and at -// least one peer. -func (c *Config) active() bool { - return c.Enabled && strings.TrimSpace(c.Key) != "" && len(c.peerList()) > 0 +// seedList returns the trimmed, non-empty bootstrap URLs. +func (c Config) seedList() []string { + return cleanURLs(c.Seeds) } -// peerList returns the trimmed, non-empty peer URLs. -func (c *Config) peerList() []string { - out := make([]string, 0, len(c.Peers)) - for _, p := range c.Peers { - if p = strings.TrimRight(strings.TrimSpace(p), "/"); p != "" { - out = append(out, p) +func cleanURLs(in []string) []string { + out := make([]string, 0, len(in)) + seen := make(map[string]struct{}, len(in)) + for _, p := range in { + p = strings.TrimRight(strings.TrimSpace(p), "/") + if p == "" { + continue } - } - return out -} - -// shouldShare decides whether a storage of the given driver/mount is in scope. -func (c *Config) shouldShare(driver, mountPath string) bool { - if len(c.ShareDrivers) > 0 && !contains(c.ShareDrivers, driver) { - return false - } - if len(c.ShareMounts) > 0 && !contains(c.ShareMounts, mountPath) { - return false - } - return true -} - -func contains(haystack []string, needle string) bool { - for _, h := range haystack { - if strings.EqualFold(strings.TrimSpace(h), needle) { - return true + if _, dup := seen[p]; dup { + continue } + seen[p] = struct{}{} + out = append(out, p) } - return false + return out } // configStore handles persistence of Config under a directory. @@ -112,14 +100,15 @@ func newConfigStore(dir string) *configStore { func (cs *configStore) path() string { return filepath.Join(cs.dir, "config.json") } -// loadOrInit reads config.json, or returns a zero (disabled) config if absent. +// loadOrInit reads config.json, or returns a default (disabled, apply-remote-on) +// config if absent. func (cs *configStore) loadOrInit() (Config, error) { cs.mu.Lock() defer cs.mu.Unlock() b, err := os.ReadFile(cs.path()) if err != nil { if os.IsNotExist(err) { - cs.cfg = Config{} + cs.cfg = Config{ApplyRemote: true} return cs.cfg, nil } return Config{}, err diff --git a/internal/cluster/config_test.go b/internal/cluster/config_test.go index 13327f64bd..2105246887 100644 --- a/internal/cluster/config_test.go +++ b/internal/cluster/config_test.go @@ -8,11 +8,12 @@ func TestConfigActive(t *testing.T) { cfg Config want bool }{ - {"disabled", Config{Enabled: false, Key: "k", Peers: []string{"http://p"}}, false}, - {"no key", Config{Enabled: true, Peers: []string{"http://p"}}, false}, - {"no peers", Config{Enabled: true, Key: "k"}, false}, - {"blank peers only", Config{Enabled: true, Key: "k", Peers: []string{" ", "/"}}, false}, - {"active", Config{Enabled: true, Key: "k", Peers: []string{"http://p"}}, true}, + {"disabled", Config{Enabled: false, Key: "k"}, false}, + {"no key", Config{Enabled: true}, false}, + {"blank key", Config{Enabled: true, Key: " "}, false}, + // A reachable node needs no peer/seed to be active — it can accept-only. + {"active no seed", Config{Enabled: true, Key: "k"}, true}, + {"active with seed", Config{Enabled: true, Key: "k", Seeds: []string{"http://p"}}, true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -23,29 +24,19 @@ func TestConfigActive(t *testing.T) { } } -func TestConfigShouldShare(t *testing.T) { - cfg := Config{ShareDrivers: []string{"115 Open"}, ShareMounts: []string{"/115"}} - if !cfg.shouldShare("115 Open", "/115") { - t.Fatal("in-scope storage should be shared") - } - if cfg.shouldShare("Local", "/115") { - t.Fatal("driver out of filter must not be shared") - } - if cfg.shouldShare("115 Open", "/other") { - t.Fatal("mount out of filter must not be shared") - } - - // Empty filters => share everything. - open := Config{} - if !open.shouldShare("AnyDriver", "/anywhere") { - t.Fatal("empty filters should share everything") +func TestConfigSeedListNormalizes(t *testing.T) { + cfg := Config{Seeds: []string{" http://a/ ", "http://b", "", " ", "http://a"}} + got := cfg.seedList() + if len(got) != 2 || got[0] != "http://a" || got[1] != "http://b" { + t.Fatalf("seedList normalize/dedup failed: %#v", got) } } -func TestConfigPeerListNormalizes(t *testing.T) { - cfg := Config{Peers: []string{" http://a/ ", "http://b", "", " "}} - got := cfg.peerList() - if len(got) != 2 || got[0] != "http://a" || got[1] != "http://b" { - t.Fatalf("peerList normalize failed: %#v", got) +func TestConfigAnnounceIntervalDefault(t *testing.T) { + if (Config{}).announceInterval() != defaultAnnounceIntervalSec { + t.Fatal("zero interval should fall back to default") + } + if (Config{AnnounceIntervalSec: 10}).announceInterval() != 10 { + t.Fatal("explicit interval should be honored") } } diff --git a/internal/cluster/conn.go b/internal/cluster/conn.go index 65059855cc..2a6f43282a 100644 --- a/internal/cluster/conn.go +++ b/internal/cluster/conn.go @@ -44,17 +44,21 @@ type peerConn struct { ws *websocket.Conn send chan []byte outbound bool // true if we dialed it + addr string // base URL we dialed (outbound only); "" for inbound + since int64 // unix seconds the connection was established nodeID string // learned from the first authenticated frame closeOnce sync.Once closed chan struct{} } -func newPeerConn(mgr *Manager, ws *websocket.Conn, outbound bool) *peerConn { +func newPeerConn(mgr *Manager, ws *websocket.Conn, outbound bool, addr string) *peerConn { return &peerConn{ mgr: mgr, ws: ws, send: make(chan []byte, sendQueueLen), outbound: outbound, + addr: addr, + since: now(), closed: make(chan struct{}), } } @@ -208,31 +212,34 @@ func wsURL(peer string) string { return peer + wsPath } -// startConn registers a connection, starts its pumps, and greets the peer with -// our digests so anti-entropy begins immediately. +// startConn registers a connection, starts its pumps, and greets the peer with a +// hello (our inventory + known peer addresses for PEX + groups doc + credential +// digests) so discovery and anti-entropy begin immediately. func (m *Manager) startConn(c *peerConn) { m.conns.add(c) go c.writePump() go c.readPump() - m.sendTo(c, &syncMessage{Type: "announce", Digests: m.state.digests()}) + m.sendTo(c, m.helloMessage()) } // dialPeer opens an outbound persistent connection to a peer and blocks until it // closes. Dialing OUT is what lets a NAT'd node participate without being // reachable itself. -func (m *Manager) dialPeer(peer string) error { - ws, _, err := websocket.DefaultDialer.Dial(wsURL(peer), nil) +func (m *Manager) dialPeer(addr string) error { + ws, _, err := websocket.DefaultDialer.Dial(wsURL(addr), nil) if err != nil { return err } - c := newPeerConn(m, ws, true) + c := newPeerConn(m, ws, true, addr) m.startConn(c) <-c.closed return nil } // ServeWS upgrades an inbound HTTP request to a persistent connection. Auth is -// deferred to the first sealed frame (PSK), so the upgrade itself is open. +// deferred to the first sealed frame (PSK), so the upgrade itself is open. Any +// node that can open our AEAD envelope is admitted as a peer — this is the +// "authenticated remotes auto-join" half of discovery. func (m *Manager) ServeWS(w http.ResponseWriter, r *http.Request) { cfg := m.cfgStore.get() if !cfg.active() { @@ -243,16 +250,21 @@ func (m *Manager) ServeWS(w http.ResponseWriter, r *http.Request) { if err != nil { return } - m.startConn(newPeerConn(m, ws, false)) + m.startConn(newPeerConn(m, ws, false, "")) } -// dialSupervisor keeps one live outbound connection to every configured peer, -// re-dialing as connections drop or config changes. +// dialSupervisor keeps live outbound connections to bootstrap seeds and to every +// auto-discovered peer address we are not already connected to, re-dialing as +// connections drop. Seeds bootstrap the first join; learned addresses (from peer +// inventory exchange) keep public nodes meshed without manual config. func (m *Manager) dialSupervisor() { for { if cfg := m.cfgStore.get(); cfg.active() { - for _, peer := range cfg.peerList() { - m.ensureDial(peer) + for _, seed := range cfg.seedList() { + m.ensureDial(seed) + } + for _, addr := range m.discoveredDialTargets() { + m.ensureDial(addr) } } select { @@ -263,25 +275,55 @@ func (m *Manager) dialSupervisor() { } } +// discoveredDialTargets returns advertised peer addresses we should dial: those +// belonging to nodes we do not already have a live connection to, and that are +// not our own advertised address. +func (m *Manager) discoveredDialTargets() []string { + self := m.cfgStore.get().Addr + self = trimURL(self) + var out []string + for _, n := range m.state.inventoryList() { + if n.NodeID == m.id.NodeID || n.Addr == "" { + continue + } + if trimURL(n.Addr) == self && self != "" { + continue + } + if m.conns.hasNode(n.NodeID) { + continue + } + out = append(out, n.Addr) + } + return out +} + +func trimURL(s string) string { + return strings.TrimRight(strings.TrimSpace(s), "/") +} + // ensureDial starts (at most one) outbound dial loop for a peer URL. dialPeer // blocks for the connection's lifetime, so the dialing flag also prevents a // duplicate link while connected. -func (m *Manager) ensureDial(peer string) { +func (m *Manager) ensureDial(addr string) { + addr = trimURL(addr) + if addr == "" { + return + } m.dialMu.Lock() - if m.dialing[peer] { + if m.dialing[addr] { m.dialMu.Unlock() return } - m.dialing[peer] = true + m.dialing[addr] = true m.dialMu.Unlock() go func() { defer func() { m.dialMu.Lock() - delete(m.dialing, peer) + delete(m.dialing, addr) m.dialMu.Unlock() }() - if err := m.dialPeer(peer); err != nil { - utils.Log.Debugf("[cluster] dial %s failed: %v", peer, err) + if err := m.dialPeer(addr); err != nil { + utils.Log.Debugf("[cluster] dial %s failed: %v", addr, err) select { case <-m.stopCh: case <-time.After(dialBackoffMin): diff --git a/internal/cluster/creds.go b/internal/cluster/creds.go new file mode 100644 index 0000000000..14fcd50262 --- /dev/null +++ b/internal/cluster/creds.go @@ -0,0 +1,180 @@ +package cluster + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "strings" +) + +// This file isolates the one thing the cluster actually replicates: a storage's +// CREDENTIAL fields. The user requirement is explicit — "除了 key 其他的都不用同步" +// (sync only the key/credential, nothing else). So instead of shipping a whole +// storage config we extract just the secret-bearing subset of a driver's Addition +// JSON, propagate that, and overlay it onto the matching storage on peer nodes, +// leaving every node-local field (mount path, root folder, cache, order, proxy…) +// untouched. +// +// OpenList drivers do not tag credential fields, so we identify them by name. The +// keyword set covers the credential fields used across drivers (refresh_token, +// access_token, cookie, password, *_secret, api_key, authorization, …). Matching +// is done on a normalized (lowercased, separator-stripped) field name so +// "refresh_token", "RefreshToken" and "refreshToken" all match. The match runs +// only over Addition keys, where non-credential fields are structural +// (root_folder_id, order_by, …) and never contain these tokens — so false +// positives are not a concern in practice. +var credKeywords = []string{ + "token", // access_token, refresh_token, token + "cookie", // cookie / cookies + "password", // password + "passwd", // passwd + "secret", // client_secret, app_secret, secret_key + "auth", // authorization, auth + "session", // session, session_id + "refresh", // refresh_token (redundant w/ token, kept for clarity) + "access", // access_token (redundant w/ token) + "credential", // credential / credentials + "apikey", // api_key, apikey + "appkey", // app_key + "privatekey", // private_key + "signkey", // sign_key + "ticket", // some drivers use a login ticket + "passport", // 123pan-style passport secret +} + +// normalizeKey lowercases and strips separators so "Refresh_Token" == "refreshtoken". +func normalizeKey(name string) string { + var b strings.Builder + for _, r := range name { + switch { + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): + b.WriteRune(r) + // drop '_', '-', spaces, etc. + } + } + return b.String() +} + +// isCredField reports whether an Addition field name denotes a credential. +func isCredField(name string) bool { + n := normalizeKey(name) + if n == "" { + return false + } + for _, k := range credKeywords { + if strings.Contains(n, k) { + return true + } + } + return false +} + +// extractCreds parses a storage Addition JSON blob and returns only its +// credential fields, in a canonical map keyed by the ORIGINAL JSON key (so they +// can be overlaid back verbatim on a peer). Returns an empty map for an empty or +// unparseable blob. +func extractCreds(additionJSON string) map[string]json.RawMessage { + out := map[string]json.RawMessage{} + if strings.TrimSpace(additionJSON) == "" { + return out + } + var all map[string]json.RawMessage + if err := json.Unmarshal([]byte(additionJSON), &all); err != nil { + return out + } + for k, v := range all { + if isCredField(k) { + out[k] = v + } + } + return out +} + +// credFieldNames returns the sorted credential field names present in a payload. +func credFieldNames(creds map[string]json.RawMessage) []string { + names := make([]string, 0, len(creds)) + for k := range creds { + names = append(names, k) + } + sort.Strings(names) + return names +} + +// canonicalCreds serializes a credential map deterministically (keys sorted) so +// the same credentials always hash identically on every node. +func canonicalCreds(creds map[string]json.RawMessage) []byte { + names := credFieldNames(creds) + var b strings.Builder + b.WriteByte('{') + for i, k := range names { + if i > 0 { + b.WriteByte(',') + } + kb, _ := json.Marshal(k) + b.Write(kb) + b.WriteByte(':') + b.Write(creds[k]) + } + b.WriteByte('}') + return []byte(b.String()) +} + +// credHash is the idempotency key: identical credentials → identical hash on +// every node. An empty credential set hashes to "" so it is never propagated +// (the engine treats "" as "nothing to share / nothing learned"). +func credHash(creds map[string]json.RawMessage) string { + if len(creds) == 0 { + return "" + } + sum := sha256.Sum256(canonicalCreds(creds)) + return hex.EncodeToString(sum[:]) +} + +// applyCreds overlays credential fields onto an existing Addition JSON blob and +// returns the updated blob. Only the keys present in creds are replaced; every +// other field of the target Addition is preserved exactly. Returns (json,true) +// when the blob actually changed, (original,false) when it was already identical +// (so the caller can skip a needless re-init / avoid churn). +func applyCreds(additionJSON string, creds map[string]json.RawMessage) (string, bool) { + if len(creds) == 0 { + return additionJSON, false + } + all := map[string]json.RawMessage{} + if strings.TrimSpace(additionJSON) != "" { + if err := json.Unmarshal([]byte(additionJSON), &all); err != nil { + // Unparseable target: rebuild from creds alone rather than corrupt it. + all = map[string]json.RawMessage{} + } + } + changed := false + for k, v := range creds { + if cur, ok := all[k]; !ok || !jsonEqual(cur, v) { + all[k] = v + changed = true + } + } + if !changed { + return additionJSON, false + } + // Marshal with sorted keys for stability (encoding/json already sorts map keys). + b, err := json.Marshal(all) + if err != nil { + return additionJSON, false + } + return string(b), true +} + +// jsonEqual compares two raw JSON values semantically (after re-normalizing) so +// whitespace differences do not count as a change. +func jsonEqual(a, b json.RawMessage) bool { + var av, bv interface{} + if json.Unmarshal(a, &av) != nil || json.Unmarshal(b, &bv) != nil { + return string(a) == string(b) + } + an, _ := json.Marshal(av) + bn, _ := json.Marshal(bv) + return string(an) == string(bn) +} diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 9f1d7514c8..1070af6544 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -3,13 +3,14 @@ package cluster import ( "context" "encoding/json" + "fmt" "os" "path/filepath" + "sort" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -19,12 +20,16 @@ const ( replayWindowSec = 300 // wsPath is the persistent-connection endpoint peers dial. wsPath = "/api/cluster/ws" - // dialReconcile is how often the dial supervisor re-checks that every - // configured peer has a live outbound connection. + // dialReconcile is how often the dial supervisor re-checks connectivity. dialReconcile = 5 * time.Second + // peerLivenessSec is how long a node is considered online after last contact + // (absent a live connection). + peerLivenessSec = 130 + // maxEvents bounds the in-memory activity log surfaced to the UI. + maxEvents = 60 ) -// Manager is the running cluster-sync engine for this node. +// Manager is the running cluster credential-sync engine for this node. type Manager struct { dir string id *identity @@ -38,6 +43,12 @@ type Manager struct { dialMu sync.Mutex dialing map[string]bool // peer URLs with an in-flight/live outbound dial + invMu sync.Mutex + lastSelfVer uint64 // monotonic version for our own inventory entry + + evMu sync.Mutex + events []EventView + stopCh chan struct{} once sync.Once } @@ -48,10 +59,9 @@ var Default *Manager func now() int64 { return time.Now().Unix() } -// Init constructs the manager: loads identity + config + persisted CRDT state and +// Init constructs the manager: loads identity + config + persisted state and // registers the storage hook so local credential changes propagate. It does NOT -// start the network loops — call Start once storages are loaded. Safe to call -// even when the feature is disabled (it simply stays dormant). +// start the network loops — call Start once storages are loaded. func Init(dataDir string) (*Manager, error) { dir := filepath.Join(dataDir, "cluster") id, err := loadOrCreateIdentity(dir) @@ -81,59 +91,8 @@ func Init(dataDir string) (*Manager, error) { // NodeID returns this node's stable identity string. func (m *Manager) NodeID() string { return m.id.NodeID } -// RecordView is a redaction-safe summary of one synced mount for the admin UI. -// It deliberately omits the Addition (which holds secrets/tokens). -type RecordView struct { - MountPath string `json:"mount_path"` - Driver string `json:"driver"` - Version uint64 `json:"version"` - Origin string `json:"origin"` - Tombstone bool `json:"tombstone"` - UpdatedAt int64 `json:"updated_at"` - Self bool `json:"self"` // true if this node authored the current version -} - -// Status is the admin overview of the cluster state. -type Status struct { - NodeID string `json:"node_id"` - Enabled bool `json:"enabled"` - Active bool `json:"active"` - Peers []string `json:"peers"` - Records []RecordView `json:"records"` -} - -// Status returns a redaction-safe snapshot for the admin UI. -func (m *Manager) Status() Status { - cfg := m.cfgStore.get() - recs := m.state.snapshot() - views := make([]RecordView, 0, len(recs)) - for _, r := range recs { - views = append(views, RecordView{ - MountPath: r.MountPath, - Driver: r.Config.Driver, - Version: r.Version, - Origin: r.Origin, - Tombstone: r.Tombstone, - UpdatedAt: r.UpdatedAt, - Self: r.Origin == m.id.NodeID, - }) - } - return Status{ - NodeID: m.id.NodeID, - Enabled: cfg.Enabled, - Active: cfg.active(), - Peers: cfg.peerList(), - Records: views, - } -} - // ---- persistence ---- -type persistedState struct { - Lamport uint64 `json:"lamport"` - Records []*record `json:"records"` -} - func (m *Manager) statePath() string { return filepath.Join(m.dir, "state.json") } func (m *Manager) loadState() { @@ -146,13 +105,19 @@ func (m *Manager) loadState() { utils.Log.Warnf("[cluster] corrupt state file, ignoring: %v", err) return } - m.state.load(ps.Records, ps.Lamport) + m.state.load(ps) + // resume our self-inventory version so peers never see us go backwards. + for _, n := range ps.Inventory { + if n.NodeID == m.id.NodeID && n.Version > m.lastSelfVer { + m.lastSelfVer = n.Version + } + } } func (m *Manager) persist() { m.persistMu.Lock() defer m.persistMu.Unlock() - ps := persistedState{Lamport: m.state.lamportNow(), Records: m.state.snapshot()} + ps := m.state.export() b, err := json.MarshalIndent(ps, "", " ") if err != nil { return @@ -186,24 +151,65 @@ func (m *Manager) SetConfig(c Config) error { if c.Key == "" || c.Key == "********" { c.Key = old.Key } + c.Seeds = cleanURLs(c.Seeds) + c.Addr = trimURL(c.Addr) if err := m.cfgStore.save(c); err != nil { return err } - // Drop all live connections so they reconnect with the new settings (key, - // peers, ...); the dial supervisor re-dials the configured peers. + // Drop all live connections so they reconnect with the new settings. m.conns.closeAll() - // Seed records for any newly in-scope local storages so we announce them. - go m.seedLocalStorages() + go m.refreshInventory() + go m.seedLocalCreds() + return nil +} + +// GroupSpec / MemberSpec are the exported, API-facing shapes for editing groups. +type MemberSpec struct { + NodeID string `json:"node_id"` + MountPath string `json:"mount_path"` +} + +type GroupSpec struct { + ID string `json:"id"` + Name string `json:"name"` + Members []MemberSpec `json:"members"` +} + +// SetGroups replaces the cluster-shared sync-group document (admin edit) and +// propagates it. It then re-evaluates local credentials against the new groups. +func (m *Manager) SetGroups(specs []GroupSpec) error { + groups := make([]group, 0, len(specs)) + for _, s := range specs { + g := group{ID: s.ID, Name: s.Name} + for _, ms := range s.Members { + if ms.NodeID == "" || ms.MountPath == "" { + continue + } + g.Members = append(g.Members, member{NodeID: ms.NodeID, MountPath: ms.MountPath}) + } + if g.ID == "" || len(g.Members) == 0 { + continue + } + groups = append(groups, g) + } + d := m.state.setGroups(m.id, groups, now()) + m.state.pruneCreds() + m.persist() + m.recordEvent("groups", "", fmt.Sprintf("updated to %d group(s)", len(groups))) + gd := d + m.broadcast(&syncMessage{Type: "push", Groups: &gd}) + go m.seedLocalCreds() return nil } // ---- lifecycle ---- -// Start seeds records from local storages and launches the connection dialer and -// anti-entropy loop. +// Start seeds inventory + credentials from local storages and launches the +// connection dialer and anti-entropy loop. func (m *Manager) Start() { m.once.Do(func() { - go m.seedLocalStorages() + m.state.setLocalInventory(m.selfNodeInfo()) + go m.seedLocalCreds() go m.announceLoop() go m.dialSupervisor() }) @@ -219,32 +225,93 @@ func (m *Manager) Stop() { m.conns.closeAll() } -// seedLocalStorages records the current healthy, in-scope local storages as CRDT -// records (idempotent: unchanged configs cause no version churn) so this node has -// something to announce/serve. -func (m *Manager) seedLocalStorages() { +// ---- self inventory ---- + +// selfNodeInfo builds this node's inventory entry from currently-loaded storages. +func (m *Manager) selfNodeInfo() *nodeInfo { cfg := m.cfgStore.get() - if !cfg.active() { - return - } - var changed bool + var sts []storageInfo for _, d := range op.GetAllStorages() { st := d.GetStorage() - if st.Status != op.WORK || !cfg.shouldShare(st.Driver, st.MountPath) { + sts = append(sts, storageInfo{MountPath: st.MountPath, Driver: st.Driver, Status: st.Status}) + } + sort.Slice(sts, func(i, j int) bool { return sts[i].MountPath < sts[j].MountPath }) + + m.invMu.Lock() + v := uint64(now()) + if v <= m.lastSelfVer { + v = m.lastSelfVer + 1 + } + m.lastSelfVer = v + m.invMu.Unlock() + + return &nodeInfo{ + NodeID: m.id.NodeID, + Label: cfg.Label, + Addr: trimURL(cfg.Addr), + Storages: sts, + Version: v, + UpdatedAt: now(), + } +} + +// refreshInventory rebuilds and broadcasts our inventory entry (call after a +// storage is added/removed or its status changes). +func (m *Manager) refreshInventory() { + if !m.cfgStore.get().active() { + return + } + self := m.selfNodeInfo() + m.state.setLocalInventory(self) + m.broadcast(m.announceMessage()) +} + +// knownNodesForPEX returns inventory entries that advertise a dialable address, +// so peers can auto-discover the rest of the mesh. +func (m *Manager) knownNodesForPEX() []*nodeInfo { + var out []*nodeInfo + for _, n := range m.state.inventoryList() { + if n.Addr == "" { continue } - if _, ok := m.state.localChange(m.id, fromModel(st), now()); ok { - changed = true - } + cp := n + out = append(out, &cp) } - if changed { - m.persist() - m.broadcast(&syncMessage{Type: "announce", Digests: m.state.digests()}) + return out +} + +// helloMessage greets a freshly-connected peer with everything needed to +// converge: our inventory, known dialable peers (PEX), the groups doc, and our +// credential digests. +func (m *Manager) helloMessage() *syncMessage { + self := m.selfNodeInfo() + m.state.setLocalInventory(self) + gd := m.state.groupDoc() + return &syncMessage{ + Type: "hello", + Node: self, + Nodes: m.knownNodesForPEX(), + Groups: &gd, + GroupsVer: gd.Version, + CredDigests: m.state.credDigests(), } } -// announceLoop periodically broadcasts our digest so peers can pull anything they -// missed (anti-entropy backstop for lost push messages). +// announceMessage is the periodic anti-entropy + inventory advert. +func (m *Manager) announceMessage() *syncMessage { + self := m.selfNodeInfo() + gd := m.state.groupDoc() + return &syncMessage{ + Type: "announce", + Node: self, + Nodes: m.knownNodesForPEX(), + Groups: &gd, + GroupsVer: gd.Version, + CredDigests: m.state.credDigests(), + } +} + +// announceLoop periodically rebroadcasts inventory + digests for convergence. func (m *Manager) announceLoop() { for { cfg := m.cfgStore.get() @@ -254,182 +321,256 @@ func (m *Manager) announceLoop() { return case <-time.After(interval): } - cfg = m.cfgStore.get() - if !cfg.active() { + if !m.cfgStore.get().active() { continue } - m.broadcast(&syncMessage{Type: "announce", Digests: m.state.digests()}) + m.state.setLocalInventory(m.selfNodeInfo()) + m.broadcast(m.announceMessage()) } } -// ---- storage hook (local change source) ---- +// ---- credential seeding / hooks ---- + +// seedLocalCreds records credentials for local healthy mounts that belong to a +// group, pulls for member-groups we have no credential for yet, and re-applies +// any held credential to local mounts (e.g. after a groups change). +func (m *Manager) seedLocalCreds() { + cfg := m.cfgStore.get() + if !cfg.active() { + return + } + selfID := m.id.NodeID + var changed bool + for _, d := range op.GetAllStorages() { + st := d.GetStorage() + groups := m.state.groupsForMount(selfID, st.MountPath) + if len(groups) == 0 { + continue + } + if st.Status != op.WORK { + for _, g := range groups { + m.pullGroup(g.ID) + } + continue + } + creds := extractCreds(st.Addition) + for _, g := range groups { + if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { + changed = true + m.recordEvent("share", g.ID, fmt.Sprintf("%s shared %d credential field(s)", st.MountPath, len(rec.Fields))) + m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) + } + } + } + // member-groups we hold no credential for: ask peers. + for _, g := range m.state.groupList() { + if len(g.mountsForNode(selfID)) == 0 { + continue + } + if _, ok := m.state.getCred(g.ID); !ok { + m.pullGroup(g.ID) + } + } + // (re)apply held credentials to local mounts. + for _, r := range m.state.credSnapshot() { + m.applyCredRecord(r) + } + if changed { + m.persist() + } +} +// onStorageHook reacts to local storage lifecycle/credential changes. func (m *Manager) onStorageHook(typ string, d driver.Driver) { cfg := m.cfgStore.get() if !cfg.active() { return } st := d.GetStorage() - if !cfg.shouldShare(st.Driver, st.MountPath) { + // Any storage change may alter our inventory (added/removed mount, status). + go m.refreshInventory() + + selfID := m.id.NodeID + groups := m.state.groupsForMount(selfID, st.MountPath) + if len(groups) == 0 { return } switch typ { case "add", "update": if st.Status != op.WORK { - // Health gating: never propagate a broken/expired token. Instead try - // to recover a good one from a peer. - m.pullMount(st.MountPath) + // Health gating: never propagate a broken/expired token. Try to recover + // a good one from peers instead. + for _, g := range groups { + m.pullGroup(g.ID) + } return } - rec, ok := m.state.localChange(m.id, fromModel(st), now()) - if ok { - m.persist() - m.broadcast(&syncMessage{Type: "push", Records: []*record{rec}}) - } - case "del": - if !cfg.ShareDeletes { - return + creds := extractCreds(st.Addition) + var dirty bool + for _, g := range groups { + if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { + dirty = true + m.recordEvent("share", g.ID, fmt.Sprintf("%s refreshed credentials", st.MountPath)) + m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) + } } - rec, ok := m.state.localDelete(m.id, st.MountPath, now()) - if ok { + if dirty { m.persist() - m.broadcast(&syncMessage{Type: "push", Records: []*record{rec}}) } + case "del": + // A mount was removed locally. We keep the group's credential record (other + // members still rely on it); only our inventory changes (handled above). case "token-invalid": - // A driver reported its token is dead; pull a fresh one from peers. - m.pullMount(st.MountPath) + for _, g := range groups { + m.pullGroup(g.ID) + } } } -// pullMount asks every connected peer for the latest record of a single mount. -// Answers arrive asynchronously and are applied by handleFrame. -func (m *Manager) pullMount(mountPath string) { - cfg := m.cfgStore.get() - if !cfg.active() { +// pullGroup asks peers for the latest credential of a group. +func (m *Manager) pullGroup(groupID string) { + if !m.cfgStore.get().active() { return } - m.broadcast(&syncMessage{Type: "pull", Wants: []string{mountPath}}) + m.broadcast(&syncMessage{Type: "pull", Wants: []string{groupID}}) } -// ---- message processing ---- - -// buildReply produces the anti-entropy answer to a peer's digests/wants. It does -// NOT absorb the incoming records (handleFrame does that first, separately, so it -// can relay newly-applied ones). Returns nil when there is nothing to send back. -func (m *Manager) buildReply(in *syncMessage) *syncMessage { - reply := &syncMessage{Type: "reply"} - - // Anti-entropy on digests: tell the peer where we differ. - if len(in.Digests) > 0 { - peerHas := make(map[string]digest, len(in.Digests)) - for _, dg := range in.Digests { - peerHas[dg.MountPath] = dg - local, ok := m.state.get(dg.MountPath) - if !ok { - reply.Wants = append(reply.Wants, dg.MountPath) // we lack it - continue - } - cmp := digestOf(local) - switch { - case digestDominates(cmp, dg): - reply.Records = append(reply.Records, local) // ours is newer - case digestDominates(dg, cmp): - reply.Wants = append(reply.Wants, dg.MountPath) // theirs is newer - } +// applyCredRecord overlays a group's credential onto this node's member mounts. +func (m *Manager) applyCredRecord(r *credRecord) { + if r == nil { + return + } + cfg := m.cfgStore.get() + if !cfg.ApplyRemote { + return + } + g, ok := m.state.groupByID(r.GroupID) + if !ok { + return + } + ctx := context.Background() + for _, mp := range g.mountsForNode(m.id.NodeID) { + d, err := op.GetStorageByMountPath(mp) + if err != nil { + continue } - // Records we hold that the peer never mentioned — it is missing them. - for _, local := range m.state.snapshot() { - if _, seen := peerHas[local.MountPath]; !seen { - reply.Records = append(reply.Records, local) - } + st := *d.GetStorage() // copy; preserve ID/Status/local fields + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", + r.GroupID, mp, st.Driver, r.OriginDriver) + continue } + newAdd, changed := applyCreds(st.Addition, r.Payload) + if !changed { + continue // already has these credentials — no churn, no re-init + } + st.Addition = newAdd + if err := op.UpdateStorage(ctx, st); err != nil { + utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) + continue + } + m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) + utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) } +} + +// ---- absorb (merge) helpers ---- - // Serve explicit wants (pull). - for _, mp := range in.Wants { - if r, ok := m.state.get(mp); ok { - reply.Records = append(reply.Records, r) +func (m *Manager) absorbInventory(nodes []*nodeInfo) []*nodeInfo { + var merged []*nodeInfo + for _, n := range nodes { + if n == nil || n.NodeID == "" || n.NodeID == m.id.NodeID { + continue + } + if m.state.mergeInventory(n, now()) { + merged = append(merged, n) } } + return merged +} - if len(reply.Records) == 0 && len(reply.Wants) == 0 { - return nil +func (m *Manager) absorbGroups(d *groupDoc) bool { + if d == nil || !d.verify() { + return false } - return reply + if m.state.mergeGroups(d) { + m.state.pruneCreds() + m.persist() + m.recordEvent("groups", "", fmt.Sprintf("received %d group(s) from %s", len(d.Groups), shortNode(d.Origin))) + go m.seedLocalCreds() + return true + } + return false } -// applyRecords verifies, merges and (if configured) applies a batch of records, -// returning the records that were newly applied/tombstoned so the caller can -// relay them onward (hub behaviour). Only genuine changes are returned, so relay -// naturally terminates. -func (m *Manager) applyRecords(recs []*record) []*record { - if len(recs) == 0 { - return nil - } - cfg := m.cfgStore.get() - var applied []*record - var dirty bool +func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { + var merged []*credRecord for _, r := range recs { if r == nil || !r.verify() { continue } - switch m.state.merge(r) { - case mergeApplied: - dirty = true - applied = append(applied, r) - if cfg.ApplyRemote { - m.applyToLocal(r) - } - case mergeTombstone: - dirty = true - applied = append(applied, r) - if cfg.ApplyRemote { - m.deleteLocal(r.MountPath) - } + if m.state.mergeCred(r) { + merged = append(merged, r) + m.applyCredRecord(r) } } - if dirty { + if len(merged) > 0 { m.persist() } - return applied + return merged } -// applyToLocal creates or updates the local storage from a record's config. The -// resulting op hook is neutralized by content-hash idempotency: state already -// holds this exact hash, so onStorageHook's localChange is a no-op. -func (m *Manager) applyToLocal(r *record) { - ctx := context.Background() - if d, err := op.GetStorageByMountPath(r.MountPath); err == nil { - existing := *d.GetStorage() // copy; preserve ID/Status - r.Config.applyTo(&existing) - if err := op.UpdateStorage(ctx, existing); err != nil { - utils.Log.Warnf("[cluster] apply update %s failed: %v", r.MountPath, err) - } else { - utils.Log.Infof("[cluster] applied peer config for %s (v%d from %s)", r.MountPath, r.Version, r.Origin) +// ---- anti-entropy reply ---- + +// buildReply answers a peer's digests / wants, telling it what we hold that it +// lacks and requesting what it holds that we lack. +func (m *Manager) buildReply(in *syncMessage) *syncMessage { + reply := &syncMessage{Type: "reply"} + + // Groups: if ours is newer, offer it. + local := m.state.groupDoc() + if local.Version > in.GroupsVer { + reply.Groups = &local + } + + // Credential anti-entropy on digests. + peerHas := make(map[string]credDigest, len(in.CredDigests)) + for _, dg := range in.CredDigests { + peerHas[dg.GroupID] = dg + cur, ok := m.state.getCred(dg.GroupID) + if !ok { + reply.Wants = append(reply.Wants, dg.GroupID) + continue + } + mine := digestOfCred(cur) + switch { + case credDigestDominates(mine, dg): + reply.Creds = append(reply.Creds, cur) + case credDigestDominates(dg, mine): + reply.Wants = append(reply.Wants, dg.GroupID) } - return } - var st model.Storage - r.Config.applyTo(&st) - if _, err := op.CreateStorage(ctx, st); err != nil { - utils.Log.Warnf("[cluster] apply create %s failed: %v", r.MountPath, err) - } else { - utils.Log.Infof("[cluster] created storage %s from peer (v%d from %s)", r.MountPath, r.Version, r.Origin) + // Records we hold the peer never mentioned. + for _, cur := range m.state.credSnapshot() { + if _, seen := peerHas[cur.GroupID]; !seen { + reply.Creds = append(reply.Creds, cur) + } } -} -func (m *Manager) deleteLocal(mountPath string) { - d, err := op.GetStorageByMountPath(mountPath) - if err != nil { - return + // Explicit pull wants. + for _, gid := range in.Wants { + if cur, ok := m.state.getCred(gid); ok { + reply.Creds = append(reply.Creds, cur) + } } - id := d.GetStorage().ID - if err := op.DeleteStorageById(context.Background(), id); err != nil { - utils.Log.Warnf("[cluster] apply delete %s failed: %v", mountPath, err) + + if reply.Groups == nil && len(reply.Creds) == 0 && len(reply.Wants) == 0 { + return nil } + return reply } -// ---- networking (persistent connections) ---- +// ---- networking ---- // seal wraps a message in an encrypted envelope addressed from this node. func (m *Manager) seal(msg *syncMessage) ([]byte, error) { @@ -449,9 +590,7 @@ func (m *Manager) sendTo(c *peerConn, msg *syncMessage) { c.enqueue(frame) } -// broadcast enqueues a sealed message on every live connection. The same sealed -// frame is reused for all peers (each peer keeps its own replay cache, so a -// shared nonce is fine). +// broadcast enqueues a sealed message on every live connection. func (m *Manager) broadcast(msg *syncMessage) { conns := m.conns.all() if len(conns) == 0 { @@ -466,16 +605,15 @@ func (m *Manager) broadcast(msg *syncMessage) { } } -// relay forwards newly-applied records to every connection except the one they +// relayMsg forwards newly-merged info to every connection except the one it // arrived on — this is what lets two NAT'd nodes converge through a common -// reachable peer. Records carry their own origin signature, so re-sealing them in -// our envelope does not weaken authenticity. -func (m *Manager) relay(recs []*record, except *peerConn) { +// reachable peer. Signed records/groups stay authentic across the relay. +func (m *Manager) relayMsg(msg *syncMessage, except *peerConn) { conns := m.conns.all() if len(conns) <= 1 { return } - frame, err := m.seal(&syncMessage{Type: "push", Records: recs}) + frame, err := m.seal(msg) if err != nil { return } @@ -487,9 +625,8 @@ func (m *Manager) relay(recs []*record, except *peerConn) { } } -// handleFrame authenticates and processes one inbound frame from a connection. -// A frame that fails to open (wrong cluster key / replay / stale) drops the -// connection — only PSK holders are admitted. +// handleFrame authenticates and processes one inbound frame. A frame that fails +// to open (wrong key / replay / stale) drops the connection. func (m *Manager) handleFrame(c *peerConn, data []byte) { cfg := m.cfgStore.get() if !cfg.active() { @@ -507,33 +644,77 @@ func (m *Manager) handleFrame(c *peerConn, data []byte) { c.close() return } - if c.nodeID == "" && env.Sender != m.id.NodeID { + if env.Sender == m.id.NodeID { + c.close() // connected to ourselves + return + } + if c.nodeID == "" { m.conns.bind(c, env.Sender) } - // Absorb offered records first, then relay the ones that were new. - if applied := m.applyRecords(msg.Records); len(applied) > 0 { - m.relay(applied, c) + + relay := &syncMessage{Type: "push"} + relayHas := false + + // Inventory + PEX. + var invs []*nodeInfo + if msg.Node != nil { + invs = append(invs, msg.Node) } - // Answer the peer's anti-entropy digests / pull wants. - if reply := m.buildReply(msg); reply != nil { - m.sendTo(c, reply) + invs = append(invs, msg.Nodes...) + if merged := m.absorbInventory(invs); len(merged) > 0 { + relay.Nodes = merged + relayHas = true + } + // Groups. + if m.absorbGroups(msg.Groups) { + gd := m.state.groupDoc() + relay.Groups = &gd + relayHas = true + } + // Credentials. + if merged := m.absorbCreds(msg.Creds); len(merged) > 0 { + relay.Creds = merged + relayHas = true + } + if relayHas { + m.relayMsg(relay, c) + } + + // Anti-entropy reply only for digest/want-bearing messages (avoids echo). + switch msg.Type { + case "hello", "announce", "pull": + if reply := m.buildReply(msg); reply != nil { + m.sendTo(c, reply) + } } } -// ---- digest helpers ---- +// ---- events ---- -func digestOf(r *record) digest { - return digest{MountPath: r.MountPath, ContentHash: r.ContentHash, Version: r.Version, Origin: r.Origin, Tombstone: r.Tombstone} +func (m *Manager) recordEvent(kind, groupID, detail string) { + m.evMu.Lock() + defer m.evMu.Unlock() + m.events = append(m.events, EventView{Time: now(), Kind: kind, GroupID: groupID, Detail: detail}) + if len(m.events) > maxEvents { + m.events = m.events[len(m.events)-maxEvents:] + } } -// digestDominates reports whether a wins over b under the same LWW ordering as -// records. -func digestDominates(a, b digest) bool { - if a.Version != b.Version { - return a.Version > b.Version +func (m *Manager) eventList() []EventView { + m.evMu.Lock() + defer m.evMu.Unlock() + out := make([]EventView, len(m.events)) + copy(out, m.events) + // newest first + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] } - if a.Origin != b.Origin { - return a.Origin > b.Origin + return out +} + +func shortNode(id string) string { + if len(id) <= 8 { + return id } - return a.ContentHash > b.ContentHash + return id[:8] } diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 5c5835bb19..3af17c7455 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -2,144 +2,130 @@ package cluster import ( "crypto/sha256" - "encoding/hex" "encoding/json" "sort" "sync" - - "github.com/OpenListTeam/OpenList/v4/internal/model" ) -// syncableStorage is the canonical, node-independent view of a storage mount that -// we replicate between cluster nodes. Node-local runtime fields (ID, Status, -// Modified) are deliberately excluded so the content hash is identical on every -// node for the same logical config — that idempotency is what lets a node ignore -// a peer that reports "the same token I already have" ("其他节点看到一样的token就不用变"). -type syncableStorage struct { - MountPath string `json:"mount_path"` - Order int `json:"order"` - Driver string `json:"driver"` - CacheExpiration int `json:"cache_expiration"` - CustomCachePolicies string `json:"custom_cache_policies"` - Addition string `json:"addition"` - Remark string `json:"remark"` - Disabled bool `json:"disabled"` - DisableIndex bool `json:"disable_index"` - EnableSign bool `json:"enable_sign"` - // Sort - OrderBy string `json:"order_by"` - OrderDirection string `json:"order_direction"` - ExtractFolder string `json:"extract_folder"` - // Proxy - WebProxy bool `json:"web_proxy"` - WebdavPolicy string `json:"webdav_policy"` - ProxyRange bool `json:"proxy_range"` - DownProxyURL string `json:"down_proxy_url"` - DisableProxySign bool `json:"disable_proxy_sign"` -} - -func fromModel(s *model.Storage) syncableStorage { - return syncableStorage{ - MountPath: s.MountPath, - Order: s.Order, - Driver: s.Driver, - CacheExpiration: s.CacheExpiration, - CustomCachePolicies: s.CustomCachePolicies, - Addition: s.Addition, - Remark: s.Remark, - Disabled: s.Disabled, - DisableIndex: s.DisableIndex, - EnableSign: s.EnableSign, - OrderBy: s.OrderBy, - OrderDirection: s.OrderDirection, - ExtractFolder: s.ExtractFolder, - WebProxy: s.WebProxy, - WebdavPolicy: s.WebdavPolicy, - ProxyRange: s.ProxyRange, - DownProxyURL: s.DownProxyURL, - DisableProxySign: s.DisableProxySign, - } -} - -// applyTo writes the synced fields onto an existing storage model, preserving the -// node-local runtime fields (ID/Status/Modified) of dst. -func (sc syncableStorage) applyTo(dst *model.Storage) { - dst.MountPath = sc.MountPath - dst.Order = sc.Order - dst.Driver = sc.Driver - dst.CacheExpiration = sc.CacheExpiration - dst.CustomCachePolicies = sc.CustomCachePolicies - dst.Addition = sc.Addition - dst.Remark = sc.Remark - dst.Disabled = sc.Disabled - dst.DisableIndex = sc.DisableIndex - dst.EnableSign = sc.EnableSign - dst.OrderBy = sc.OrderBy - dst.OrderDirection = sc.OrderDirection - dst.ExtractFolder = sc.ExtractFolder - dst.WebProxy = sc.WebProxy - dst.WebdavPolicy = sc.WebdavPolicy - dst.ProxyRange = sc.ProxyRange - dst.DownProxyURL = sc.DownProxyURL - dst.DisableProxySign = sc.DisableProxySign -} - -// canonicalJSON marshals deterministically. encoding/json already sorts struct -// fields by declaration order (stable), so a plain Marshal is canonical here. -func (sc syncableStorage) canonicalJSON() []byte { - b, _ := json.Marshal(sc) - return b +// ---------------------------------------------------------------------------- +// Sync groups (the multipartite mapping) +// +// A group links storages ACROSS nodes that should share one credential — e.g. +// the same 115 account mounted on cfscan, tx and home-nas. The set of groups is +// cluster-wide shared state, replicated as a single last-writer-wins document +// (admin edits are infrequent, so whole-doc LWW is the simplest convergent +// choice). Each group is a connected component of the overall k-partite graph. +// ---------------------------------------------------------------------------- + +// member is one storage on one node. +type member struct { + NodeID string `json:"node_id"` + MountPath string `json:"mount_path"` } -// contentHash is the idempotency key: identical config → identical hash on every -// node, regardless of who authored it. -func (sc syncableStorage) contentHash() string { - sum := sha256.Sum256(sc.canonicalJSON()) - return hex.EncodeToString(sum[:]) -} - -// record is one replicated mount entry: the config plus CRDT version metadata. -// Ordering is a Lamport clock with the origin node id as a deterministic -// tiebreaker, giving a total order for last-writer-wins convergence. -type record struct { - MountPath string `json:"mount_path"` - ContentHash string `json:"content_hash"` - Version uint64 `json:"version"` // Lamport logical clock - Origin string `json:"origin"` // node id that authored this version - OriginPub []byte `json:"origin_pub"` // origin's ed25519 pubkey; node id == hash(pubkey) - Tombstone bool `json:"tombstone"` // true => mount was deleted - UpdatedAt int64 `json:"updated_at"` // unix seconds, informational only - Config syncableStorage `json:"config"` // empty when Tombstone - Sig []byte `json:"sig"` // origin's ed25519 signature over signingBytes -} - -// verify checks a record is internally authentic: the origin node id is the hash -// of the embedded pubkey (so a member cannot claim another node's id without its -// private key), the signature is valid under that pubkey, and the content hash -// matches the carried config. The cluster PSK (transport seal) gates membership; -// this gates per-record origin integrity for relayed records. -func (r *record) verify() bool { - if nodeIDFromPub(r.OriginPub) != r.Origin { - return false - } - if !r.Tombstone { - if r.Config.contentHash() != r.ContentHash { - return false +type group struct { + ID string `json:"id"` + Name string `json:"name"` + Members []member `json:"members"` +} + +// mountsForNode returns the mount paths this node contributes to the group. +func (g group) mountsForNode(nodeID string) []string { + var out []string + for _, m := range g.Members { + if m.NodeID == nodeID { + out = append(out, m.MountPath) } - } else if r.ContentHash != (syncableStorage{MountPath: r.MountPath}).contentHash() { + } + return out +} + +// groupDoc is the replicated set of groups plus LWW version metadata. +type groupDoc struct { + Groups []group `json:"groups"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + OriginPub []byte `json:"origin_pub"` + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` +} + +// canonicalGroups returns the groups sorted deterministically so signing and +// comparison are stable regardless of insertion order. +func canonicalGroups(groups []group) []group { + out := make([]group, len(groups)) + copy(out, groups) + for i := range out { + ms := make([]member, len(out[i].Members)) + copy(ms, out[i].Members) + sort.Slice(ms, func(a, b int) bool { + if ms[a].NodeID != ms[b].NodeID { + return ms[a].NodeID < ms[b].NodeID + } + return ms[a].MountPath < ms[b].MountPath + }) + out[i].Members = ms + } + sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) + return out +} + +func (d *groupDoc) signingBytes() []byte { + canon := struct { + Groups []group `json:"groups"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + }{canonicalGroups(d.Groups), d.Version, d.Origin} + b, _ := json.Marshal(canon) + sum := sha256.Sum256(b) + return sum[:] +} + +func (d *groupDoc) verify() bool { + if d.Version == 0 { + return true // the empty/default doc is implicitly valid + } + if nodeIDFromPub(d.OriginPub) != d.Origin { return false } - return verifySig(r.OriginPub, r.signingBytes(), r.Sig) + return verifySig(d.OriginPub, d.signingBytes(), d.Sig) +} + +func (d *groupDoc) dominates(other *groupDoc) bool { + if d.Version != other.Version { + return d.Version > other.Version + } + return d.Origin > other.Origin } -// signingBytes is the stable byte string an origin node signs to authenticate a -// version. It excludes the signature itself and the (informational) timestamp. -func (r *record) signingBytes() []byte { - // length-free, delimiter-joined fields; ContentHash already binds Config. +// ---------------------------------------------------------------------------- +// Credential records (the thing actually replicated per group) +// ---------------------------------------------------------------------------- + +// credRecord carries a group's current credential payload. The payload holds the +// secret fields and only ever travels inside the AEAD-sealed envelope. Records +// are signed by their origin so they stay authentic across relays. +type credRecord struct { + GroupID string `json:"group_id"` + OriginDriver string `json:"origin_driver"` // apply only to same-driver mounts + Fields []string `json:"fields"` // credential field names included + CredHash string `json:"cred_hash"` // idempotency key + Payload map[string]json.RawMessage `json:"payload"` // field -> value (secret) + Version uint64 `json:"version"` // Lamport clock + Origin string `json:"origin"` // authoring node id + OriginPub []byte `json:"origin_pub"` // origin ed25519 pubkey + OriginMount string `json:"origin_mount"` // informational + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` +} + +func (r *credRecord) signingBytes() []byte { var b []byte - b = append(b, r.MountPath...) + b = append(b, r.GroupID...) b = append(b, 0) - b = append(b, r.ContentHash...) + b = append(b, r.OriginDriver...) + b = append(b, 0) + b = append(b, r.CredHash...) b = append(b, 0) v := make([]byte, 8) for i := 0; i < 8; i++ { @@ -148,195 +134,357 @@ func (r *record) signingBytes() []byte { b = append(b, v...) b = append(b, 0) b = append(b, r.Origin...) - b = append(b, 0) - if r.Tombstone { - b = append(b, 1) - } else { - b = append(b, 0) - } return b } -// dominates reports whether r should win over other under LWW ordering. -func (r *record) dominates(other *record) bool { +func (r *credRecord) verify() bool { + if nodeIDFromPub(r.OriginPub) != r.Origin { + return false + } + if credHash(r.Payload) != r.CredHash { + return false + } + return verifySig(r.OriginPub, r.signingBytes(), r.Sig) +} + +func (r *credRecord) dominates(other *credRecord) bool { if r.Version != other.Version { return r.Version > other.Version } - // Equal Lamport time: break ties deterministically by origin id, then by - // content hash so two distinct concurrent edits still converge identically - // on every node. if r.Origin != other.Origin { return r.Origin > other.Origin } - return r.ContentHash > other.ContentHash + return r.CredHash > other.CredHash } -// digest is the compact form announced to peers so they can detect divergence -// without shipping full configs (and secrets) on every heartbeat. -type digest struct { - MountPath string `json:"mount_path"` - ContentHash string `json:"content_hash"` - Version uint64 `json:"version"` - Origin string `json:"origin"` - Tombstone bool `json:"tombstone"` +// credDigest is the compact (no-secret) advert of a held credential record. +type credDigest struct { + GroupID string `json:"group_id"` + CredHash string `json:"cred_hash"` + Version uint64 `json:"version"` + Origin string `json:"origin"` } -// store is the in-memory CRDT keyed by mount path, with a monotonically -// non-decreasing Lamport clock shared across all mounts. -type store struct { - mu sync.RWMutex - records map[string]*record - lamport uint64 +func digestOfCred(r *credRecord) credDigest { + return credDigest{GroupID: r.GroupID, CredHash: r.CredHash, Version: r.Version, Origin: r.Origin} } -func newStore() *store { - return &store{records: make(map[string]*record)} +func credDigestDominates(a, b credDigest) bool { + if a.Version != b.Version { + return a.Version > b.Version + } + if a.Origin != b.Origin { + return a.Origin > b.Origin + } + return a.CredHash > b.CredHash } -// tick advances and returns the Lamport clock for a locally-originated change. -func (s *store) tick() uint64 { - s.lamport++ - return s.lamport +// ---------------------------------------------------------------------------- +// Node inventory (soft state powering the UI + group editing) +// ---------------------------------------------------------------------------- + +type storageInfo struct { + MountPath string `json:"mount_path"` + Driver string `json:"driver"` + Status string `json:"status"` +} + +type nodeInfo struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Storages []storageInfo `json:"storages"` + Version uint64 `json:"version"` + UpdatedAt int64 `json:"updated_at"` + // seenAt is set locally on receipt for liveness; not part of the wire form's + // trust (it is overwritten each time we hear from/about the node). + seenAt int64 `json:"-"` +} + +// ---------------------------------------------------------------------------- +// store: the in-memory cluster state (groups + creds + inventory) +// ---------------------------------------------------------------------------- + +type store struct { + mu sync.RWMutex + groups groupDoc + creds map[string]*credRecord // keyed by group id + inventory map[string]*nodeInfo // keyed by node id + lamport uint64 +} + +func newStore() *store { + return &store{ + creds: make(map[string]*credRecord), + inventory: make(map[string]*nodeInfo), + } } -// observe bumps the Lamport clock to stay ahead of a value seen from a peer. func (s *store) observe(v uint64) { if v > s.lamport { s.lamport = v } } -func (s *store) get(mountPath string) (*record, bool) { +// ---- groups ---- + +func (s *store) groupDoc() groupDoc { + s.mu.RLock() + defer s.mu.RUnlock() + return s.groups +} + +func (s *store) groupList() []group { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]group, len(s.groups.Groups)) + copy(out, s.groups.Groups) + return out +} + +// setGroups authors a new groups document locally (admin edit). Returns the +// signed doc to broadcast. +func (s *store) setGroups(id *identity, groups []group, now int64) groupDoc { + s.mu.Lock() + defer s.mu.Unlock() + s.lamport++ + d := groupDoc{ + Groups: canonicalGroups(groups), + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + UpdatedAt: now, + } + d.Sig = id.sign(d.signingBytes()) + s.groups = d + return d +} + +// mergeGroups integrates a peer's groups document under LWW. Returns true if it +// replaced ours. +func (s *store) mergeGroups(d *groupDoc) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.observe(d.Version) + if d.dominates(&s.groups) { + s.groups = *d + return true + } + return false +} + +// groupsForMount returns the groups that include a (this-node) mount path. +func (s *store) groupsForMount(nodeID, mountPath string) []group { + s.mu.RLock() + defer s.mu.RUnlock() + var out []group + for _, g := range s.groups.Groups { + for _, m := range g.Members { + if m.NodeID == nodeID && m.MountPath == mountPath { + out = append(out, g) + break + } + } + } + return out +} + +func (s *store) groupByID(id string) (group, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, g := range s.groups.Groups { + if g.ID == id { + return g, true + } + } + return group{}, false +} + +// ---- creds ---- + +func (s *store) getCred(groupID string) (*credRecord, bool) { s.mu.RLock() defer s.mu.RUnlock() - r, ok := s.records[mountPath] + r, ok := s.creds[groupID] return r, ok } -func (s *store) snapshot() []*record { +func (s *store) credSnapshot() []*credRecord { s.mu.RLock() defer s.mu.RUnlock() - out := make([]*record, 0, len(s.records)) - for _, r := range s.records { + out := make([]*credRecord, 0, len(s.creds)) + for _, r := range s.creds { out = append(out, r) } - sort.Slice(out, func(i, j int) bool { return out[i].MountPath < out[j].MountPath }) + sort.Slice(out, func(i, j int) bool { return out[i].GroupID < out[j].GroupID }) return out } -func (s *store) digests() []digest { +func (s *store) credDigests() []credDigest { s.mu.RLock() defer s.mu.RUnlock() - out := make([]digest, 0, len(s.records)) - for _, r := range s.records { - out = append(out, digest{ - MountPath: r.MountPath, - ContentHash: r.ContentHash, - Version: r.Version, - Origin: r.Origin, - Tombstone: r.Tombstone, - }) + out := make([]credDigest, 0, len(s.creds)) + for _, r := range s.creds { + out = append(out, digestOfCred(r)) } - sort.Slice(out, func(i, j int) bool { return out[i].MountPath < out[j].MountPath }) + sort.Slice(out, func(i, j int) bool { return out[i].GroupID < out[j].GroupID }) return out } -// localChange records a config a node authored itself. It returns the new record -// to broadcast, or (nil,false) when nothing changed (idempotent no-op): same -// content hash and not resurrecting a tombstone. -func (s *store) localChange(id *identity, cfg syncableStorage, now int64) (*record, bool) { +// localCredChange records a credential a node observed on one of its own mounts. +// Returns (nil,false) when the credential is unchanged (idempotent: "其他节点看到 +// 一样的 token 就不用变") or empty. +func (s *store) localCredChange(id *identity, groupID, driver, mount string, payload map[string]json.RawMessage, now int64) (*credRecord, bool) { + h := credHash(payload) + if h == "" { + return nil, false + } s.mu.Lock() defer s.mu.Unlock() - h := cfg.contentHash() - if cur, ok := s.records[cfg.MountPath]; ok && !cur.Tombstone && cur.ContentHash == h { - return nil, false // unchanged — do not bump version or churn peers + if cur, ok := s.creds[groupID]; ok && cur.CredHash == h { + return nil, false // identical credential already known — no churn } s.lamport++ - r := &record{ - MountPath: cfg.MountPath, - ContentHash: h, - Version: s.lamport, - Origin: id.NodeID, - OriginPub: id.Pub, - Tombstone: false, - UpdatedAt: now, - Config: cfg, + r := &credRecord{ + GroupID: groupID, + OriginDriver: driver, + Fields: credFieldNames(payload), + CredHash: h, + Payload: payload, + Version: s.lamport, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + UpdatedAt: now, } r.Sig = id.sign(r.signingBytes()) - s.records[cfg.MountPath] = r + s.creds[groupID] = r return r, true } -// localDelete authors a tombstone for a mount. Returns (nil,false) if already -// tombstoned or unknown. -func (s *store) localDelete(id *identity, mountPath string, now int64) (*record, bool) { +// mergeCred integrates a peer's credential record under LWW. The caller must have +// verified the signature. Returns true if it replaced/added ours. +func (s *store) mergeCred(r *credRecord) bool { s.mu.Lock() defer s.mu.Unlock() - cur, ok := s.records[mountPath] - if !ok || cur.Tombstone { - return nil, false + s.observe(r.Version) + cur, ok := s.creds[r.GroupID] + if ok { + if cur.CredHash == r.CredHash { + return false // idempotent: same credential + } + if !r.dominates(cur) { + return false + } } - s.lamport++ - r := &record{ - MountPath: mountPath, - // hash of an empty config keeps signingBytes well-defined for tombstones - ContentHash: syncableStorage{MountPath: mountPath}.contentHash(), - Version: s.lamport, - Origin: id.NodeID, - OriginPub: id.Pub, - Tombstone: true, - UpdatedAt: now, + cp := *r + s.creds[r.GroupID] = &cp + return true +} + +// pruneCreds drops credential records for groups that no longer exist. +func (s *store) pruneCreds() { + s.mu.Lock() + defer s.mu.Unlock() + live := make(map[string]struct{}, len(s.groups.Groups)) + for _, g := range s.groups.Groups { + live[g.ID] = struct{}{} + } + for id := range s.creds { + if _, ok := live[id]; !ok { + delete(s.creds, id) + } } - r.Sig = id.sign(r.signingBytes()) - s.records[mountPath] = r - return r, true } -// mergeResult describes what merge did with an incoming record. -type mergeResult int +// ---- inventory ---- -const ( - mergeIgnored mergeResult = iota // incoming did not win (older/equal/duplicate) - mergeApplied // incoming won and replaced local (config changed) - mergeTombstone // incoming won and is a delete -) +func (s *store) setLocalInventory(info *nodeInfo) { + s.mu.Lock() + defer s.mu.Unlock() + info.seenAt = info.UpdatedAt + s.inventory[info.NodeID] = info +} -// merge integrates a peer's record. The caller must have already verified r.Sig -// against the origin's known public key. merge enforces LWW ordering and the -// idempotency rule, and advances the Lamport clock. -func (s *store) merge(r *record) mergeResult { +// mergeInventory integrates a peer's inventory entry (LWW by version). seenAt is +// always refreshed so liveness reflects the latest contact. +func (s *store) mergeInventory(info *nodeInfo, now int64) bool { s.mu.Lock() defer s.mu.Unlock() - if r.Version > s.lamport { - s.lamport = r.Version + cur, ok := s.inventory[info.NodeID] + if ok && info.Version < cur.Version { + cur.seenAt = now // still heard about it; keep liveness fresh + return false } - cur, ok := s.records[r.MountPath] - if ok { - // Idempotent: identical live content, ignore regardless of version churn. - if !r.Tombstone && !cur.Tombstone && cur.ContentHash == r.ContentHash { - return mergeIgnored - } - if !r.dominates(cur) { - return mergeIgnored + cp := *info + cp.seenAt = now + s.inventory[info.NodeID] = &cp + return true +} + +func (s *store) inventoryList() []nodeInfo { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]nodeInfo, 0, len(s.inventory)) + for _, n := range s.inventory { + cp := *n + cp.seenAt = n.seenAt + out = append(out, cp) + } + sort.Slice(out, func(i, j int) bool { return out[i].NodeID < out[j].NodeID }) + return out +} + +// dialableAddrs returns advertised peer addresses (excluding our own node id) for +// auto-discovery dialing. +func (s *store) dialableAddrs(selfID string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + var out []string + for id, n := range s.inventory { + if id == selfID || n.Addr == "" { + continue } + out = append(out, n.Addr) } - cp := *r - s.records[r.MountPath] = &cp - if cp.Tombstone { - return mergeTombstone + return out +} + +// ---- persistence ---- + +type persistedState struct { + Lamport uint64 `json:"lamport"` + Groups groupDoc `json:"groups"` + Creds []*credRecord `json:"creds"` + Inventory []*nodeInfo `json:"inventory"` +} + +func (s *store) export() persistedState { + s.mu.RLock() + defer s.mu.RUnlock() + ps := persistedState{Lamport: s.lamport, Groups: s.groups} + for _, r := range s.creds { + ps.Creds = append(ps.Creds, r) + } + for _, n := range s.inventory { + ps.Inventory = append(ps.Inventory, n) } - return mergeApplied + return ps } -// load replaces the store contents from a persisted snapshot. -func (s *store) load(records []*record, lamport uint64) { +func (s *store) load(ps persistedState) { s.mu.Lock() defer s.mu.Unlock() - s.records = make(map[string]*record, len(records)) - for _, r := range records { - s.records[r.MountPath] = r + s.lamport = ps.Lamport + s.groups = ps.Groups + s.creds = make(map[string]*credRecord, len(ps.Creds)) + for _, r := range ps.Creds { + s.creds[r.GroupID] = r + } + s.inventory = make(map[string]*nodeInfo, len(ps.Inventory)) + for _, n := range ps.Inventory { + s.inventory[n.NodeID] = n } - s.lamport = lamport } func (s *store) lamportNow() uint64 { @@ -344,3 +492,11 @@ func (s *store) lamportNow() uint64 { defer s.mu.RUnlock() return s.lamport } + +// shortHash returns a short prefix of a hash for display. +func shortHash(h string) string { + if len(h) <= 12 { + return h + } + return h[:12] +} diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index bf9f36b228..8d8033a826 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -1,138 +1,204 @@ package cluster import ( + "encoding/json" "testing" - - "github.com/OpenListTeam/OpenList/v4/internal/model" ) -func mkCfg(mount, addition string) syncableStorage { - return fromModel(&model.Storage{MountPath: mount, Driver: "115 Open", Addition: addition}) -} +// ---- credential extraction ---- -func TestLocalChangeIdempotent(t *testing.T) { - s := newStore() - id, _ := newIdentity() - cfg := mkCfg("/115", `{"token":"v1"}`) +func TestIsCredField(t *testing.T) { + creds := []string{"refresh_token", "RefreshToken", "access_token", "cookie", "password", "client_secret", "api_key", "authorization", "session_id"} + for _, c := range creds { + if !isCredField(c) { + t.Errorf("%q should be detected as a credential field", c) + } + } + nonCreds := []string{"root_folder_id", "root_folder_path", "order_by", "region", "endpoint", "chunk_size", "device_id"} + for _, n := range nonCreds { + if isCredField(n) { + t.Errorf("%q should NOT be detected as a credential field", n) + } + } +} - r1, ok := s.localChange(id, cfg, 1) - if !ok || r1 == nil { - t.Fatal("first change should produce a record") +func TestExtractAndApplyCreds(t *testing.T) { + addition := `{"root_folder_id":"123","refresh_token":"OLD","access_token":"AOLD","order_by":"name"}` + creds := extractCreds(addition) + if len(creds) != 2 { + t.Fatalf("expected 2 credential fields, got %d (%v)", len(creds), creds) + } + if _, ok := creds["root_folder_id"]; ok { + t.Fatal("structural field leaked into credential set") } - v1 := r1.Version - // Same content again => no-op, no version bump ("same token, don't change"). - if r2, ok := s.localChange(id, cfg, 2); ok || r2 != nil { - t.Fatal("identical config must be a no-op") + // Apply new creds onto a peer Addition that has a DIFFERENT root folder. + peer := `{"root_folder_id":"999","refresh_token":"OLDER","access_token":"AOLDER","order_by":"size"}` + newCreds := map[string]json.RawMessage{ + "refresh_token": json.RawMessage(`"NEW"`), + "access_token": json.RawMessage(`"ANEW"`), + } + out, changed := applyCreds(peer, newCreds) + if !changed { + t.Fatal("applyCreds should report a change") } - if cur, _ := s.get("/115"); cur.Version != v1 { - t.Fatalf("version churned on identical config: %d != %d", cur.Version, v1) + var m map[string]string + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + if m["refresh_token"] != "NEW" || m["access_token"] != "ANEW" { + t.Fatalf("credentials not overlaid: %v", m) + } + // node-local fields preserved. + if m["root_folder_id"] != "999" || m["order_by"] != "size" { + t.Fatalf("node-local fields must be preserved: %v", m) } - // Changed content => new version. - r3, ok := s.localChange(id, mkCfg("/115", `{"token":"v2"}`), 3) - if !ok || r3.Version <= v1 { - t.Fatalf("changed config must bump version (%d should be > %d)", r3.Version, v1) + // Re-applying identical creds is a no-op (idempotent — no churn). + if _, changed := applyCreds(out, newCreds); changed { + t.Fatal("re-applying identical creds must not report a change") } } -func TestRecordVerify(t *testing.T) { - s := newStore() - id, _ := newIdentity() - r, _ := s.localChange(id, mkCfg("/a", `{"t":"1"}`), 1) - if !r.verify() { - t.Fatal("self-produced record must verify") - } - // Tamper with the content hash. - bad := *r - bad.ContentHash = "deadbeef" - if bad.verify() { - t.Fatal("record with mismatched content hash must not verify") - } - // Forge origin (claim a different node id without its key). - forged := *r - forged.Origin = "SOMEONEELSE" - if forged.verify() { - t.Fatal("record whose origin != hash(pubkey) must not verify") +func TestCredHashIdempotent(t *testing.T) { + a := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`), "access_token": json.RawMessage(`"y"`)} + b := map[string]json.RawMessage{"access_token": json.RawMessage(`"y"`), "refresh_token": json.RawMessage(`"x"`)} + if credHash(a) != credHash(b) { + t.Fatal("credHash must be order-independent") + } + if credHash(map[string]json.RawMessage{}) != "" { + t.Fatal("empty creds must hash to empty string") + } + c := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"z"`)} + if credHash(a) == credHash(c) { + t.Fatal("different creds must hash differently") } } -func TestMergeLWWAndIdempotency(t *testing.T) { - // Two nodes author the same mount; higher Lamport version wins. - nodeA, _ := newIdentity() - nodeB, _ := newIdentity() +// ---- groups doc LWW ---- - sa := newStore() - rA, _ := sa.localChange(nodeA, mkCfg("/m", `{"t":"A"}`), 1) +func TestGroupDocSignAndMerge(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() + s := newStore() - sb := newStore() - // B is ahead on the Lamport clock. - sb.lamport = 5 - rB, _ := sb.localChange(nodeB, mkCfg("/m", `{"t":"B"}`), 1) + g := []group{{ID: "g1", Name: "115", Members: []member{{NodeID: id1.NodeID, MountPath: "/115"}}}} + d1 := s.setGroups(id1, g, 100) + if !d1.verify() { + t.Fatal("authored groups doc must verify") + } - // On a third node, apply A then B: B (higher version) must win. - s := newStore() - if got := s.merge(rA); got != mergeApplied { - t.Fatalf("first merge = %v, want applied", got) + // A peer's newer doc dominates. + d2 := groupDoc{ + Groups: append(g, group{ID: "g2", Name: "ali", Members: []member{{NodeID: id2.NodeID, MountPath: "/ali"}}}), + Version: d1.Version + 5, + Origin: id2.NodeID, + OriginPub: id2.Pub, + UpdatedAt: 200, } - if got := s.merge(rB); got != mergeApplied { - t.Fatalf("higher-version merge = %v, want applied", got) + d2.Sig = id2.sign(d2.signingBytes()) + if !s.mergeGroups(&d2) { + t.Fatal("newer groups doc should be adopted") } - cur, _ := s.get("/m") - if cur.Origin != nodeB.NodeID { - t.Fatal("LWW: higher Lamport version (B) should win") + if len(s.groupList()) != 2 { + t.Fatalf("expected 2 groups after merge, got %d", len(s.groupList())) } - - // Re-merging A (lower version) is ignored. - if got := s.merge(rA); got != mergeIgnored { - t.Fatalf("stale merge = %v, want ignored", got) + // An older doc is ignored. + if s.mergeGroups(&d1) { + t.Fatal("older groups doc must be ignored") } - - // Merging an identical-content record is ignored (idempotent). - dupB := *rB - if got := s.merge(&dupB); got != mergeIgnored { - t.Fatalf("duplicate-content merge = %v, want ignored", got) + // Tampered doc rejected. + d2.Groups[0].Name = "tampered" + if d2.verify() { + t.Fatal("tampered groups doc must fail verification") } } -func TestMergeTombstone(t *testing.T) { +func TestGroupsForMount(t *testing.T) { id, _ := newIdentity() s := newStore() - s.localChange(id, mkCfg("/d", `{"t":"1"}`), 1) - del, ok := s.localDelete(id, "/d", 2) - if !ok { - t.Fatal("delete should produce a tombstone") + s.setGroups(id, []group{ + {ID: "g1", Members: []member{{NodeID: "N1", MountPath: "/a"}, {NodeID: "N2", MountPath: "/a"}}}, + {ID: "g2", Members: []member{{NodeID: "N1", MountPath: "/b"}}}, + }, 1) + if got := s.groupsForMount("N1", "/a"); len(got) != 1 || got[0].ID != "g1" { + t.Fatalf("groupsForMount mismatch: %#v", got) } - - other := newStore() - other.merge(mustRecord(t, id, "/d", `{"t":"1"}`, 1)) - if got := other.merge(del); got != mergeTombstone { - t.Fatalf("tombstone merge = %v, want tombstone", got) - } - cur, _ := other.get("/d") - if !cur.Tombstone { - t.Fatal("record should be tombstoned after merge") + if got := s.groupsForMount("N3", "/a"); len(got) != 0 { + t.Fatal("unknown node should match no groups") } } -func mustRecord(t *testing.T, id *identity, mount, addition string, now int64) *record { - t.Helper() +// ---- credential record LWW + idempotency ---- + +func TestCredRecordMergeLWW(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() s := newStore() - r, ok := s.localChange(id, mkCfg(mount, addition), now) - if !ok { - t.Fatal("failed to build record") + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + r1, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 1) + if !ok || !r1.verify() { + t.Fatal("first credential change should be recorded and verify") + } + // idempotent: same payload again -> no-op. + if _, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 2); ok { + t.Fatal("identical credential must be a no-op") + } + + // peer record with higher version dominates. + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + r2 := &credRecord{GroupID: "g1", OriginDriver: "115", Fields: []string{"refresh_token"}, Payload: payloadB, Version: r1.Version + 10, Origin: id2.NodeID, OriginPub: id2.Pub, CredHash: credHash(payloadB)} + r2.Sig = id2.sign(r2.signingBytes()) + if !r2.verify() { + t.Fatal("peer record should verify") + } + if !s.mergeCred(r2) { + t.Fatal("newer peer credential should win") + } + cur, _ := s.getCred("g1") + if cur.CredHash != credHash(payloadB) { + t.Fatal("store should hold the newer credential") + } + // older loses. + if s.mergeCred(r1) { + t.Fatal("older credential must be ignored") + } + // idempotent merge of identical hash. + if s.mergeCred(r2) { + t.Fatal("merging identical credential must be a no-op") } - return r } -func TestApplyToPreservesLocalRuntimeFields(t *testing.T) { - cfg := mkCfg("/x", `{"t":"shared"}`) - dst := &model.Storage{ID: 42, Status: "work", Addition: `{"t":"old"}`} - cfg.applyTo(dst) - if dst.ID != 42 || dst.Status != "work" { - t.Fatal("applyTo must preserve local ID/Status") +func TestCredRecordVerifyRejectsForgery(t *testing.T) { + id, _ := newIdentity() + other, _ := newIdentity() + payload := map[string]json.RawMessage{"token": json.RawMessage(`"x"`)} + r := &credRecord{GroupID: "g", Payload: payload, CredHash: credHash(payload), Version: 1, Origin: id.NodeID, OriginPub: id.Pub} + r.Sig = id.sign(r.signingBytes()) + // claim someone else's origin without their key. + r.Origin = other.NodeID + if r.verify() { + t.Fatal("a record claiming a foreign origin must fail verification") } - if dst.Addition != `{"t":"shared"}` { - t.Fatalf("applyTo must overwrite Addition, got %q", dst.Addition) +} + +// ---- inventory ---- + +func TestInventoryMergeAndDialTargets(t *testing.T) { + s := newStore() + n1 := &nodeInfo{NodeID: "N1", Addr: "https://n1", Version: 1} + s.mergeInventory(n1, 1000) + // newer version replaces. + s.mergeInventory(&nodeInfo{NodeID: "N1", Addr: "https://n1b", Version: 2}, 1001) + // older ignored (but liveness refreshed). + s.mergeInventory(&nodeInfo{NodeID: "N1", Addr: "https://old", Version: 1}, 1002) + list := s.inventoryList() + if len(list) != 1 || list[0].Addr != "https://n1b" { + t.Fatalf("inventory LWW failed: %#v", list) + } + s.mergeInventory(&nodeInfo{NodeID: "SELF", Addr: "https://self", Version: 1}, 1003) + addrs := s.dialableAddrs("SELF") + if len(addrs) != 1 || addrs[0] != "https://n1b" { + t.Fatalf("dialableAddrs should exclude self: %#v", addrs) } } diff --git a/internal/cluster/status.go b/internal/cluster/status.go new file mode 100644 index 0000000000..ff8a5c607f --- /dev/null +++ b/internal/cluster/status.go @@ -0,0 +1,222 @@ +package cluster + +// This file builds the redaction-safe overview the admin UI renders: the full +// peer roster with liveness, every sync group with its members and current +// credential state, live connections, recent activity, and rollup stats. No +// secrets (credential payloads) are ever included. + +// MemberView is one storage participating in a group, annotated for display. +type MemberView struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + MountPath string `json:"mount_path"` + Online bool `json:"online"` + Present bool `json:"present"` // the node currently advertises this mount + IsOrigin bool `json:"is_origin"` // authored the group's current credential + Self bool `json:"self"` +} + +// GroupView is a sync group plus its current (no-secret) credential state. +type GroupView struct { + ID string `json:"id"` + Name string `json:"name"` + Members []MemberView `json:"members"` + Fields []string `json:"fields"` // credential field names being synced + CredHash string `json:"cred_hash"` // short, for "same/different" comparison + Version uint64 `json:"version"` // credential record version + Origin string `json:"origin"` // node id that authored the credential + UpdatedAt int64 `json:"updated_at"` // last credential change + HasCred bool `json:"has_cred"` // a credential has been observed/shared +} + +// NodeView is one cluster node with its advertised storages and liveness. +type NodeView struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Self bool `json:"self"` + Online bool `json:"online"` + LastSeen int64 `json:"last_seen"` + Storages []storageInfo `json:"storages"` +} + +// ConnView is one live peer connection. +type ConnView struct { + NodeID string `json:"node_id"` + Addr string `json:"addr"` + Outbound bool `json:"outbound"` + Since int64 `json:"since"` +} + +// EventView is one entry in the activity log. +type EventView struct { + Time int64 `json:"time"` + Kind string `json:"kind"` // share | apply | groups | error + GroupID string `json:"group_id"` + Detail string `json:"detail"` +} + +// StatsView is the rollup shown at the top of the cluster panel. +type StatsView struct { + NodesTotal int `json:"nodes_total"` + NodesOnline int `json:"nodes_online"` + GroupsTotal int `json:"groups_total"` + CredsTotal int `json:"creds_total"` + Connections int `json:"connections"` +} + +// Status is the complete admin overview of the cluster. +type Status struct { + NodeID string `json:"node_id"` + Label string `json:"label"` + Addr string `json:"addr"` + Enabled bool `json:"enabled"` + Active bool `json:"active"` + Nodes []NodeView `json:"nodes"` + Groups []GroupView `json:"groups"` + Connections []ConnView `json:"connections"` + Events []EventView `json:"events"` + Stats StatsView `json:"stats"` +} + +// isOnline reports whether a node id has a live connection or was heard from +// recently. +func (m *Manager) isOnline(nodeID string, seenAt int64) bool { + if nodeID == m.id.NodeID { + return true + } + if m.conns.hasNode(nodeID) { + return true + } + return seenAt > 0 && now()-seenAt < peerLivenessSec +} + +// Status returns a redaction-safe snapshot for the admin UI. +func (m *Manager) Status() Status { + cfg := m.cfgStore.get() + + // Roster: inventory + a freshly-computed self entry. + self := m.selfNodeInfo() + byNode := map[string]nodeInfo{} + for _, n := range m.state.inventoryList() { + byNode[n.NodeID] = n + } + selfEntry := *self + selfEntry.seenAt = now() + byNode[self.NodeID] = selfEntry + + mountPresent := func(nodeID, mount string) bool { + n, ok := byNode[nodeID] + if !ok { + return false + } + for _, s := range n.Storages { + if s.MountPath == mount { + return true + } + } + return false + } + labelOf := func(nodeID string) string { + if n, ok := byNode[nodeID]; ok && n.Label != "" { + return n.Label + } + return shortNode(nodeID) + } + + nodes := make([]NodeView, 0, len(byNode)) + online := 0 + for id, n := range byNode { + isSelf := id == m.id.NodeID + on := m.isOnline(id, n.seenAt) + if on { + online++ + } + lbl := n.Label + if lbl == "" { + lbl = shortNode(id) + } + nodes = append(nodes, NodeView{ + NodeID: id, + Label: lbl, + Addr: n.Addr, + Self: isSelf, + Online: on, + LastSeen: n.seenAt, + Storages: n.Storages, + }) + } + sortNodeViews(nodes) + + // Groups + credential state. + groups := make([]GroupView, 0) + for _, g := range m.state.groupList() { + gv := GroupView{ID: g.ID, Name: g.Name} + if cr, ok := m.state.getCred(g.ID); ok { + gv.HasCred = true + gv.Fields = cr.Fields + gv.CredHash = shortHash(cr.CredHash) + gv.Version = cr.Version + gv.Origin = cr.Origin + gv.UpdatedAt = cr.UpdatedAt + } + for _, mem := range g.Members { + n := byNode[mem.NodeID] + gv.Members = append(gv.Members, MemberView{ + NodeID: mem.NodeID, + Label: labelOf(mem.NodeID), + MountPath: mem.MountPath, + Online: m.isOnline(mem.NodeID, n.seenAt), + Present: mountPresent(mem.NodeID, mem.MountPath), + IsOrigin: gv.HasCred && mem.NodeID == gv.Origin, + Self: mem.NodeID == m.id.NodeID, + }) + } + groups = append(groups, gv) + } + + // Connections. + var conns []ConnView + for _, c := range m.conns.all() { + conns = append(conns, ConnView{NodeID: c.nodeID, Addr: c.addr, Outbound: c.outbound, Since: c.since}) + } + + return Status{ + NodeID: m.id.NodeID, + Label: cfg.Label, + Addr: trimURL(cfg.Addr), + Enabled: cfg.Enabled, + Active: cfg.active(), + Nodes: nodes, + Groups: groups, + Connections: conns, + Events: m.eventList(), + Stats: StatsView{ + NodesTotal: len(nodes), + NodesOnline: online, + GroupsTotal: len(groups), + CredsTotal: len(m.state.credSnapshot()), + Connections: len(conns), + }, + } +} + +func sortNodeViews(ns []NodeView) { + // self first, then by label/id. + for i := 1; i < len(ns); i++ { + for j := i; j > 0; j-- { + a, b := ns[j-1], ns[j] + less := false + if b.Self && !a.Self { + less = true + } else if a.Self == b.Self { + less = b.Label < a.Label + } + if less { + ns[j-1], ns[j] = ns[j], ns[j-1] + } else { + break + } + } + } +} diff --git a/internal/cluster/transport.go b/internal/cluster/transport.go index 41e1e8ca28..f8752414a2 100644 --- a/internal/cluster/transport.go +++ b/internal/cluster/transport.go @@ -10,17 +10,27 @@ import ( ) // syncMessage is the inner (encrypted) protocol payload exchanged between nodes. -// One message type carries push, anti-entropy announce, and pull in a single -// round trip: -// - Records: full configs the sender is offering (push, or reply to a Want). -// - Digests: compact (no-secrets) advert of everything the sender holds, so the -// receiver can detect what it is missing. -// - Wants: mount paths the sender wants the receiver to send back in full. +// A single struct carries every interaction (hello, inventory/PEX, group-doc +// sync, credential push/pull, and anti-entropy) so one frame format covers them +// all. Unused fields are omitted on the wire. +// +// - Node: the sender's own inventory entry (who am I, my storages, addr). +// - Nodes: relayed inventory entries + advertised peer addresses (PEX). +// - Groups: the shared sync-group document (LWW). +// - GroupsVer: the sender's groups version, for cheap anti-entropy. +// - Creds: credential records being offered (push, or reply to a Want). +// - CredDigests: compact, no-secret advert of the credential records the sender +// holds, so the receiver can detect what it is missing. +// - Wants: group ids the sender wants the receiver to send creds for. type syncMessage struct { - Type string `json:"type"` // "push" | "announce" | "pull" | "reply" - Records []*record `json:"records,omitempty"` - Digests []digest `json:"digests,omitempty"` - Wants []string `json:"wants,omitempty"` + Type string `json:"type"` // "hello" | "push" | "announce" | "pull" | "reply" + Node *nodeInfo `json:"node,omitempty"` + Nodes []*nodeInfo `json:"nodes,omitempty"` + Groups *groupDoc `json:"groups,omitempty"` + GroupsVer uint64 `json:"groups_ver,omitempty"` + Creds []*credRecord `json:"creds,omitempty"` + CredDigests []credDigest `json:"cred_digests,omitempty"` + Wants []string `json:"wants,omitempty"` } // envelope is the outer, on-the-wire structure. Its Cipher is the syncMessage diff --git a/server/handles/cluster.go b/server/handles/cluster.go index dbb195eee9..0b20bace78 100644 --- a/server/handles/cluster.go +++ b/server/handles/cluster.go @@ -76,3 +76,23 @@ func ClusterStatus(c *gin.Context) { } common.SuccessResp(c, m.Status()) } + +// ClusterSetGroups replaces the cluster-shared sync-group document. Admin only. +func ClusterSetGroups(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + var req struct { + Groups []cluster.GroupSpec `json:"groups"` + } + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := m.SetGroups(req.Groups); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, m.Status()) +} diff --git a/server/router.go b/server/router.go index f919672ec4..26deace696 100644 --- a/server/router.go +++ b/server/router.go @@ -160,6 +160,7 @@ func admin(g *gin.RouterGroup) { clusterGrp.GET("/config", handles.ClusterGetConfig) clusterGrp.POST("/config", handles.ClusterSetConfig) clusterGrp.GET("/status", handles.ClusterStatus) + clusterGrp.POST("/groups", handles.ClusterSetGroups) // Admin-only user operations (create/delete/role-affecting). The profile-edit // endpoints (list/get/update) live in a separate group that also accepts a From f09087fa325633977fd97c3de4dea7fa58999d56 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 04:28:31 +0800 Subject: [PATCH 087/107] docs(cluster): rewrite deep-dive for credential-only redesign --- docs/cluster-sync.md | 530 +++++++++++-------------------------------- 1 file changed, 129 insertions(+), 401 deletions(-) diff --git a/docs/cluster-sync.md b/docs/cluster-sync.md index bff539ec73..5f7c60a5ea 100644 --- a/docs/cluster-sync.md +++ b/docs/cluster-sync.md @@ -1,401 +1,129 @@ -# Cluster Storage Config Sync — Deep Dive - -Package: `internal/cluster` -Status: implemented, integration test in progress (2026-06-21) -Journal entry: [2026-06-21 — Cluster Storage Config Sync Plugin](../../JOURNAL.md) - ---- - -## Overview - -The cluster sync plugin lets multiple OpenList nodes securely share storage driver -configurations in real time. Key properties: - -- **Cryptographically authenticated** — pre-shared key (PSK) + per-node ed25519 identity -- **NAT-friendly** — nodes behind NAT dial out; a persistent WebSocket holds the NAT mapping -- **CRDT-consistent** — Last-Writer-Wins, idempotent, tombstone deletes, no churn on identical content -- **Health-gated** — broken / expired tokens never leave the originating node - ---- - -## Package Layout - -```text -internal/cluster/ - crypto.go HKDF key derivation, AES-256-GCM seal/open, ed25519 identity - identity.go Seed persistence (loadOrCreateIdentity) - state.go CRDT store: syncableStorage, record, digest, merge - config.go Config struct, configStore (atomic save), active/peerList/shouldShare - transport.go syncMessage, envelope, sealEnvelope/openEnvelope, replayCache - conn.go peerConn, connRegistry, dialPeer, ServeWS, dialSupervisor, ensureDial - engine.go Manager (Default singleton), Init/Start/Stop, hook wiring, relay logic - *_test.go Unit tests — all passing -``` - ---- - -## Cryptographic Design - -### Key Derivation - -```text -PSK (user-supplied string) - └─► HKDF-SHA256 - salt = "openlist-cluster" - info = "openlist-cluster-aead-v1" - length = 32 bytes - └─► AES-256-GCM AEAD key -``` - -HKDF provides domain separation — the same PSK could be re-used with different `info` -strings for future sub-keys without collisions. - -### Wire Frame - -Every message is wrapped in a sealed envelope: - -```text -[nonce: 12 bytes][GCM ciphertext + tag][AAD: frame type byte] -``` - -- `sealEnvelope(key, plaintext, aad)` → `envelope{Nonce, Ciphertext}` -- `openEnvelope(key, env, aad)` → plaintext or error -- Only PSK holders can seal or open — unauthenticated nodes see only opaque bytes. - -### Replay Protection - -`replayCache` tracks seen nonces within a sliding time window. A frame is rejected if: - -- its nonce has been seen before (replay), or -- its embedded timestamp falls outside the acceptance window (stale / future frame). - -### Node Identity - -Each node generates an ed25519 keypair on first start. The 32-byte seed is persisted at -`/cluster/identity.key` (mode 0600). - -```text -seed (32 B, persisted) - └─► ed25519 keypair (newIdentity / identityFromSeed) - -node_id = base32( sha256(pubkey)[:16] ) — 16 uppercase characters -``` - -Sync records are self-authenticating: - -```text -record.OriginPub — sender's public key (32 bytes) -record.Sig — ed25519 signature over content hash -record.verify() checks: - 1. nodeIDFromPub(OriginPub) == record.Origin - 2. sha256(canonical JSON) matches embedded ContentHash - 3. ed25519.Verify(OriginPub, ContentHash, Sig) -``` - -A node that does not know the PSK cannot open envelopes. A node that knows the PSK but -forges a record will fail `record.verify()` because it cannot sign with the origin node's -private key. - ---- - -## CRDT State Machine - -### Keying Strategy - -The CRDT is keyed by **content hash** — a SHA-256 of the storage configuration's canonical -JSON with identity-mutable fields excluded: - -```text -excluded: ID, Status, Modified -included: driver name, mount path, all driver-specific config fields -``` - -Consequences: - -- Two storages with identical config but different IDs share the same hash — idempotent, - no churn. -- A token refresh changes the config → new hash → a real LWW update is generated and - propagated. -- Renaming a mount path changes the hash — treated as delete + create (tombstone + new - record). - -### Record Lifecycle - -```text -localChange(storage) - ├─ health check: skip if storage.Status == StatusBroken or token appears invalid - ├─ compute contentHash - ├─ if store already has same hash at same-or-higher Lamport clock → no-op (idempotent) - └─ create signed record, bump Lamport clock, insert into store - -localDelete(storage) - └─ create tombstone record (Deleted=true), signed, bump clock, insert - -merge(remote record) - ├─ verify signature - ├─ if local has same contentHash at higher-or-equal clock → discard (dominates) - ├─ if tombstone + local has live record at lower clock → apply tombstone - └─ otherwise apply: insert/update store entry -``` - -`record.dominates(other)` implements the LWW tiebreak: higher Lamport clock wins; equal -clocks tiebreak by node_id lexicographic order (deterministic, no coin flip needed). - -### Persistence - -`state.json` in `/cluster/` is written atomically (tmp file + rename) on every -change. `snapshot()` serialises the full store. `load()` re-hydrates on startup, so -nodes survive restarts without losing sync state. - ---- - -## Transport Layer - -### Design Choice: Persistent WebSocket over Stateless HTTP - -Stateless HTTP POST was considered and rejected: it cannot reach nodes behind NAT because -the caller must initiate — a node behind NAT has no publicly reachable address for peers -to POST to. - -**Persistent WebSocket** solves this: the NAT'd node dials out, the TCP connection is -held open, and the remote peer can push frames back at any time over the same connection. -Ping keepalives prevent the NAT mapping from timing out. - -### Connection Model - -```text -┌──────────────┐ ┌──────────────┐ -│ NAT'd node │ ── WS dial ──────────► │ Reachable B │ -│ │ ◄─────────── push/relay ─ │ -└──────────────┘ └──────┬───────┘ - │ relay - ┌──────▼───────┐ - │ Reachable C │ - └──────────────┘ -``` - -- `dialPeer(url)` — dials and blocks until the connection closes, then returns so - `dialSupervisor` can retry with exponential backoff. -- `ServeWS(c *gin.Context)` — upgrades an HTTP request to WebSocket and registers the - connection in `connRegistry`. -- `ensureDial(peer)` — idempotent: if a dial goroutine is already running for `peer`, - does nothing. -- `relay(records, excludeConn)` — after `applyRecords` returns the set of newly applied - records, `relay` forwards them to all other connected peers (excluding the sender to - prevent echo). - -### Keepalive Constants - -| Constant | Value | Purpose | -|---|---|---| -| `pingPeriod` | 54 s | How often the server sends a WS ping frame | -| `pongWait` | 60 s | How long to wait for a pong before closing | -| `writeWait` | 10 s | Deadline for a single write operation | -| `dialBackoffMin` | 2 s | Initial retry delay after dial failure | -| `dialBackoffMax` | 5 min | Maximum retry delay (exponential cap) | -| `sendQueueLen` | 128 | Per-connection outbound channel depth | -| `maxFrameBytes` | 16 MB | Maximum inbound frame size | - -### Connection Registry - -`connRegistry` is a goroutine-safe map of active `*peerConn` entries, keyed by connection -ID. A node_id is bound to a connection once identity is established during the WS -handshake (`bind`). `hasNode(nodeID)` lets the engine avoid duplicate dials. - -### URL Normalisation - -`wsURL(rawURL, path)` converts HTTP base URLs to WebSocket URLs: - -```text -http://host/... → ws://host/api/cluster/ws -https://host/... → wss://host/api/cluster/ws -``` - ---- - -## Engine (Manager) - -`Manager` is the central coordinator. `Default` is the package-level singleton initialised -by `InitClusterSync()` in `internal/bootstrap/cluster.go`. - -### Lifecycle - -```text -InitClusterSync() called during Init(), after InitPlugins() - └─ Manager.Init(dataDir) - ├─ loadOrCreateIdentity - ├─ load configStore - └─ load state.json (if present) - -StartClusterSync() called during Start(), after LoadStorages() - └─ Manager.Start(ctx) - ├─ seedLocalStorages() — feed local storages into CRDT on first start - ├─ announceLoop() — periodic full-state announce to all peers - ├─ dialSupervisor() — maintain outbound WS connections to configured peers - └─ register storage hooks -``` - -### Storage Hooks - -Two hooks wire the engine into the storage lifecycle: - -| Hook event | Source | Engine action | -|---|---|---| -| `"update"` | `saveDriverStorage` (after token refresh) | `onStorageHook` → `localChange` → broadcast | -| `"token-invalid"` | `NotifyStorageTokenInvalid` | `onStorageHook` → skip propagation (health gate) | - -The hook fires as a goroutine (`go callStorageHooks(...)`) so it never blocks the caller's -hot path. - -### Announce Loop - -Every `AnnounceIntervalSec` (default 30 s) the engine broadcasts a digest summary of its -CRDT store to all peers. Peers that have diverged request missing records; peers that are -up to date discard the digest without generating traffic. This provides eventual consistency -for nodes that were offline during a change. - -### Apply + Relay - -```go -func (m *Manager) applyRecords(records []record) []record -``` - -- Verifies each record's signature. -- Calls `store.merge()` for each. -- Returns the subset of records that were actually applied (i.e., advanced the CRDT state). -- The caller (`handleFrame`) passes the applied set to `relay()`. - ---- - -## API Endpoints - -Registered in `server/router.go`: - -| Method | Path | Handler | Auth | -|---|---|---|---| -| `GET` | `/api/cluster/ws` | `ClusterWS` | API key (peer-to-peer) | -| `GET` | `/api/admin/cluster/config` | `ClusterGetConfig` | Admin | -| `POST` | `/api/admin/cluster/config` | `ClusterSetConfig` | Admin | -| `GET` | `/api/admin/cluster/status` | `ClusterStatus` | Admin | - -`ClusterSetConfig` calls `Manager.SetConfig` which atomically saves the new config and -calls `connRegistry.closeAll()` — existing connections drop and `dialSupervisor` reconnects -with the new peer list. - ---- - -## Configuration Reference - -Persisted at `/cluster/config.json`. Editable via the admin UI or directly as JSON. - -```json -{ - "enabled": true, - "key": "", - "peers": ["https://peer-b.example.com"], - "share_drivers": [], - "share_mounts": ["/cluster-test"], - "share_deletes": false, - "apply_remote": true, - "announce_interval_sec": 30, - "request_timeout_sec": 10 -} -``` - -| Field | Type | Meaning | -|---|---|---| -| `enabled` | bool | Master switch — `false` disables all sync activity | -| `key` | string | Pre-shared key; all cluster members must use the same value | -| `peers` | `[]string` | Base URLs of peer nodes this node dials out to | -| `share_drivers` | `[]string` | Only share storages using these driver names (empty = all) | -| `share_mounts` | `[]string` | Only share storages whose mount path matches a prefix (empty = all) | -| `share_deletes` | bool | Whether to propagate tombstone (delete) records | -| `apply_remote` | bool | Whether to apply incoming records to the local database | -| `announce_interval_sec` | int | Seconds between periodic full-state announce broadcasts | -| `request_timeout_sec` | int | HTTP / WS dial timeout | - -`active()`, `peerList()`, and `shouldShare()` are **pointer receivers** on `*Config` to -ensure callers always read the live configuration after a `SetConfig` call. - ---- - -## Frontend UI (`ClusterConfig.tsx`) - -Located at `src/pages/manage/plugins/ClusterConfig.tsx`, rendered inside the Plugins -management page. - -Sections: - -1. **Enable switch** — master on/off toggle -2. **Pre-shared Key** — password field (masked) -3. **Peers** — textarea, one URL per line -4. **Scope** — `share_drivers` (comma-separated), `share_mounts` (comma-separated) -5. **Behaviour** — `apply_remote`, `share_deletes` switches; interval number inputs -6. **Live Status** — polls `/api/admin/cluster/status`; shows `node_id`, connected peer - count, and number of synced records - ---- - -## Operational Notes - -### Data Files - -```text -/cluster/ - identity.key 32-byte ed25519 seed (mode 0600, never share) - config.json cluster configuration - state.json CRDT store snapshot (rebuilt on startup if missing) -``` - -### Safe Scoping for Testing - -Set `share_mounts` to a single test path (e.g. `/cluster-test`) so that production -storages are never touched by the sync mechanism. - -### NAT'd Node Setup - -A NAT'd node only needs to list reachable peers in `peers`. It dials out on startup and -after each disconnect. It does not need an open inbound port. - -### Token Refresh Propagation - -When `saveDriverStorage` is called after a token refresh, the `"update"` hook fires -asynchronously. The engine calls `localChange` (health-checked) and, if the content hash -changed, broadcasts to all connected peers. Peers receiving the update call `applyToLocal` -→ `op.UpdateStorage`, which persists the fresh token without triggering another hook cycle -(the content hash will be identical on the receiving side after apply). - -### Avoiding Churn - -The idempotency guarantee means that if node A and node B both store the same token -(same content hash), neither will generate a CRDT update for the other. The announce -digest handshake confirms they agree; no record is transmitted. - -### Broken Token Gate - -`localChange` checks storage health before accepting a record into the CRDT store. If the -storage status is `StatusBroken`, or if the token fields appear invalid, the record is -silently dropped. This prevents a node that has entered a broken-token state from -poisoning healthy peers. - ---- - -## Testing - -| File | Coverage | -|---|---| -| `crypto_test.go` | AEAD round-trip, wrong key, tampered AAD, identity binding | -| `state_test.go` | LWW merge, idempotency (same content hash), tombstone, Lamport ordering | -| `config_test.go` | `active()` / `peerList()` pointer-receiver correctness, `shouldShare()` filter | -| `transport_test.go` | Envelope replay rejection, stale-timestamp rejection | -| `conn_test.go` | `wsURL` conversion (http→ws, https→wss), `connRegistry` add/bind/remove/closeAll | - -All tests pass as of rev `2c8a9827`. - ---- - -## Related Docs - -- [streaming-and-caching.md](streaming-and-caching.md) — RangeReader, SeekableStream pitfalls -- [new-apis-2026-06-21.md](new-apis-2026-06-21.md) — storage loading progress, permission bit 16, video_play, favorites -- [architecture.md](architecture.md) — driver system, request flow, packages, startup sequence +# Cluster credential sync + +A self-contained plugin (`internal/cluster`) that lets multiple OpenList nodes +share **storage credentials** (tokens / cookies / secrets) in real time, so when +one node refreshes a token the others adopt it automatically. Disabled by +default; configured per-node under **Manage → Plugins → Cluster credential +sharing**. + +The design satisfies three hard requirements from the product owner: + +1. **Sync ONLY the credential** — never the whole storage config. Mount path, + root folder, cache, order, proxy and every other field stay local to each + node. ("除了 key 其他的都不用同步") +2. **Manual, multipartite pairing** — the admin explicitly groups the storages + (across any number of nodes) that represent the same account. A group is a + connected component of a k-partite graph. ("手动选择 storage … 多分图") +3. **Auto-discovery** — no hand-maintained peer list. Any node that authenticates + with the shared key joins; reachable nodes advertise an address and the rest + of the mesh is learned automatically. ("远程连过来，认证过了就加入 peer") + +## Layers + +| File | Responsibility | +|------|----------------| +| `crypto.go` | PSK → HKDF-SHA256 → AES-256-GCM; ed25519 identity (`node_id = base32(sha256(pubkey)[:16])`); sign/verify | +| `identity.go` | Persist a 32-byte seed at `/cluster/identity.key` (0600) | +| `creds.go` | Extract / overlay / hash the **credential subset** of a driver's Addition | +| `state.go` | CRDTs: sync-group document, per-group credential records, node inventory | +| `transport.go` | `syncMessage` wire form + AEAD `envelope` + replay cache | +| `conn.go` | Persistent WebSocket peer connections, registry, dial supervisor, PEX dialing | +| `engine.go` | `Manager`: lifecycle, storage hook, push/pull/apply, relay, anti-entropy, events | +| `status.go` | Redaction-safe overview (nodes/groups/connections/events/stats) for the UI | +| `config.go` | Per-node `Config` (`enabled/key/label/addr/seeds/apply_remote`) + atomic store | + +## What a "credential" is + +OpenList drivers do not tag credential fields, so `creds.go` identifies them by +name over the Addition JSON keys: any key whose normalized name contains +`token, cookie, password, passwd, secret, auth, session, refresh, access, +credential, apikey, appkey, privatekey, signkey, ticket, passport`. Structural +fields (`root_folder_id`, `order_by`, …) never match, so they are never shared. +`applyCreds` overlays only those keys onto a peer's Addition and reports whether +anything actually changed — identical credentials cause no write, no re-init, no +churn. `credHash` (sha256 of the canonicalized credential map) is the +idempotency key. + +## Sync groups + +`group{ id, name, members[]{ node_id, mount_path } }`. The whole set is a +**single LWW document** (`groupDoc`) — signed by its author, versioned by a +Lamport clock, tie-broken by origin id. Admin edits are rare, so whole-doc LWW is +simple and convergent. The document is gossiped to every node; `pruneCreds` +drops credential records for groups that no longer exist. + +## Credential records + +Per group, `credRecord{ group_id, origin_driver, fields[], cred_hash, payload, +version, origin, origin_pub, sig }`. The secret `payload` only ever travels +inside the AEAD-sealed envelope. Records are signed by their origin so they stay +authentic across relays. Merge is LWW by `(version, origin, cred_hash)`, and +idempotent: an identical `cred_hash` is ignored regardless of version. + +### Flow + +1. A driver refreshes its token → `op.MustSaveDriverStorage` → `callStorageHooks("update")`. +2. `onStorageHook` finds the groups containing `(this_node, mount)`. If the + storage is healthy (`status == work`) it extracts the credential, and for each + group `localCredChange` authors a new record (skipped if the hash is + unchanged) and broadcasts a `push`. If the storage is **broken** + (`status != work`) or a driver fired `token-invalid`, it does **not** push — + it `pullGroup`s to recover a good credential from peers instead. ("坏 token 不推") +3. A peer receives the record, `verify`s the signature, `mergeCred`s under LWW, + then `applyCredRecord` overlays the payload onto its own member mounts + (`op.UpdateStorage`, which re-inits the driver with the new token). The + resulting `update` hook is a no-op because the credential now matches the + record's hash — no echo, no loop. +4. The record is `relay`ed to other connections, so two NAT'd nodes converge + through a common reachable peer. + +Health gating ensures a broken token is never propagated; idempotency ensures a +node that already holds the same token does nothing. ("其他节点看到一样的 token 就不用变") + +## Discovery & transport + +The link is a persistent WebSocket at `/api/cluster/ws` (NAT-friendly: a node +behind NAT dials out and keeps the link open; a reachable node accepts inbound +and relays). Authentication is the PSK: any frame that opens the AEAD envelope +admits the sender as a peer. + +- Each node advertises a `nodeInfo` inventory (its `label`, `addr`, and the + `mount_path`/`driver`/`status` of every storage). Inventory is gossiped and + powers the UI graph + group editor. +- **PEX**: nodes exchange the dialable `addr`s they know; the dial supervisor + opens a connection to any advertised peer it is not already connected to. A + `seed` URL is only needed to bootstrap the very first join. +- Per-frame freshness: timestamp window + nonce replay cache. + +## API + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/api/cluster/ws` | Peer WebSocket endpoint (PSK-authenticated) | +| GET | `/api/admin/cluster/config` | Config (key redacted) + live status | +| POST | `/api/admin/cluster/config` | Save config (blank/`********` key keeps current) | +| GET | `/api/admin/cluster/status` | Live status only | +| POST | `/api/admin/cluster/groups` | Replace the sync-group document | + +## Config + +`enabled`, `key` (shared secret), `label` (this node's display name), `addr` +(this node's public URL — advertise to be auto-discovered; leave empty behind +NAT), `seeds` (optional bootstrap URLs), `apply_remote` (adopt peer creds; +default on), `announce_interval_sec`. `active() == enabled && key != ""` — a +reachable node can run accept-only with no seed. + +## State on disk + +`/cluster/identity.key` (seed), `config.json`, `state.json` +(groups + credential records + inventory + Lamport clock). State is written +atomically (temp + rename). + +## Tests + +`*_test.go` cover credential extraction / overlay (node-local fields preserved) / +idempotency, group-doc and credential-record LWW + signature forgery rejection, +inventory merge + dialable-address selection, envelope round-trip / replay / +stale / wrong-key, and `wsURL` / connection registry. A two-node integration run +(discovery → inventory → group propagation → credential overlay on token change → +events) is validated end to end. From 0bfdd259364f5b0f448812f1cb5d0d79ef8d0157 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 16:37:30 +0800 Subject: [PATCH 088/107] docs(frontend): document dist_dir serving + in-memory index.html gotcha --- docs/frontend.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/docs/frontend.md b/docs/frontend.md index f5c31c1f88..5ea8d7837e 100644 --- a/docs/frontend.md +++ b/docs/frontend.md @@ -36,6 +36,81 @@ ReloadStatic() → swap staticFS to new dist, re-render index.html --- +## Serving Priority and the In-Memory index.html Gotcha + +There are three sources of truth for the frontend, in strict priority order: + +| Priority | Source | When active | +|-------------|---------------------------------------------------|--------------------------------------------------------------------| +| 1 (highest) | `dist_dir` (config.json) | Set explicitly; `initStatic()` calls `os.DirFS(conf.Conf.DistDir)` | +| 2 | Watcher-fetched dist (`data/frontend_dist/dist`) | `shouldAutoFetch()` true and a newer rolling release was found | +| 3 (lowest) | Embedded dist (`public/dist/` via `go:embed`) | Default fallback when neither of the above applies | + +**Critical implementation detail — index.html lives in memory, not on disk.** + +`initIndex()` reads `index.html` from `staticFS` (or the CDN) once and stores it in `conf.RawIndexHtml`. `UpdateIndex()` derives `conf.ManageHtml` (for `/@manage/*`) and `conf.IndexHtml` (all other routes) from that raw copy. The SPA catch-all handler writes the in-memory string directly to the response — it never opens the file on disk per-request. + +Consequence: **replacing files on disk does not affect what the server returns for `index.html`** until one of the following happens: + +- The process restarts (which calls `initStatic()` / `initIndex()` again), OR +- The watcher downloads a genuinely newer rolling release and calls `ReloadStatic()`. + +Meanwhile, static assets (`/assets/`, `/images/`, `/streamer/`, `/static/`) **do** serve live from `staticFS` via `http.FS`. So new asset chunks load immediately after you copy files to disk — but the old `index.html` still references the old chunk hashes, causing a mix of old UI and new files. + +**Symptom pattern**: new asset requests return 200; the UI loads stale chunks; the browser console shows hash-mismatch errors or the manage page looks like a previous version. + +`ReloadStatic()` (called by the watcher) fixes both: it calls `staticFS.swap(os.DirFS(distPath))` to update the asset FS, then re-runs `initIndex()` to reload index.html into memory. + +--- + +## Deterministic Deploy Procedure (dist_dir method) + +This is the recommended deployment path when you need a specific frontend build on disk — especially on hosts that cannot reach GitHub (e.g., hosts behind a firewall or with restricted egress). + +``` +# 1. Build locally +cd OpenList-Frontend/ +pnpm install && pnpm build +# Output: dist/ + +# 2. Copy to host, atomically swap +# NEVER delete frontend_dist/ itself — the watcher owns that directory. +# Back up the old dist, then rename the new one into place. +REMOTE_DATA=/root/docker/opdata # host-side path; maps to /opt/openlist/data inside container +CONTAINER_DATA=/opt/openlist/data + +scp -r dist/ user@cfscan:${REMOTE_DATA}/frontend_dist/dist.new +ssh user@cfscan " + cd ${REMOTE_DATA}/frontend_dist + mv dist dist.bak.\$(date +%s) # keep old as backup, never delete it + mv dist.new dist +" + +# 3. Set dist_dir in config.json (container path) +# In config.json on the host, set: +# "dist_dir": "/opt/openlist/data/frontend_dist/dist" +# (the container-internal path, not the host path) + +# 4. Restart ONLY the openlist service — never bare `compose up -d` +ssh user@cfscan "docker restart openlist" +# On restart: initStatic() reads from dist_dir, initIndex() loads the new index.html. +``` + +**Production path mapping** (reference): + +| Host | Host path | Container path | +|--------------------------------|------------------------|------------------------| +| cfscan (op.rnarket.com) | `/root/docker/opdata` | `/opt/openlist/data` | +| txcloud (trans.zoterosync.top) | `/root/docker/data` | `/opt/openlist/data` | + +So for both hosts: `dist_dir = /opt/openlist/data/frontend_dist/dist` + +After restart, `initStatic()` picks up `dist_dir` deterministically — no GitHub access required, no watcher race, no index.html staleness. + +**Why not delete `frontend_dist/` entirely?** The watcher goroutine tracks state inside that directory (`.frontend_version` file, the `dist/` subdirectory). Removing the whole `frontend_dist/` directory races with the watcher. Back up `dist/` as `dist.bak.` instead; that's atomic and the watcher ignores the `.bak` sibling. + +--- + ## Frontend Build Commands ```bash From f1290ef3f952aeebe8fce472f4b85a2b3d3bfce2 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 16:37:31 +0800 Subject: [PATCH 089/107] feat(video): signed same-origin proxy for provider transcoded streams 115 online-play returns HLS on 115's CDN; the browser fetching it directly fails CORS. Add /video_proxy: a signed (sign.Sign over the upstream URL, no open-relay) endpoint that streams an upstream media URL through OpenList, rewriting inner segment/key/sub-playlist URIs in HLS manifests so they also route back through the proxy. FsVideoPlay now wraps every transcoded source URL through it. Unit tests cover URL build/verify, m3u8 rewrite, isM3U8. --- server/handles/fsread.go | 9 ++ server/handles/video_proxy.go | 180 +++++++++++++++++++++++++++++ server/handles/video_proxy_test.go | 141 ++++++++++++++++++++++ server/router.go | 5 + 4 files changed, 335 insertions(+) create mode 100644 server/handles/video_proxy.go create mode 100644 server/handles/video_proxy_test.go diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 531131e1e5..d1928b60da 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -459,6 +459,15 @@ func FsVideoPlay(c *gin.Context) { common.ErrorResp(c, err, 500) return } + // Route every transcoded source through OpenList's signed video proxy so the + // browser fetches them same-origin instead of hitting the provider CDN + // directly (which fails CORS for 115's online-play HLS). + apiURL := common.GetApiUrl(c) + for i := range sources { + if sources[i].URL != "" { + sources[i].URL = BuildVideoProxyURL(apiURL, sources[i].URL) + } + } common.SuccessResp(c, sources) } diff --git a/server/handles/video_proxy.go b/server/handles/video_proxy.go new file mode 100644 index 0000000000..731b4718c9 --- /dev/null +++ b/server/handles/video_proxy.go @@ -0,0 +1,180 @@ +package handles + +import ( + "io" + "net/http" + "net/url" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +// videoProxyPath is the route (relative to conf.URL.Path, sibling to /p) that +// streams an arbitrary signed upstream media URL through OpenList. It exists so +// the browser never fetches a provider's transcoded CDN directly (which fails +// CORS, e.g. 115's online-play HLS); everything is fetched same-origin and the +// provider request happens server-side. +const videoProxyPath = "/video_proxy" + +// BuildVideoProxyURL wraps an upstream media URL in a signed OpenList proxy URL. +// The signature (over the raw upstream URL) prevents the endpoint from acting as +// an open relay: only URLs OpenList itself produced are accepted. +func BuildVideoProxyURL(apiURL, rawURL string) string { + return apiURL + videoProxyPath + + "?url=" + url.QueryEscape(rawURL) + + "&sign=" + url.QueryEscape(sign.Sign(rawURL)) +} + +// isM3U8 reports whether the upstream response is an HLS manifest, by +// content-type or by the upstream URL's path extension. +func isM3U8(contentType, rawURL string) bool { + ct := strings.ToLower(contentType) + if strings.Contains(ct, "mpegurl") { + return true + } + if u, err := url.Parse(rawURL); err == nil { + if strings.HasSuffix(strings.ToLower(u.Path), ".m3u8") { + return true + } + } + return false +} + +// rewriteM3U8 rewrites every segment / sub-playlist / key URI in an HLS manifest +// so they are also fetched through the OpenList video proxy. baseRawURL is the +// upstream URL the manifest was fetched from (used to resolve relative URIs); +// apiURL is the OpenList base used to build the proxy links. Without this, the +// segment URLs inside the manifest would still point at the provider CDN and +// re-introduce the CORS failure we are trying to avoid. +func rewriteM3U8(manifest, baseRawURL, apiURL string) string { + baseURL, _ := url.Parse(baseRawURL) + var b strings.Builder + lines := strings.Split(manifest, "\n") + for i, line := range lines { + if i > 0 { + b.WriteByte('\n') + } + trimmed := strings.TrimSpace(line) + if trimmed == "" { + b.WriteString(line) + continue + } + if strings.HasPrefix(trimmed, "#") { + // Tag line: rewrite any URI="..." attribute (EXT-X-KEY, EXT-X-MAP, + // EXT-X-MEDIA, EXT-X-I-FRAME-STREAM-INF, EXT-X-SESSION-KEY, ...). + b.WriteString(rewriteTagURIs(line, baseURL, apiURL)) + continue + } + // Resource line: a segment or sub-playlist URI on its own line. + b.WriteString(proxyResolve(trimmed, baseURL, apiURL)) + } + return b.String() +} + +// rewriteTagURIs rewrites every URI="..." occurrence on an HLS tag line. +func rewriteTagURIs(line string, baseURL *url.URL, apiURL string) string { + const marker = `URI="` + var b strings.Builder + rest := line + for { + idx := strings.Index(rest, marker) + if idx < 0 { + b.WriteString(rest) + break + } + b.WriteString(rest[:idx+len(marker)]) + rest = rest[idx+len(marker):] + end := strings.IndexByte(rest, '"') + if end < 0 { + b.WriteString(rest) + break + } + b.WriteString(proxyResolve(rest[:end], baseURL, apiURL)) + b.WriteByte('"') + rest = rest[end+1:] + } + return b.String() +} + +// proxyResolve resolves ref against the manifest's base URL (no-op if ref is +// already absolute) and wraps the result in a signed proxy URL. +func proxyResolve(ref string, baseURL *url.URL, apiURL string) string { + abs := ref + if baseURL != nil { + if u, err := url.Parse(ref); err == nil { + abs = baseURL.ResolveReference(u).String() + } + } + return BuildVideoProxyURL(apiURL, abs) +} + +// hopByHopOrRangeHeaders are upstream response headers worth forwarding to the +// client for non-manifest passthrough (segments / progressive mp4). +var passthroughHeaders = []string{ + "Content-Type", + "Content-Length", + "Content-Range", + "Accept-Ranges", + "Last-Modified", + "ETag", +} + +// VideoProxy streams a signed upstream media URL through OpenList. For HLS +// manifests it rewrites the inner URIs to route back through this same endpoint; +// everything else is streamed verbatim (with Range support for seeking). +func VideoProxy(c *gin.Context) { + rawURL := c.Query("url") + signParam := c.Query("sign") + if rawURL == "" { + common.ErrorStrResp(c, "missing url", 400) + return + } + if err := sign.Verify(rawURL, signParam); err != nil { + common.ErrorResp(c, err, 401) + return + } + + req, err := http.NewRequestWithContext(c.Request.Context(), c.Request.Method, rawURL, nil) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // Forward Range so the player can byte-range seek within segments / mp4. + if rng := c.GetHeader("Range"); rng != "" { + req.Header.Set("Range", rng) + } + req.Header.Set("User-Agent", base.UserAgent) + + resp, err := base.HttpClient.Do(req) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + defer resp.Body.Close() + + if isM3U8(resp.Header.Get("Content-Type"), rawURL) { + body, err := io.ReadAll(resp.Body) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + out := rewriteM3U8(string(body), rawURL, common.GetApiUrl(c)) + c.Header("Content-Type", "application/vnd.apple.mpegurl") + c.Header("Cache-Control", "no-cache") + c.String(resp.StatusCode, out) + return + } + + for _, h := range passthroughHeaders { + if v := resp.Header.Get(h); v != "" { + c.Header(h, v) + } + } + c.Status(resp.StatusCode) + if c.Request.Method != http.MethodHead { + _, _ = io.Copy(c.Writer, resp.Body) + } +} diff --git a/server/handles/video_proxy_test.go b/server/handles/video_proxy_test.go new file mode 100644 index 0000000000..735356c701 --- /dev/null +++ b/server/handles/video_proxy_test.go @@ -0,0 +1,141 @@ +package handles + +import ( + "net/url" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/sign" +) + +const testAPIURL = "http://openlist.test" + +// seedSign installs a deterministic signing token in the setting cache so +// sign.Sign / sign.Verify round-trip without touching the database. +func seedSign(t *testing.T) { + t.Helper() + op.Cache.SetSetting(conf.LinkExpiration, &model.SettingItem{Key: conf.LinkExpiration, Value: "0"}) + op.Cache.SetSetting(conf.Token, &model.SettingItem{Key: conf.Token, Value: "test-token"}) + t.Cleanup(func() { op.Cache.ClearAll() }) +} + +// parseProxied pulls the upstream url + sign back out of a proxy link and +// asserts the signature is valid. Returns the decoded upstream URL. +func parseProxied(t *testing.T, proxied string) string { + t.Helper() + if !strings.HasPrefix(proxied, testAPIURL+videoProxyPath+"?") { + t.Fatalf("proxied url has unexpected prefix: %s", proxied) + } + u, err := url.Parse(proxied) + if err != nil { + t.Fatalf("parse proxied url: %v", err) + } + raw := u.Query().Get("url") + s := u.Query().Get("sign") + if raw == "" || s == "" { + t.Fatalf("proxied url missing url/sign params: %s", proxied) + } + if err := sign.Verify(raw, s); err != nil { + t.Fatalf("sign verify failed for %q: %v", raw, err) + } + return raw +} + +func TestBuildVideoProxyURL(t *testing.T) { + seedSign(t) + raw := "https://cdn.115.com/v/movie.m3u8?t=123&k=abc/def" + got := BuildVideoProxyURL(testAPIURL, raw) + + // The upstream URL must be query-escaped (no bare '?'/'&' leaking past the + // first '?'), and must round-trip + verify. + if decoded := parseProxied(t, got); decoded != raw { + t.Fatalf("round-trip mismatch:\n have %q\n want %q", decoded, raw) + } + if strings.Count(got, "?") != 1 { + t.Fatalf("upstream query not escaped, multiple '?': %s", got) + } +} + +func TestIsM3U8(t *testing.T) { + cases := []struct { + ct, rawURL string + want bool + }{ + {"application/vnd.apple.mpegurl", "https://x/y", true}, + {"application/x-mpegURL", "https://x/y", true}, + {"", "https://cdn.115.com/v/movie.m3u8", true}, + {"", "https://cdn.115.com/v/movie.m3u8?token=1", true}, + {"video/mp4", "https://cdn.115.com/v/movie.mp4", false}, + {"binary/octet-stream", "https://cdn.115.com/v/seg001.ts", false}, + } + for _, c := range cases { + if got := isM3U8(c.ct, c.rawURL); got != c.want { + t.Errorf("isM3U8(%q,%q) = %v, want %v", c.ct, c.rawURL, got, c.want) + } + } +} + +func TestRewriteM3U8(t *testing.T) { + seedSign(t) + base := "https://cdn.115.com/hls/720/index.m3u8?auth=xyz" + manifest := strings.Join([]string{ + "#EXTM3U", + "#EXT-X-VERSION:3", + "#EXT-X-KEY:METHOD=AES-128,URI=\"https://key.115.com/k?id=1\",IV=0x00", + "#EXT-X-TARGETDURATION:10", + "#EXTINF:9.009,", + "seg001.ts", + "#EXTINF:9.009,", + "https://cdn.115.com/hls/720/seg002.ts?auth=xyz", + "", + "#EXT-X-ENDLIST", + }, "\n") + + out := rewriteM3U8(manifest, base, testAPIURL) + lines := strings.Split(out, "\n") + + // Structural tags preserved verbatim. + for _, want := range []string{"#EXTM3U", "#EXT-X-VERSION:3", "#EXT-X-TARGETDURATION:10", "#EXTINF:9.009,", "#EXT-X-ENDLIST"} { + if !strings.Contains(out, want) { + t.Errorf("missing preserved tag %q in output", want) + } + } + // Blank line preserved (same line count as input). + if len(lines) != len(strings.Split(manifest, "\n")) { + t.Errorf("line count changed: got %d want %d", len(lines), len(strings.Split(manifest, "\n"))) + } + + // Relative segment resolves against the manifest base, then proxied. + relLine := lines[5] + if got := parseProxied(t, relLine); got != "https://cdn.115.com/hls/720/seg001.ts" { + t.Errorf("relative segment resolved to %q", got) + } + // Absolute segment kept as-is, proxied. + absLine := lines[7] + if got := parseProxied(t, absLine); got != "https://cdn.115.com/hls/720/seg002.ts?auth=xyz" { + t.Errorf("absolute segment resolved to %q", got) + } + + // EXT-X-KEY URI rewritten in place (tag prefix kept, URI proxied). + keyLine := lines[2] + if !strings.HasPrefix(keyLine, "#EXT-X-KEY:METHOD=AES-128,URI=\"") || !strings.HasSuffix(keyLine, ",IV=0x00") { + t.Fatalf("key line structure broken: %s", keyLine) + } + inner := keyLine[strings.Index(keyLine, "URI=\"")+len("URI=\"") : strings.LastIndex(keyLine, "\"")] + if got := parseProxied(t, inner); got != "https://key.115.com/k?id=1" { + t.Errorf("key URI proxied to %q", got) + } + + // No raw provider CDN URL should survive outside an escaped 'url=' param. + for i, l := range lines { + if strings.HasPrefix(l, "#") { + continue + } + if strings.Contains(l, "cdn.115.com") && !strings.Contains(l, "url=") { + t.Errorf("line %d leaks raw CDN url: %s", i, l) + } + } +} diff --git a/server/router.go b/server/router.go index 26deace696..fb963d712c 100644 --- a/server/router.go +++ b/server/router.go @@ -49,6 +49,11 @@ func Init(e *gin.Engine) { g.GET("/p/*path", middlewares.PathParse, signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", middlewares.PathParse, signCheck, handles.Down) g.HEAD("/p/*path", middlewares.PathParse, signCheck, handles.Proxy) + // /video_proxy streams a signed upstream media URL (query-param sign verified + // in-handler) so the browser never fetches a provider's transcoded CDN + // directly, e.g. 115 online-play HLS (avoids CORS). Sibling of /p. + g.GET("/video_proxy", downloadLimiter, handles.VideoProxy) + g.HEAD("/video_proxy", handles.VideoProxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveProxy) From 15726735e4293d97a61faa74003c9212aed90272 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 22:59:22 +0800 Subject: [PATCH 090/107] feat(cluster): admin endpoint to reveal cluster key in plaintext ClusterGetConfig redacts the key to '********' (it is polled live), so add a dedicated admin-only GET /admin/cluster/key returning the real key for the UI's show-key toggle. --- server/handles/cluster.go | 12 ++++++++++++ server/router.go | 1 + 2 files changed, 13 insertions(+) diff --git a/server/handles/cluster.go b/server/handles/cluster.go index 0b20bace78..d07c573185 100644 --- a/server/handles/cluster.go +++ b/server/handles/cluster.go @@ -46,6 +46,18 @@ func ClusterGetConfig(c *gin.Context) { }) } +// ClusterRevealKey returns the cluster pre-shared key in PLAINTEXT for the admin +// UI's "show key" toggle. ClusterGetConfig redacts the key (it is polled live), +// so this dedicated endpoint serves the real value only on explicit request. It +// lives in the admin-only cluster group, same as the other config routes. +func ClusterRevealKey(c *gin.Context) { + m := clusterMgr(c) + if m == nil { + return + } + common.SuccessResp(c, gin.H{"key": m.GetConfig(false).Key}) +} + // ClusterSetConfig persists a new cluster config. A blank or "********" key keeps // the existing key. Admin only. func ClusterSetConfig(c *gin.Context) { diff --git a/server/router.go b/server/router.go index fb963d712c..0b4523d27c 100644 --- a/server/router.go +++ b/server/router.go @@ -163,6 +163,7 @@ func admin(g *gin.RouterGroup) { // Cluster storage-sharing config/status. Admin only. clusterGrp := g.Group("/cluster") clusterGrp.GET("/config", handles.ClusterGetConfig) + clusterGrp.GET("/key", handles.ClusterRevealKey) clusterGrp.POST("/config", handles.ClusterSetConfig) clusterGrp.GET("/status", handles.ClusterStatus) clusterGrp.POST("/groups", handles.ClusterSetGroups) From 13b67b7b28d7ca48f2bdabd1e13bcaa05e5de751 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 22 Jun 2026 23:51:40 +0800 Subject: [PATCH 091/107] feat: generic ObjExtra metadata bag + profile permission gate - model.ObjExtra interface + GetExtra() helper (walks ObjUnwrap chain) - fsread ObjResp.extra: forward-compatible metadata passthrough - 115_open Obj.Extra(): starred/duration/resolution/tags from API json - auth UpdateCurrent: block username/password change without manage_user_info --- drivers/115_open/types.go | 75 +++++++++++++++++++++++++++++++++++++++ internal/model/obj.go | 28 +++++++++++++++ internal/model/user.go | 1 + server/handles/auth.go | 7 ++++ server/handles/fsread.go | 7 ++++ 5 files changed, 118 insertions(+) diff --git a/drivers/115_open/types.go b/drivers/115_open/types.go index 0bd95bfd19..7bc7443ba5 100644 --- a/drivers/115_open/types.go +++ b/drivers/115_open/types.go @@ -55,5 +55,80 @@ func (o *Obj) ModTime() time.Time { return time.Unix(o.Upt, 0) } +// Extra surfaces 115-specific metadata the universal Obj cannot carry: media +// duration (seconds), video resolution label, the starred flag and file tags. +// Only non-empty values are included, so the frontend renders whatever is +// present and ignores the rest — adding/removing keys never breaks the client, +// and a changed 115 payload degrades to "no extra info" rather than an error. +func (o *Obj) Extra() map[string]any { + extra := map[string]any{} + if o.Ism == "1" { + extra["starred"] = true + } + if d := o.durationSec(); d > 0 { + extra["duration"] = d + } + if label := defLabel(o.maxDef()); label != "" { + extra["resolution"] = label + } + if len(o.Fl) > 0 { + tags := make([]string, 0, len(o.Fl)) + for _, t := range o.Fl { + if t.Name != "" { + tags = append(tags, t.Name) + } + } + if len(tags) > 0 { + extra["tags"] = tags + } + } + if len(extra) == 0 { + return nil + } + return extra +} + +// durationSec parses 115's play_long (json.Number, seconds) defensively. +func (o *Obj) durationSec() float64 { + if o.PlayLong == "" { + return 0 + } + f, err := o.PlayLong.Float64() + if err != nil || f <= 0 { + return 0 + } + return f +} + +// maxDef prefers the higher of the two resolution fields 115 reports. +func (o *Obj) maxDef() int64 { + if o.Def2 > o.Def { + return o.Def2 + } + return o.Def +} + +// defLabel maps 115's video-definition code to a short badge. Unknown codes +// (including 0 for non-videos) return "" so no badge is shown. +func defLabel(def int64) string { + switch def { + case 1: + return "SD" + case 2: + return "HD" + case 3: + return "FHD" + case 4: + return "1080P" + case 5: + return "4K" + case 100: + return "原画" + default: + return "" + } +} + var _ model.Obj = (*Obj)(nil) var _ model.Thumb = (*Obj)(nil) +var _ model.ObjExtra = (*Obj)(nil) diff --git a/internal/model/obj.go b/internal/model/obj.go index ba7467f740..22e8de4c3f 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -75,6 +75,15 @@ type Thumb interface { Thumb() string } +// ObjExtra is an optional interface a driver's Obj may implement to surface +// extra, driver-specific metadata (e.g. media duration, video resolution, +// starred flag) without bloating the universal Obj. Keys are stable strings; +// the frontend renders the keys it knows and ignores the rest, so adding or +// removing keys never breaks clients. Implementations should omit empty values. +type ObjExtra interface { + Extra() map[string]any +} + type SetPath interface { SetPath(path string) } @@ -162,6 +171,25 @@ func GetThumb(obj Obj) (thumb string, ok bool) { } } +// GetExtra walks the unwrap chain and returns the driver-specific extra metadata +// if the underlying Obj implements ObjExtra. An empty map is treated as absent. +func GetExtra(obj Obj) (extra map[string]any, ok bool) { + for { + switch o := obj.(type) { + case ObjExtra: + e := o.Extra() + if len(e) == 0 { + return nil, false + } + return e, true + case ObjUnwrap: + obj = o.Unwrap() + default: + return nil, false + } + } +} + func GetUrl(obj Obj) (url string, ok bool) { for { switch o := obj.(type) { diff --git a/internal/model/user.go b/internal/model/user.go index 8043ba822f..314b38d20a 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -28,6 +28,7 @@ const ( TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." GuestCannotUpdateProfile = "Guest user can not update profile" GuestCannotGenerate2FA = "Guest user can not generate 2FA code" + NoPermissionUpdateProfile = "You are not allowed to modify your username or password" ) var LoginCache = cache.NewMemCache[int]() diff --git a/server/handles/auth.go b/server/handles/auth.go index dcb8a05ee6..0baeb63db8 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -113,6 +113,13 @@ func UpdateCurrent(c *gin.Context) { common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } + // Changing the username or setting a new password requires the manage-user-info + // permission (admins always have it). SSO link/unlink — which posts the + // unchanged username with an empty password — stays open to every non-guest. + if (req.Username != user.Username || req.Password != "") && !user.CanManageUserInfo() { + common.ErrorStrResp(c, model.NoPermissionUpdateProfile, 403) + return + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index d1928b60da..314caddff6 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -46,6 +46,9 @@ type ObjResp struct { HashInfoStr string `json:"hashinfo"` HashInfo map[*utils.HashType]string `json:"hash_info"` MountDetails *model.StorageDetails `json:"mount_details,omitempty"` + // Extra carries optional driver-specific metadata (e.g. media duration, + // video resolution, starred). Clients render known keys and ignore the rest. + Extra map[string]any `json:"extra,omitempty"` } type FsListResp struct { @@ -253,6 +256,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + extra, _ := model.GetExtra(obj) hashInfo := obj.GetHash().Export() if hashInfo == nil { hashInfo = make(map[*utils.HashType]string) @@ -269,6 +273,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), MountDetails: mountDetails, + Extra: extra, }) } return resp @@ -383,6 +388,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + extra, _ := model.GetExtra(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ Name: obj.GetName(), @@ -396,6 +402,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { Type: utils.GetFileType(obj.GetName()), Thumb: thumb, MountDetails: mountDetails, + Extra: extra, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), From 91acf580cb78c0905b5c7b0b405f2b3cf6ea911b Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 24 Jun 2026 20:03:36 +0800 Subject: [PATCH 092/107] fix(cluster): auto-recover dead 115 token from a healthy peer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 115 refresh tokens rotate single-use, so independently-refreshing cluster nodes mutually invalidate each other; and a node booting with a stale/dead token had no recovery path — NotifyStorageTokenInvalid fired a "token-invalid" hook that NO driver ever called, so the pull path was dead code. Design (per spec: token defaults invalid; only share a proven-valid token; pull when invalid; on success mark valid + push): - 115-sdk-go v0.2.11: WithOnTokenValid/WithOnTokenInvalid callbacks fired from the authRequest choke point — valid on a successful authed request, invalid on a direct 40140120 or when a refresh dies (40140117 throttle does NOT fire). - 115 driver: edge-trigger via tokenInvalid atomic.Bool so op hooks fire only on real invalid<->valid transitions (no per-request churn): valid->invalid calls NotifyStorageTokenInvalid, invalid->valid calls NotifyStorageTokenValid. - op/hook: add NotifyStorageTokenValid ("token-valid" storage hook) alongside the existing NotifyStorageTokenInvalid. - cluster engine: onStorageHook gains "token-valid" (re-share if WORK, else pull) and "token-invalid" (dropOwnCred + pullGroup). canOfferCred/ownTokenHealthy gate anti-entropy so a node only advertises its OWN cred while its token is proven healthy (Status==WORK) — peer-authored records always relayable. This stops a stale/boot token from poisoning peers. seedLocalCreds drops own cred for unhealthy member mounts at boot, then pulls. - state: dropOwnCred removes only our own-origin record so a dead token's Lamport version can't out-rank a peer's valid one and block recovery. Tests: cluster TestDropOwnCred + TestCanOfferCred green; SDK callback tests green. Wire format unchanged (still credDigest/syncMessage) — new nodes are backward-compatible with old nodes, so canary (one node at a time) is safe. --- drivers/115_open/driver.go | 19 ++++++++ go.mod | 2 +- go.sum | 2 + internal/cluster/engine.go | 85 +++++++++++++++++++++++++++++++--- internal/cluster/state.go | 15 ++++++ internal/cluster/state_test.go | 67 +++++++++++++++++++++++++++ internal/op/hook.go | 9 ++++ 7 files changed, 192 insertions(+), 7 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index bcfde457f8..a634d823ff 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -9,6 +9,7 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" sdk "github.com/OpenListTeam/115-sdk-go" @@ -31,6 +32,11 @@ type Open115 struct { client *sdk.Client limiter *rate.Limiter parentPath string + // tokenInvalid is the edge-trigger latch for the cluster validity protocol: + // set once the refresh_token is found dead, cleared once a request succeeds + // again. It debounces the token-valid/invalid notifications down to real + // transitions instead of firing them on every request. + tokenInvalid atomic.Bool } var ( @@ -54,6 +60,19 @@ func (d *Open115) Init(ctx context.Context) error { d.Addition.AccessToken = s1 d.Addition.RefreshToken = s2 op.MustSaveDriverStorage(d) + }), + // Cluster token-validity protocol: a node only shares a token it has + // proven valid, and pulls a fresh one from a healthy peer when its own + // dies. Edge-triggered so the cluster only reacts to real transitions. + sdk.WithOnTokenValid(func() { + if d.tokenInvalid.CompareAndSwap(true, false) { + op.NotifyStorageTokenValid(d) + } + }), + sdk.WithOnTokenInvalid(func() { + if d.tokenInvalid.CompareAndSwap(false, true) { + op.NotifyStorageTokenInvalid(d) + } })) if flags.Debug || flags.Dev { d.client.SetDebug(true) diff --git a/go.mod b/go.mod index ad91080433..d3a2bb206d 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.10 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.11 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index f9c77dcf50..8f7410d6ee 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqm github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/Ironboxplus/115-sdk-go v0.2.10 h1:00cZGt952O3R5gICxYyMwb83YgcvMEP2VrpiecbaPlA= github.com/Ironboxplus/115-sdk-go v0.2.10/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.11 h1:B0XbwpFf1tGZDm3fjtiRyvW1QsOX4ykFBdHTuv4Y2yU= +github.com/Ironboxplus/115-sdk-go v0.2.11/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 1070af6544..e4ff37d0a4 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -293,7 +293,7 @@ func (m *Manager) helloMessage() *syncMessage { Nodes: m.knownNodesForPEX(), Groups: &gd, GroupsVer: gd.Version, - CredDigests: m.state.credDigests(), + CredDigests: m.offerableDigests(), } } @@ -307,7 +307,7 @@ func (m *Manager) announceMessage() *syncMessage { Nodes: m.knownNodesForPEX(), Groups: &gd, GroupsVer: gd.Version, - CredDigests: m.state.credDigests(), + CredDigests: m.offerableDigests(), } } @@ -348,7 +348,12 @@ func (m *Manager) seedLocalCreds() { continue } if st.Status != op.WORK { + // Unhealthy mount (e.g. token dead at boot): drop our own stale cred so + // it can't dominate a peer's valid one, then pull a fresh credential. for _, g := range groups { + if m.state.dropOwnCred(g.ID, m.id.NodeID) { + changed = true + } m.pullGroup(g.ID) } continue @@ -396,7 +401,9 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { return } switch typ { - case "add", "update": + case "add", "update", "token-valid": + // "token-valid" is fired when an authenticated request just proved the + // token good — (re)share it so peers converge on the working credential. if st.Status != op.WORK { // Health gating: never propagate a broken/expired token. Try to recover // a good one from peers instead. @@ -421,9 +428,21 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { // A mount was removed locally. We keep the group's credential record (other // members still rely on it); only our inventory changes (handled above). case "token-invalid": + // Our token died. Drop our own (now-stale) credential record so its Lamport + // version can't out-rank a peer's valid one, then pull a fresh credential + // from a healthy peer. + var dropped bool for _, g := range groups { + if m.state.dropOwnCred(g.ID, m.id.NodeID) { + dropped = true + m.recordEvent("invalidate", g.ID, + fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) + } m.pullGroup(g.ID) } + if dropped { + m.persist() + } } } @@ -435,6 +454,52 @@ func (m *Manager) pullGroup(groupID string) { m.broadcast(&syncMessage{Type: "pull", Wants: []string{groupID}}) } +// ownTokenHealthy reports whether THIS node currently holds a working token for a +// group — at least one of our member mounts is in the WORK state (its last auth +// attempt succeeded). It is the "token proven valid" gate. +func (m *Manager) ownTokenHealthy(groupID string) bool { + g, ok := m.state.groupByID(groupID) + if !ok { + return false + } + for _, mp := range g.mountsForNode(m.id.NodeID) { + d, err := op.GetStorageByMountPath(mp) + if err != nil { + continue + } + if d.GetStorage().Status == op.WORK { + return true + } + } + return false +} + +// canOfferCred reports whether this node may advertise/relay a credential record +// in anti-entropy. Records authored by peers are always relayable. Our OWN record +// is offered only while our token for that group is healthy — so a node booting +// with a stale persisted token (or whose token just died) never poisons peers +// with it: "only share a token proven valid". +func (m *Manager) canOfferCred(r *credRecord) bool { + if r == nil { + return false + } + if r.Origin != m.id.NodeID { + return true + } + return m.ownTokenHealthy(r.GroupID) +} + +// offerableDigests is credDigests() filtered to records this node may advertise. +func (m *Manager) offerableDigests() []credDigest { + var out []credDigest + for _, r := range m.state.credSnapshot() { + if m.canOfferCred(r) { + out = append(out, digestOfCred(r)) + } + } + return out +} + // applyCredRecord overlays a group's credential onto this node's member mounts. func (m *Manager) applyCredRecord(r *credRecord) { if r == nil { @@ -545,21 +610,29 @@ func (m *Manager) buildReply(in *syncMessage) *syncMessage { mine := digestOfCred(cur) switch { case credDigestDominates(mine, dg): - reply.Creds = append(reply.Creds, cur) + // Offer ours only if we may (own cred must be healthy). If we can't + // offer it (our token is stale/dead), pull theirs instead — this is + // what lets a node with a higher-versioned but DEAD credential still + // recover from a peer's lower-versioned but VALID one. + if m.canOfferCred(cur) { + reply.Creds = append(reply.Creds, cur) + } else { + reply.Wants = append(reply.Wants, dg.GroupID) + } case credDigestDominates(dg, mine): reply.Wants = append(reply.Wants, dg.GroupID) } } // Records we hold the peer never mentioned. for _, cur := range m.state.credSnapshot() { - if _, seen := peerHas[cur.GroupID]; !seen { + if _, seen := peerHas[cur.GroupID]; !seen && m.canOfferCred(cur) { reply.Creds = append(reply.Creds, cur) } } // Explicit pull wants. for _, gid := range in.Wants { - if cur, ok := m.state.getCred(gid); ok { + if cur, ok := m.state.getCred(gid); ok && m.canOfferCred(cur) { reply.Creds = append(reply.Creds, cur) } } diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 3af17c7455..e5f001cb81 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -382,6 +382,21 @@ func (s *store) mergeCred(r *credRecord) bool { return true } +// dropOwnCred removes our own-authored credential record for a group. It is used +// when the local token is found invalid: a dead/stale credential must not linger +// as a dominating record (its Lamport version could otherwise out-rank a peer's +// genuinely-valid credential and block recovery). Returns true if a record was +// removed. Peer-authored records are never touched here. +func (s *store) dropOwnCred(groupID, selfID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + if r, ok := s.creds[groupID]; ok && r.Origin == selfID { + delete(s.creds, groupID) + return true + } + return false +} + // pruneCreds drops credential records for groups that no longer exist. func (s *store) pruneCreds() { s.mu.Lock() diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 8d8033a826..545c32c813 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -182,6 +182,73 @@ func TestCredRecordVerifyRejectsForgery(t *testing.T) { } } +// dropOwnCred must remove only our own-authored record so a dead/stale local +// token can never out-rank (by Lamport version) a peer's genuinely-valid one and +// block recovery. A peer-authored record must survive. +func TestDropOwnCred(t *testing.T) { + id1, _ := newIdentity() // self + id2, _ := newIdentity() // peer + s := newStore() + + // nothing recorded yet -> nothing to drop. + if s.dropOwnCred("g1", id1.NodeID) { + t.Fatal("dropping a non-existent record must return false") + } + + // our own credential -> dropped, store no longer holds it. + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + if _, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 1); !ok { + t.Fatal("own credential should be recorded") + } + if !s.dropOwnCred("g1", id1.NodeID) { + t.Fatal("own credential should be dropped") + } + if _, ok := s.getCred("g1"); ok { + t.Fatal("store must not retain a dropped own credential") + } + + // a peer's credential must NOT be dropped by us. + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + r := &credRecord{GroupID: "g2", OriginDriver: "115", Fields: []string{"refresh_token"}, Payload: payloadB, Version: 5, Origin: id2.NodeID, OriginPub: id2.Pub, CredHash: credHash(payloadB)} + r.Sig = id2.sign(r.signingBytes()) + if !s.mergeCred(r) { + t.Fatal("peer credential should merge") + } + if s.dropOwnCred("g2", id1.NodeID) { + t.Fatal("a peer-authored credential must never be dropped as own") + } + if _, ok := s.getCred("g2"); !ok { + t.Fatal("peer credential must survive dropOwnCred") + } +} + +// canOfferCred enforces "only share a token proven valid": a peer's record is +// always relayable, but our OWN record is offered only while our token for that +// group is healthy. With no matching group/healthy mount, ownTokenHealthy is +// false so our own record must not be advertised (no poisoning peers with a +// stale/boot token). +func TestCanOfferCred(t *testing.T) { + id1, _ := newIdentity() // self + id2, _ := newIdentity() // peer + m := &Manager{id: id1, state: newStore()} + + if m.canOfferCred(nil) { + t.Fatal("a nil record is never offerable") + } + + // peer-authored record -> always offerable (relay). + peer := &credRecord{GroupID: "g1", Origin: id2.NodeID} + if !m.canOfferCred(peer) { + t.Fatal("a peer-authored record must always be offerable") + } + + // our own record, but no group/healthy mount -> not offerable. + own := &credRecord{GroupID: "g1", Origin: id1.NodeID} + if m.canOfferCred(own) { + t.Fatal("our own record must NOT be offered when the token is not proven healthy") + } +} + // ---- inventory ---- func TestInventoryMergeAndDialTargets(t *testing.T) { diff --git a/internal/op/hook.go b/internal/op/hook.go index f149ccfce8..dcd0c8e82d 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -120,3 +120,12 @@ func RegisterStorageHook(hook StorageHook) { func NotifyStorageTokenInvalid(storage driver.Driver) { go callStorageHooks("token-invalid", storage) } + +// NotifyStorageTokenValid signals that a storage's credentials were just proven +// good by a successful authenticated request. It fires the storage hook with the +// "token-valid" type so listeners — notably cluster sync — can (re)share the +// proven token with peers. Drivers should call this only on a real transition +// from invalid→valid to avoid per-request churn. +func NotifyStorageTokenValid(storage driver.Driver) { + go callStorageHooks("token-valid", storage) +} From f5d5eec270560cb156eb9cc65bfd61f1a8e972e4 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 24 Jun 2026 20:03:50 +0800 Subject: [PATCH 093/107] feat(115): expose provider subtitles for cross-source subtitle sharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The movi 115 transcoded preview serves adaptive HLS that drops the source mkv's embedded subtitle tracks. The frontend can't call 115 directly (no auth), so expose 115's VideoSubtitle API through the backend; the player merges these provider-sourced subtitles with sidecar subs so subtitles are available on every quality source, including transcoded streams. - internal/driver: VideoSubtitleProvider interface + VideoSubtitleInfo struct. - 115 driver: VideoSubtitle() maps the 115 /open/video/subtitle response, skipping caption-map entries (IsCaptionMap==1) and URL-less items; Title falls back to Language. - handles/fsread: FsVideoSubtitle (mirrors FsVideoPlay) wraps subtitle URLs in BuildVideoProxyURL (signed same-origin proxy). - router: POST /api/fs/video_subtitle. Note: 115's subtitle list may not include every embedded track of the source stream — it's a best-effort superset of what 115 exposes. --- drivers/115_open/video.go | 51 ++++++++++++++++++++++++++++++++ internal/driver/video.go | 19 ++++++++++++ server/handles/fsread.go | 62 +++++++++++++++++++++++++++++++++++++++ server/router.go | 1 + 4 files changed, 133 insertions(+) diff --git a/drivers/115_open/video.go b/drivers/115_open/video.go index 04b5a6f9b5..6c6f23c4d3 100644 --- a/drivers/115_open/video.go +++ b/drivers/115_open/video.go @@ -28,6 +28,57 @@ func toVideoPlayInfos(urls []sdk.VideoPlayURL) []driver.VideoPlayInfo { return infos } +// toVideoSubtitleInfos maps the SDK subtitle list to driver.VideoSubtitleInfo, +// dropping caption-map placeholder entries and any without a downloadable URL. +func toVideoSubtitleInfos(list []sdk.SubtitleItem) []driver.VideoSubtitleInfo { + infos := make([]driver.VideoSubtitleInfo, 0, len(list)) + for _, s := range list { + if s.URL == "" || s.IsCaptionMap == 1 { + continue + } + title := s.Title + if title == "" { + title = s.Language + } + infos = append(infos, driver.VideoSubtitleInfo{ + Language: s.Language, + Title: title, + URL: s.URL, + Type: s.Type, + }) + } + return infos +} + +// VideoSubtitle returns the provider's subtitle tracks for a video. 115 extracts +// a container's embedded (and any sidecar) subtitles during transcoding and +// serves them as standalone files, so these work on every play source — +// including the transcoded HLS streams that drop the original embedded tracks. +func (d *Open115) VideoSubtitle(ctx context.Context, file model.Obj) ([]driver.VideoSubtitleInfo, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + obj, ok := file.(*Obj) + if !ok { + return nil, errors.New("can't convert obj") + } + pc := obj.Pc + if pc == "" { + return nil, errors.New("can't get pick code") + } + + resp, err := d.client.VideoSubtitle(ctx, pc) + if err != nil { + log.Errorf("[115] VideoSubtitle API failed: %v", err) + return nil, errors.WithStack(err) + } + if resp == nil { + return nil, nil + } + + return toVideoSubtitleInfos(resp.List), nil +} + // VideoPlay returns the official online-play (transcoded streaming) sources // for a video at multiple resolutions. func (d *Open115) VideoPlay(ctx context.Context, file model.Obj) ([]driver.VideoPlayInfo, error) { diff --git a/internal/driver/video.go b/internal/driver/video.go index b6b7e0d884..423b50fa88 100644 --- a/internal/driver/video.go +++ b/internal/driver/video.go @@ -18,3 +18,22 @@ type VideoPlayInfo struct { type VideoPlayer interface { VideoPlay(ctx context.Context, file model.Obj) ([]VideoPlayInfo, error) } + +// VideoSubtitleInfo describes one external subtitle track for a video, exposed +// by providers that index/extract subtitles independently of the media stream +// (e.g. 115 extracts a container's embedded subtitle tracks during transcoding +// and serves them as standalone files). Because they are independent of the +// play source, they render on every quality tier — including transcoded HLS +// streams that don't carry the original container's embedded subtitles. +type VideoSubtitleInfo struct { + Language string `json:"language"` // provider language code, e.g. "chi", "eng" + Title string `json:"title"` // human label, e.g. "简体中文" + URL string `json:"url"` + Type string `json:"type"` // subtitle format: "srt" | "ass" | "vtt" +} + +// VideoSubtitleProvider is an optional driver capability exposing the provider's +// subtitle tracks for a video, independent of the play source. +type VideoSubtitleProvider interface { + VideoSubtitle(ctx context.Context, file model.Obj) ([]VideoSubtitleInfo, error) +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 314caddff6..23c26c9b1c 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -478,6 +478,68 @@ func FsVideoPlay(c *gin.Context) { common.SuccessResp(c, sources) } +// FsVideoSubtitle exposes the provider's subtitle tracks for a video, for +// drivers that implement driver.VideoSubtitleProvider (e.g. 115_open). These +// subtitles are independent of the play source, so the frontend can render them +// on every quality tier — including transcoded HLS streams that drop the +// original container's embedded subtitle tracks. +func FsVideoSubtitle(c *gin.Context) { + var req FsVideoPlayReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if user.IsGuest() && user.Disabled { + common.ErrorStrResp(c, "Guest user is disabled, login please", 401) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + common.GinAppendValues(c, conf.MetaKey, meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + vsp, ok := storage.(driver.VideoSubtitleProvider) + if !ok { + common.ErrorStrResp(c, "driver does not support subtitle tracks", 400) + return + } + obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + subs, err := vsp.VideoSubtitle(c.Request.Context(), obj) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + // Route subtitle files through the signed video proxy so the browser fetches + // them same-origin (provider CDNs reject cross-origin subtitle fetches). + apiURL := common.GetApiUrl(c) + for i := range subs { + if subs[i].URL != "" { + subs[i].URL = BuildVideoProxyURL(apiURL, subs[i].URL) + } + } + common.SuccessResp(c, subs) +} + func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { var related []model.Obj nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) diff --git a/server/router.go b/server/router.go index 0b4523d27c..157410c508 100644 --- a/server/router.go +++ b/server/router.go @@ -238,6 +238,7 @@ func fsAndShare(g *gin.RouterGroup) { g.Any("/list", handles.FsListSplit) g.Any("/get", handles.FsGetSplit) g.POST("/video_play", handles.FsVideoPlay) + g.POST("/video_subtitle", handles.FsVideoSubtitle) a := g.Group("/archive") a.Any("/meta", handles.FsArchiveMetaSplit) a.Any("/list", handles.FsArchiveListSplit) From 88aaabc833e191255483ece515615c7334bdbd74 Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 25 Jun 2026 15:59:29 +0800 Subject: [PATCH 094/107] feat(proxy): per-storage custom User-Agent for proxy download Adds a proxy_user_agent storage option (default empty = passthrough the client's UA). When set, the configured UA is applied to the client request header BEFORE the link is resolved, so drivers that sign the download URL with the request UA (e.g. 115's DownURL, keyed by LinkCacheUA) use it, and transparent-proxy drivers pick it up via ProcessHeader. Deliberately not mutating the cached *model.Link header, which would race across concurrent downloads and leak the UA into the link cache. --- internal/model/storage.go | 3 +++ internal/op/driver.go | 5 +++++ server/common/proxy.go | 22 ++++++++++++++++++++++ server/handles/down.go | 4 ++++ server/webdav/webdav.go | 3 +++ 5 files changed, 37 insertions(+) diff --git a/internal/model/storage.go b/internal/model/storage.go index 264d7b2a64..0f8eb896f4 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -36,6 +36,9 @@ type Proxy struct { DownProxyURL string `json:"down_proxy_url"` // Disable sign for DownProxyURL DisableProxySign bool `json:"disable_proxy_sign"` + // Custom User-Agent for proxy-download upstream requests. + // Empty = passthrough the client's User-Agent (default behavior). + ProxyUserAgent string `json:"proxy_user_agent"` } func (s *Storage) GetStorage() *Storage { diff --git a/internal/op/driver.go b/internal/op/driver.go index 5b79b0aed6..90f7cc84eb 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -145,6 +145,11 @@ func getMainItems(config driver.Config) []driver.Item { Default: "false", Help: "Disable sign for Download proxy URL", }) + items = append(items, driver.Item{ + Name: "proxy_user_agent", + Type: conf.TypeString, + Help: "Custom User-Agent for proxy download. Empty = passthrough the client's User-Agent.", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", diff --git a/server/common/proxy.go b/server/common/proxy.go index c7c975d255..06f8c21d58 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -70,6 +70,28 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. }) return err } +// ApplyProxyUserAgent overrides the User-Agent on the client request header used to +// resolve and proxy a download when the storage configured a custom one. It mutates +// the (request-scoped) request header BEFORE the link is resolved — deliberately not +// the cached *model.Link — so it: +// - feeds drivers that sign the download URL with the request UA (e.g. 115 calls +// DownURL with args.Header's User-Agent), and is keyed correctly by drivers whose +// LinkCacheMode includes the UA; +// - covers transparent-proxy drivers too, since ProcessHeader copies the request +// header (the override) when link.Header carries no User-Agent; +// - never races on or pollutes the shared link cache (the previous approach mutated +// link.Header after caching, leaking the UA across concurrent/later downloads). +// +// Empty config keeps the default behavior (passthrough the client's User-Agent). A +// driver that pins a fixed UA of its own still wins, which is intended — that UA is +// usually required for the link to resolve. +func ApplyProxyUserAgent(r *http.Request, storage *model.Storage) { + if r == nil || storage == nil || storage.ProxyUserAgent == "" { + return + } + r.Header.Set("User-Agent", storage.ProxyUserAgent) +} + func attachHeader(w http.ResponseWriter, file model.Obj, link *model.Link) { fileName := file.GetName() w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(fileName)) diff --git a/server/handles/down.go b/server/handles/down.go index 50025a0b69..828fc7033b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -58,6 +58,10 @@ func Proxy(c *gin.Context) { return } } + // Override the upstream User-Agent before resolving the link so drivers that + // sign the download URL with it (e.g. 115) get the configured UA, not the + // passthrough client UA. No-op when proxy_user_agent is unset. + common.ApplyProxyUserAgent(c.Request, storage.GetStorage()) link, file, err := fs.Link(c.Request.Context(), rawPath, model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 06d1431ac3..318ff78d38 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -271,6 +271,9 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } } + // Override the upstream User-Agent before resolving the link (see down.go); + // no-op when proxy_user_agent is unset. + common.ApplyProxyUserAgent(r, storage.GetStorage()) link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header}) if err != nil { return http.StatusInternalServerError, err From dd646e250612c37e91593cea40c038d1f228598f Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 25 Jun 2026 15:59:29 +0800 Subject: [PATCH 095/107] feat(video): resolve same-name sidecar subtitles in video_subtitle FsVideoSubtitle becomes the unified subtitle source: it lists the video's folder once and returns same-name sidecar subs (basename.*{srt,vtt,ass,ssa,sup}, any driver, signed /p URLs) alongside provider tracks (e.g. 115). Matching requires a '.' right after the basename so "Movie2.srt" / "passthrough.ass" don't false-match "Movie.mkv". No longer 400s for non-provider drivers. --- server/handles/fsread.go | 94 +++++++++++++++++++++++++++++------ server/handles/fsread_test.go | 31 ++++++++++++ 2 files changed, 111 insertions(+), 14 deletions(-) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 23c26c9b1c..e9c9ae0e35 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -514,30 +514,96 @@ func FsVideoSubtitle(c *gin.Context) { common.ErrorResp(c, err, 500) return } - vsp, ok := storage.(driver.VideoSubtitleProvider) - if !ok { - common.ErrorStrResp(c, "driver does not support subtitle tracks", 400) - return - } obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } - subs, err := vsp.VideoSubtitle(c.Request.Context(), obj) + // Same-name sidecar subtitle files (any driver): list the folder once and + // match basename siblings carrying a subtitle extension. This is the unified + // subtitle source so the frontend doesn't have to compute it from `related`. + subs := resolveSidecarSubtitles(c, reqPath, obj) + // Provider subtitle tracks (e.g. 115 extracts a container's embedded subs + // during transcoding) — appended when the driver supports them. Best-effort: + // a provider error still returns the sidecar subs we already resolved. + if vsp, ok := storage.(driver.VideoSubtitleProvider); ok { + if providerSubs, perr := vsp.VideoSubtitle(c.Request.Context(), obj); perr == nil { + // Route provider subtitle files through the signed video proxy so the + // browser fetches them same-origin (provider CDNs reject cross-origin + // subtitle fetches). + apiURL := common.GetApiUrl(c) + for i := range providerSubs { + if providerSubs[i].URL != "" { + providerSubs[i].URL = BuildVideoProxyURL(apiURL, providerSubs[i].URL) + } + } + subs = append(subs, providerSubs...) + } + } + common.SuccessResp(c, subs) +} + +// subtitleExts are the sidecar subtitle file types matched as same-name siblings +// of a video. Matching is case-insensitive. +var subtitleExts = map[string]bool{ + "srt": true, "vtt": true, "ass": true, "ssa": true, "sup": true, +} + +// matchSidecarSubtitle reports whether sibling is a same-name sidecar subtitle of +// the video (basename.<...>. with ext ∈ subtitleExts), returning the +// lowercased subtitle extension. It requires a '.' right after the video's +// basename so "Movie2.srt" doesn't match video "Movie.mkv", and ignores the file +// that is the video itself. +func matchSidecarSubtitle(videoName, siblingName string) (string, bool) { + if strings.EqualFold(videoName, siblingName) { + return "", false + } + ext := strings.TrimPrefix(strings.ToLower(stdpath.Ext(siblingName)), ".") + if !subtitleExts[ext] { + return "", false + } + base := strings.ToLower(strings.TrimSuffix(videoName, stdpath.Ext(videoName))) + sib := strings.ToLower(siblingName) + if !strings.HasPrefix(sib, base) || !strings.HasPrefix(sib[len(base):], ".") { + return "", false + } + return ext, true +} + +// resolveSidecarSubtitles lists the video's parent folder and returns the +// same-name sidecar subtitle files (multi-type / multi-language) as +// VideoSubtitleInfo with signed same-origin /p proxy URLs (mirroring the +// frontend proxyLink + FsGet rawURL construction). +func resolveSidecarSubtitles(c *gin.Context, reqPath string, video model.Obj) []driver.VideoSubtitleInfo { + parentPath := stdpath.Dir(reqPath) + siblings, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) if err != nil { - common.ErrorResp(c, err, 500) - return + return nil } - // Route subtitle files through the signed video proxy so the browser fetches - // them same-origin (provider CDNs reject cross-origin subtitle fetches). + parentMeta, _ := op.GetNearestMeta(parentPath) + encrypt := isEncrypt(parentMeta, parentPath) apiURL := common.GetApiUrl(c) - for i := range subs { - if subs[i].URL != "" { - subs[i].URL = BuildVideoProxyURL(apiURL, subs[i].URL) + var subs []driver.VideoSubtitleInfo + for _, o := range siblings { + if o.IsDir() { + continue + } + ext, ok := matchSidecarSubtitle(video.GetName(), o.GetName()) + if !ok { + continue } + query := "" + if s := common.Sign(o, parentPath, encrypt); s != "" { + query = "?sign=" + s + } + siblingPath := stdpath.Join(parentPath, o.GetName()) + subs = append(subs, driver.VideoSubtitleInfo{ + Title: strings.TrimSuffix(o.GetName(), stdpath.Ext(o.GetName())), + Type: ext, + URL: fmt.Sprintf("%s/p%s%s", apiURL, utils.EncodePath(siblingPath, true), query), + }) } - common.SuccessResp(c, subs) + return subs } func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { diff --git a/server/handles/fsread_test.go b/server/handles/fsread_test.go index 3947ae27f8..7158677067 100644 --- a/server/handles/fsread_test.go +++ b/server/handles/fsread_test.go @@ -172,6 +172,37 @@ func TestGetHeader(t *testing.T) { } } +func TestMatchSidecarSubtitle(t *testing.T) { + tests := []struct { + name string + video string + sibling string + wantExt string + wantOK bool + }{ + {"plain srt", "Movie.mkv", "Movie.srt", "srt", true}, + {"lang-tagged ass", "Movie.mkv", "Movie.zh.ass", "ass", true}, + {"multi-tag vtt", "Show.S01E01.mkv", "Show.S01E01.eng.forced.vtt", "vtt", true}, + {"sup pgs", "Movie.mkv", "Movie.sup", "sup", true}, + {"ssa", "Movie.mkv", "Movie.SSA", "ssa", true}, + {"case-insensitive both", "MOVIE.MKV", "movie.SRT", "srt", true}, + {"prefix collision excluded", "Movie.mkv", "Movie2.srt", "", false}, + {"different basename", "Movie.mkv", "Other.srt", "", false}, + {"non-subtitle ext", "Movie.mkv", "Movie-poster.jpg", "", false}, + {"video itself", "Movie.mkv", "Movie.mkv", "", false}, + {"basename substring not boundary", "Movie.mkv", "Moviegoers.srt", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExt, gotOK := matchSidecarSubtitle(tt.video, tt.sibling) + if gotExt != tt.wantExt || gotOK != tt.wantOK { + t.Errorf("matchSidecarSubtitle(%q, %q) = (%q, %v), want (%q, %v)", + tt.video, tt.sibling, gotExt, gotOK, tt.wantExt, tt.wantOK) + } + }) + } +} + func TestIsEncrypt(t *testing.T) { tests := []struct { name string From cf47b014227ee74a298950b0a88ad7ba21100287 Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 2 Jul 2026 21:28:06 +0800 Subject: [PATCH 096/107] chore(deps): bump Ironboxplus/115-sdk-go to v0.2.12 (refresh gate) Stops the frequent-refresh spiral: cooldown after 40140117, dead-latch after 40140120 until a new refresh_token is injected. --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d3a2bb206d..df497926db 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.11 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.12 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 8f7410d6ee..f4bdb2de9b 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/Ironboxplus/115-sdk-go v0.2.10 h1:00cZGt952O3R5gICxYyMwb83YgcvMEP2VrpiecbaPlA= -github.com/Ironboxplus/115-sdk-go v0.2.10/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= -github.com/Ironboxplus/115-sdk-go v0.2.11 h1:B0XbwpFf1tGZDm3fjtiRyvW1QsOX4ykFBdHTuv4Y2yU= -github.com/Ironboxplus/115-sdk-go v0.2.11/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.12 h1:6WB2L9FG4n+Xv5RvYXQxZBFvbhgYlWoqKYpGTR0coas= +github.com/Ironboxplus/115-sdk-go v0.2.12/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From 53bc1db997cfa1c0cf0cdeadd902ca3cb9aa18e4 Mon Sep 17 00:00:00 2001 From: cyk Date: Fri, 3 Jul 2026 21:53:48 +0800 Subject: [PATCH 097/107] Restore healthy status after proven 115 token use 115 Open can prove credentials healthy after an earlier init or auth failure, but the stale storage status kept the existing cluster Status == WORK gate from sharing those credentials. Restore the status at the token-valid hook before dispatching the existing cluster hook. The persistence path updates only the status column so a successful request cannot overwrite unrelated storage fields from a stale in-memory copy. Constraint: Cluster credential sharing intentionally relies on storage.Status == work Rejected: Bypass cluster status gate for token-valid | would hide stale health state instead of repairing it Confidence: high Scope-risk: moderate Directive: Do not relax cluster credential sharing gates unless status recovery is proven insufficient Tested: go test ./internal/db ./internal/op ./drivers/115_open ./internal/cluster -count=1 Not-tested: Full go test ./... is blocked by missing public/dist, internal/net HTTPS proxy env expectation, and aria2 localhost:6800 dependency --- drivers/115_open/driver.go | 9 ++++- drivers/115_open/driver_test.go | 24 +++++++++++++ internal/db/storage.go | 5 +++ internal/op/hook.go | 24 ++++++++++--- internal/op/hook_test.go | 63 +++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 internal/op/hook_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index a634d823ff..bde23293cc 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -65,7 +65,7 @@ func (d *Open115) Init(ctx context.Context) error { // proven valid, and pulls a fresh one from a healthy peer when its own // dies. Edge-triggered so the cluster only reacts to real transitions. sdk.WithOnTokenValid(func() { - if d.tokenInvalid.CompareAndSwap(true, false) { + if d.shouldNotifyTokenValid() { op.NotifyStorageTokenValid(d) } }), @@ -115,6 +115,13 @@ func (d *Open115) Init(ctx context.Context) error { return nil } +func (d *Open115) shouldNotifyTokenValid() bool { + if d.tokenInvalid.CompareAndSwap(true, false) { + return true + } + return d.GetStorage().Status != op.WORK +} + func (d *Open115) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 4898103271..43d214c8e1 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -171,6 +171,30 @@ func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { t.Fatalf("remove_way item not found in 115 Open driver info") } +func TestOpen115ShouldNotifyTokenValidForInvalidTransitionOrStaleStatus(t *testing.T) { + driver := &Open115{} + driver.Storage.Status = "old init error" + if !driver.shouldNotifyTokenValid() { + t.Fatalf("expected stale storage status to trigger token-valid notification") + } + if driver.tokenInvalid.Load() { + t.Fatalf("stale-status notification should not mark tokenInvalid") + } + + driver.Storage.Status = op.WORK + if driver.shouldNotifyTokenValid() { + t.Fatalf("expected healthy storage with no invalid transition to skip notification") + } + + driver.tokenInvalid.Store(true) + if !driver.shouldNotifyTokenValid() { + t.Fatalf("expected invalid-to-valid transition to trigger token-valid notification") + } + if driver.tokenInvalid.Load() { + t.Fatalf("expected tokenInvalid latch to be cleared") + } +} + func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { t.Helper() diff --git a/internal/db/storage.go b/internal/db/storage.go index 0c660a156c..5c5d98c1c4 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -22,6 +22,11 @@ func UpdateStorage(storage *model.Storage) error { return errors.WithStack(db.Save(storage).Error) } +// UpdateStorageStatus just updates storage status in database. +func UpdateStorageStatus(id uint, status string) error { + return errors.WithStack(db.Model(&model.Storage{}).Where("id = ?", id).Update("status", status).Error) +} + // DeleteStorageById just delete storage from database by id func DeleteStorageById(id uint) error { return errors.WithStack(db.Delete(&model.Storage{}, id).Error) diff --git a/internal/op/hook.go b/internal/op/hook.go index dcd0c8e82d..9cb3cd87f8 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -122,10 +123,25 @@ func NotifyStorageTokenInvalid(storage driver.Driver) { } // NotifyStorageTokenValid signals that a storage's credentials were just proven -// good by a successful authenticated request. It fires the storage hook with the -// "token-valid" type so listeners — notably cluster sync — can (re)share the -// proven token with peers. Drivers should call this only on a real transition -// from invalid→valid to avoid per-request churn. +// good by a successful authenticated request. It restores a stale failure status +// before firing the storage hook, so listeners — notably cluster sync — can +// (re)share the proven token with peers. Drivers should call this only on a real +// transition or stale-status recovery to avoid per-request churn. func NotifyStorageTokenValid(storage driver.Driver) { + restoreStorageTokenValidStatus(storage) go callStorageHooks("token-valid", storage) } + +func restoreStorageTokenValidStatus(storage driver.Driver) { + st := storage.GetStorage() + if st == nil || st.Disabled || st.Status == WORK { + return + } + st.SetStatus(WORK) + if st.ID == 0 { + return + } + if err := db.UpdateStorageStatus(st.ID, WORK); err != nil { + log.Errorf("failed mark storage token valid: %s", err) + } +} diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go new file mode 100644 index 0000000000..b9f6088802 --- /dev/null +++ b/internal/op/hook_test.go @@ -0,0 +1,63 @@ +package op_test + +import ( + "context" + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type tokenValidStorageDriver struct { + model.Storage + addition struct{} +} + +func (d *tokenValidStorageDriver) Config() driver.Config { + return driver.Config{Name: "TokenValidTest", CheckStatus: true} +} + +func (d *tokenValidStorageDriver) GetAddition() driver.Additional { + return &d.addition +} + +func (d *tokenValidStorageDriver) Init(ctx context.Context) error { return nil } + +func (d *tokenValidStorageDriver) Drop(ctx context.Context) error { return nil } + +func (d *tokenValidStorageDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return nil, nil +} + +func (d *tokenValidStorageDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return &model.Link{Header: http.Header{}}, nil +} + +func TestNotifyStorageTokenValidRestoresStatus(t *testing.T) { + storage := model.Storage{ + Driver: "TokenValidTest", + MountPath: "/token-valid-status", + Status: "old auth error", + Addition: "{}", + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + + d := &tokenValidStorageDriver{Storage: storage} + op.NotifyStorageTokenValid(d) + + if d.GetStorage().Status != op.WORK { + t.Fatalf("in-memory status = %q, want %q", d.GetStorage().Status, op.WORK) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status != op.WORK { + t.Fatalf("persisted status = %q, want %q", got.Status, op.WORK) + } +} From d2d7f0b63c161cda374c31278b7f03c0bb0f3576 Mon Sep 17 00:00:00 2001 From: ironbox Date: Fri, 17 Jul 2026 19:46:25 +0800 Subject: [PATCH 098/107] fix(cluster): preserve independent credential sources # Conflicts: # drivers/115_open/driver.go # drivers/115_open/driver_test.go # internal/op/hook.go # internal/op/hook_test.go --- drivers/115_open/driver.go | 35 +++++- drivers/115_open/driver_test.go | 155 ++++++++++++++++++++--- internal/cluster/engine.go | 154 ++++++++++++++++------- internal/cluster/state.go | 216 +++++++++++++++++++++++++++----- internal/cluster/state_test.go | 135 ++++++++++++++++++-- internal/op/hook.go | 16 +++ internal/op/hook_test.go | 25 ++++ 7 files changed, 633 insertions(+), 103 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index bde23293cc..2919324e86 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" stdpath "path" "slices" "strconv" @@ -15,6 +16,7 @@ import ( sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -43,6 +45,7 @@ var ( // 回收站列表存在短暂最终一致性延迟，永久删除 fallback 查找增加短重试。 recycleBinLookupMaxAttempts = 4 recycleBinLookupRetryDelay = 300 * time.Millisecond + new115SDKClient = sdk.New ) func (d *Open115) Config() driver.Config { @@ -54,7 +57,7 @@ func (d *Open115) GetAddition() driver.Additional { } func (d *Open115) Init(ctx context.Context) error { - d.client = sdk.New(sdk.WithRefreshToken(d.Addition.RefreshToken), + d.client = new115SDKClient(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), sdk.WithOnRefreshToken(func(s1, s2 string) { d.Addition.AccessToken = s1 @@ -74,16 +77,18 @@ func (d *Open115) Init(ctx context.Context) error { op.NotifyStorageTokenInvalid(d) } })) + applySDKProxyIfConfigured(d.client) if flags.Debug || flags.Dev { d.client.SetDebug(true) } + d.initLimiter() + if err := d.WaitLimit(ctx); err != nil { + return err + } _, err := d.client.UserInfo(ctx) if err != nil { return err } - if d.Addition.LimitRate > 0 { - d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) - } if d.PageSize <= 0 { d.PageSize = 200 } else if d.PageSize > 1150 { @@ -93,6 +98,9 @@ func (d *Open115) Init(ctx context.Context) error { // add parent path d.parentPath = "/" if d.GetRootId() != d.Config().DefaultRoot { + if err := d.WaitLimit(ctx); err != nil { + return err + } folderInfo, err := d.client.GetFolderInfo(ctx, d.GetRootId()) if err != nil { return err @@ -122,6 +130,25 @@ func (d *Open115) shouldNotifyTokenValid() bool { return d.GetStorage().Status != op.WORK } +func (d *Open115) initLimiter() { + if d.Addition.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) + return + } + d.limiter = nil +} + +func applySDKProxyIfConfigured(client *sdk.Client) { + if client == nil || conf.Conf == nil || strings.TrimSpace(conf.Conf.ProxyAddress) == "" { + return + } + proxyAddress := strings.TrimSpace(conf.Conf.ProxyAddress) + if _, err := url.Parse(proxyAddress); err != nil { + log.Warnf("[115] invalid proxy address ignored: %v", err) + return + } + client.SetProxy(proxyAddress) +} func (d *Open115) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 43d214c8e1..b0aa657b11 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -15,6 +15,8 @@ import ( "time" sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + driverpkg "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -195,6 +197,123 @@ func TestOpen115ShouldNotifyTokenValidForInvalidTransitionOrStaleStatus(t *testi } } +func TestOpen115SDKClientUsesConfiguredProxy(t *testing.T) { + oldConf := conf.Conf + t.Cleanup(func() { + conf.Conf = oldConf + }) + + var gotURL string + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotURL = r.URL.String() + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(proxy.Close) + + conf.Conf = &conf.Config{ProxyAddress: proxy.URL} + client := sdk.New() + applySDKProxyIfConfigured(client) + + resp, err := client.Request( + context.Background(), + "http://openlist-proxy-test.invalid/ping", + http.MethodGet, + ) + if err != nil { + t.Fatalf("expected request to go through configured proxy: %v", err) + } + if resp.StatusCode() != http.StatusOK { + t.Fatalf("unexpected status via proxy: %d", resp.StatusCode()) + } + if gotURL != "http://openlist-proxy-test.invalid/ping" { + t.Fatalf("proxy did not receive the absolute target URL, got %q", gotURL) + } +} + +func TestOpen115InitRateLimitsAuthAndRootInfo(t *testing.T) { + const limitRate = 10.0 + + var ( + mu sync.Mutex + requests []recordedRequest + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + Time: time.Now(), + }) + mu.Unlock() + + switch r.URL.Path { + case "/open/user/info": + writeSDKSuccess(t, w, map[string]any{}) + case "/open/folder/get_info": + writeSDKSuccess(t, w, map[string]any{ + "file_id": "root-1", + "file_name": "Media", + "paths": []map[string]string{ + {"file_id": "0", "file_name": ""}, + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + oldNewClient := new115SDKClient + t.Cleanup(func() { + new115SDKClient = oldNewClient + }) + new115SDKClient = func(opts ...sdk.Option) *sdk.Client { + client := sdk.New(opts...) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + return client + } + + driver := &Open115{Addition: Addition{ + RootID: driverpkg.RootID{RootFolderID: "root-1"}, + AccessToken: "test-access-token", + RefreshToken: "test-refresh-token", + LimitRate: limitRate, + }} + driver.Storage.Status = "old init error" + + if err := driver.Init(context.Background()); err != nil { + t.Fatalf("Init returned error: %v", err) + } + if driver.GetStorage().Status != op.WORK { + t.Fatalf("successful authenticated Init should restore storage status to work, got %q", driver.GetStorage().Status) + } + + mu.Lock() + reqs := append([]recordedRequest(nil), requests...) + mu.Unlock() + + assertRequestPaths(t, reqs, "/open/user/info", "/open/folder/get_info") + gap := reqs[1].Time.Sub(reqs[0].Time) + minGap := time.Duration(float64(time.Second) / limitRate * 0.7) + if gap < minGap { + t.Fatalf("Init SDK requests were too close (%v), expected at least %v; auth/root-info calls are not both rate-limited", gap, minGap) + } +} + func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { t.Helper() @@ -681,24 +800,24 @@ type mockFileStreamer struct { data []byte } -func (m *mockFileStreamer) Read(p []byte) (int, error) { return 0, io.EOF } -func (m *mockFileStreamer) Close() error { return nil } -func (m *mockFileStreamer) Add(_ io.Closer) {} -func (m *mockFileStreamer) AddIfCloser(_ any) {} -func (m *mockFileStreamer) GetSize() int64 { return m.size } -func (m *mockFileStreamer) GetName() string { return m.name } -func (m *mockFileStreamer) ModTime() time.Time { return time.Time{} } -func (m *mockFileStreamer) CreateTime() time.Time { return time.Time{} } -func (m *mockFileStreamer) IsDir() bool { return false } -func (m *mockFileStreamer) GetHash() utils.HashInfo { return m.hashInfo } -func (m *mockFileStreamer) GetID() string { return "" } -func (m *mockFileStreamer) GetPath() string { return "" } -func (m *mockFileStreamer) GetMimetype() string { return "application/octet-stream" } -func (m *mockFileStreamer) NeedStore() bool { return false } -func (m *mockFileStreamer) IsForceStreamUpload() bool { return false } -func (m *mockFileStreamer) GetExist() model.Obj { return nil } -func (m *mockFileStreamer) SetExist(_ model.Obj) {} -func (m *mockFileStreamer) GetFile() model.File { return nil } +func (m *mockFileStreamer) Read(p []byte) (int, error) { return 0, io.EOF } +func (m *mockFileStreamer) Close() error { return nil } +func (m *mockFileStreamer) Add(_ io.Closer) {} +func (m *mockFileStreamer) AddIfCloser(_ any) {} +func (m *mockFileStreamer) GetSize() int64 { return m.size } +func (m *mockFileStreamer) GetName() string { return m.name } +func (m *mockFileStreamer) ModTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) CreateTime() time.Time { return time.Time{} } +func (m *mockFileStreamer) IsDir() bool { return false } +func (m *mockFileStreamer) GetHash() utils.HashInfo { return m.hashInfo } +func (m *mockFileStreamer) GetID() string { return "" } +func (m *mockFileStreamer) GetPath() string { return "" } +func (m *mockFileStreamer) GetMimetype() string { return "application/octet-stream" } +func (m *mockFileStreamer) NeedStore() bool { return false } +func (m *mockFileStreamer) IsForceStreamUpload() bool { return false } +func (m *mockFileStreamer) GetExist() model.Obj { return nil } +func (m *mockFileStreamer) SetExist(_ model.Obj) {} +func (m *mockFileStreamer) GetFile() model.File { return nil } func (m *mockFileStreamer) RangeRead(_ http_range.Range) (io.Reader, error) { return strings.NewReader(string(m.data)), nil } diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index e4ff37d0a4..5e04beb92b 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -359,8 +359,14 @@ func (m *Manager) seedLocalCreds() { continue } creds := extractCreds(st.Addition) + credID := credHash(creds) for _, g := range groups { + if m.state.hasCredHash(g.ID, credID) { + m.state.markCandidateHealthy(g.ID, credID, now()) + continue // keep the original peer signature when the token is identical + } if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { + m.state.markCandidateHealthy(g.ID, rec.CredHash, now()) changed = true m.recordEvent("share", g.ID, fmt.Sprintf("%s shared %d credential field(s)", st.MountPath, len(rec.Fields))) m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) @@ -372,14 +378,10 @@ func (m *Manager) seedLocalCreds() { if len(g.mountsForNode(selfID)) == 0 { continue } - if _, ok := m.state.getCred(g.ID); !ok { + if len(m.state.credsForGroup(g.ID)) == 0 { m.pullGroup(g.ID) } } - // (re)apply held credentials to local mounts. - for _, r := range m.state.credSnapshot() { - m.applyCredRecord(r) - } if changed { m.persist() } @@ -413,9 +415,15 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { return } creds := extractCreds(st.Addition) + credID := credHash(creds) var dirty bool for _, g := range groups { + if m.state.hasCredHash(g.ID, credID) { + m.state.markCandidateHealthy(g.ID, credID, now()) + continue + } if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { + m.state.markCandidateHealthy(g.ID, rec.CredHash, now()) dirty = true m.recordEvent("share", g.ID, fmt.Sprintf("%s refreshed credentials", st.MountPath)) m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) @@ -432,7 +440,9 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { // version can't out-rank a peer's valid one, then pull a fresh credential // from a healthy peer. var dropped bool + credID := credHash(extractCreds(st.Addition)) for _, g := range groups { + m.state.forgetCandidateHealth(g.ID, credID) if m.state.dropOwnCred(g.ID, m.id.NodeID) { dropped = true m.recordEvent("invalidate", g.ID, @@ -454,11 +464,14 @@ func (m *Manager) pullGroup(groupID string) { m.broadcast(&syncMessage{Type: "pull", Wants: []string{groupID}}) } -// ownTokenHealthy reports whether THIS node currently holds a working token for a -// group — at least one of our member mounts is in the WORK state (its last auth -// attempt succeeded). It is the "token proven valid" gate. -func (m *Manager) ownTokenHealthy(groupID string) bool { - g, ok := m.state.groupByID(groupID) +// hasHealthyLocalCandidate reports whether this node is currently using this +// exact credential successfully. A signed record received from a peer is not +// evidence that it works on this machine. +func (m *Manager) hasHealthyLocalCandidate(r *credRecord) bool { + if r == nil || !m.state.candidateHealthy(r.GroupID, r.CredHash, now()) { + return false + } + g, ok := m.state.groupByID(r.GroupID) if !ok { return false } @@ -467,26 +480,25 @@ func (m *Manager) ownTokenHealthy(groupID string) bool { if err != nil { continue } - if d.GetStorage().Status == op.WORK { + st := d.GetStorage() + if st.Status != op.WORK { + continue + } + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + continue + } + if credHash(extractCreds(st.Addition)) == r.CredHash { return true } } return false } -// canOfferCred reports whether this node may advertise/relay a credential record -// in anti-entropy. Records authored by peers are always relayable. Our OWN record -// is offered only while our token for that group is healthy — so a node booting -// with a stale persisted token (or whose token just died) never poisons peers -// with it: "only share a token proven valid". +// canOfferCred reports whether this node may advertise a credential candidate. +// Receiving a record does not turn this node into a blind relay: it must first +// be proven by a matching local working mount within the health lease. func (m *Manager) canOfferCred(r *credRecord) bool { - if r == nil { - return false - } - if r.Origin != m.id.NodeID { - return true - } - return m.ownTokenHealthy(r.GroupID) + return m.hasHealthyLocalCandidate(r) } // offerableDigests is credDigests() filtered to records this node may advertise. @@ -525,6 +537,11 @@ func (m *Manager) applyCredRecord(r *credRecord) { r.GroupID, mp, st.Driver, r.OriginDriver) continue } + if st.Status == op.WORK { + // A peer's newer Lamport value is a recovery candidate, not authority + // to overwrite credentials this mount has already proved locally. + continue + } newAdd, changed := applyCreds(st.Addition, r.Payload) if !changed { continue // already has these credentials — no churn, no re-init @@ -571,7 +588,7 @@ func (m *Manager) absorbGroups(d *groupDoc) bool { func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { var merged []*credRecord for _, r := range recs { - if r == nil || !r.verify() { + if r == nil || !r.verify() || !m.credentialAuthorized(r) { continue } if m.state.mergeCred(r) { @@ -585,6 +602,25 @@ func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { return merged } +// credentialAuthorized verifies that a record signer is an explicit member of +// the signed group at the exact mount it claims. The signature proves who wrote +// the record; the group document proves that writer may supply this group. +func (m *Manager) credentialAuthorized(r *credRecord) bool { + if r == nil || r.GroupID == "" || r.Origin == "" || r.OriginMount == "" { + return false + } + g, ok := m.state.groupByID(r.GroupID) + if !ok { + return false + } + for _, member := range g.Members { + if member.NodeID == r.Origin && member.MountPath == r.OriginMount { + return true + } + } + return false +} + // ---- anti-entropy reply ---- // buildReply answers a peer's digests / wants, telling it what we hold that it @@ -598,42 +634,67 @@ func (m *Manager) buildReply(in *syncMessage) *syncMessage { reply.Groups = &local } - // Credential anti-entropy on digests. + // Credential anti-entropy is per (group, origin) candidate. A group-level + // LWW exchange would silently promote one member into a primary and discard + // the independent recovery paths held by the other members. peerHas := make(map[string]credDigest, len(in.CredDigests)) + wanted := make(map[string]struct{}) + offered := make(map[string]struct{}) + appendWant := func(groupID string) { + if groupID == "" { + return + } + if _, exists := wanted[groupID]; exists { + return + } + wanted[groupID] = struct{}{} + reply.Wants = append(reply.Wants, groupID) + } + appendCred := func(r *credRecord) { + if r == nil || !m.canOfferCred(r) { + return + } + key := credDigestKey(digestOfCred(r)) + if _, exists := offered[key]; exists { + return + } + offered[key] = struct{}{} + reply.Creds = append(reply.Creds, r) + } for _, dg := range in.CredDigests { - peerHas[dg.GroupID] = dg - cur, ok := m.state.getCred(dg.GroupID) + if dg.GroupID == "" || dg.Origin == "" { + continue + } + key := credDigestKey(dg) + peerHas[key] = dg + cur, ok := m.state.getCredForOrigin(dg.GroupID, dg.Origin) if !ok { - reply.Wants = append(reply.Wants, dg.GroupID) + appendWant(dg.GroupID) continue } mine := digestOfCred(cur) switch { case credDigestDominates(mine, dg): - // Offer ours only if we may (own cred must be healthy). If we can't - // offer it (our token is stale/dead), pull theirs instead — this is - // what lets a node with a higher-versioned but DEAD credential still - // recover from a peer's lower-versioned but VALID one. if m.canOfferCred(cur) { - reply.Creds = append(reply.Creds, cur) + appendCred(cur) } else { - reply.Wants = append(reply.Wants, dg.GroupID) + appendWant(dg.GroupID) } case credDigestDominates(dg, mine): - reply.Wants = append(reply.Wants, dg.GroupID) + appendWant(dg.GroupID) } } // Records we hold the peer never mentioned. for _, cur := range m.state.credSnapshot() { - if _, seen := peerHas[cur.GroupID]; !seen && m.canOfferCred(cur) { - reply.Creds = append(reply.Creds, cur) + if _, seen := peerHas[credDigestKey(digestOfCred(cur))]; !seen { + appendCred(cur) } } - // Explicit pull wants. + // An explicit pull requests every locally-proven candidate in the group. for _, gid := range in.Wants { - if cur, ok := m.state.getCred(gid); ok && m.canOfferCred(cur) { - reply.Creds = append(reply.Creds, cur) + for _, cur := range m.state.credsForGroup(gid) { + appendCred(cur) } } @@ -746,8 +807,17 @@ func (m *Manager) handleFrame(c *peerConn, data []byte) { } // Credentials. if merged := m.absorbCreds(msg.Creds); len(merged) > 0 { - relay.Creds = merged - relayHas = true + // A relay must not amplify a peer credential merely because its signature + // was valid. Only candidates that this node has actually proven locally + // may leave this node again. + for _, r := range merged { + if m.canOfferCred(r) { + relay.Creds = append(relay.Creds, r) + } + } + if len(relay.Creds) > 0 { + relayHas = true + } } if relayHas { m.relayMsg(relay, c) diff --git a/internal/cluster/state.go b/internal/cluster/state.go index e5f001cb81..11f47f236c 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -82,8 +82,14 @@ func (d *groupDoc) signingBytes() []byte { } func (d *groupDoc) verify() bool { + if d == nil { + return false + } if d.Version == 0 { - return true // the empty/default doc is implicitly valid + // Version zero is reserved for the exact local bootstrap document. A + // non-empty v0 document has no authenticated ordering and must not win a + // tie-break against a fresh node's empty state. + return len(d.Groups) == 0 && d.Origin == "" && len(d.OriginPub) == 0 && len(d.Sig) == 0 && d.UpdatedAt == 0 } if nodeIDFromPub(d.OriginPub) != d.Origin { return false @@ -169,6 +175,11 @@ func digestOfCred(r *credRecord) credDigest { return credDigest{GroupID: r.GroupID, CredHash: r.CredHash, Version: r.Version, Origin: r.Origin} } +// credDigestKey identifies one origin's current candidate within a group. +func credDigestKey(d credDigest) string { + return d.GroupID + "\x00" + d.Origin +} + func credDigestDominates(a, b credDigest) bool { if a.Version != b.Version { return a.Version > b.Version @@ -206,17 +217,27 @@ type nodeInfo struct { // ---------------------------------------------------------------------------- type store struct { - mu sync.RWMutex - groups groupDoc - creds map[string]*credRecord // keyed by group id - inventory map[string]*nodeInfo // keyed by node id - lamport uint64 -} + mu sync.RWMutex + groups groupDoc + // One group is a peer set, not a primary/replica pair. Each member keeps its + // own current candidate, keyed by the signing origin; Lamport ordering only + // resolves successive credentials from the same origin. + creds map[string]map[string]*credRecord // group id -> origin node id -> candidate + // candidateHealth is an in-memory proof that this node has actually used a + // credential successfully. It is deliberately not persisted: a restart must + // prove the credential again instead of trusting a stale WORK status. + candidateHealth map[string]map[string]int64 // group id -> credential hash -> last proven unix second + inventory map[string]*nodeInfo // keyed by node id + lamport uint64 +} + +const candidateLeaseSec int64 = 15 * 60 func newStore() *store { return &store{ - creds: make(map[string]*credRecord), - inventory: make(map[string]*nodeInfo), + creds: make(map[string]map[string]*credRecord), + candidateHealth: make(map[string]map[string]int64), + inventory: make(map[string]*nodeInfo), } } @@ -305,29 +326,81 @@ func (s *store) groupByID(id string) (group, bool) { func (s *store) getCred(groupID string) (*credRecord, bool) { s.mu.RLock() defer s.mu.RUnlock() - r, ok := s.creds[groupID] + var newest *credRecord + for _, r := range s.creds[groupID] { + if newest == nil || r.dominates(newest) { + newest = r + } + } + return newest, newest != nil +} + +func (s *store) getCredForOrigin(groupID, origin string) (*credRecord, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.creds[groupID][origin] return r, ok } -func (s *store) credSnapshot() []*credRecord { +func (s *store) credsForGroup(groupID string) []*credRecord { s.mu.RLock() defer s.mu.RUnlock() - out := make([]*credRecord, 0, len(s.creds)) - for _, r := range s.creds { + byOrigin := s.creds[groupID] + out := make([]*credRecord, 0, len(byOrigin)) + for _, r := range byOrigin { out = append(out, r) } - sort.Slice(out, func(i, j int) bool { return out[i].GroupID < out[j].GroupID }) + sort.Slice(out, func(i, j int) bool { return out[i].Origin < out[j].Origin }) + return out +} + +func (s *store) hasCredHash(groupID, hash string) bool { + if hash == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + for _, r := range s.creds[groupID] { + if r.CredHash == hash { + return true + } + } + return false +} + +func (s *store) credSnapshot() []*credRecord { + s.mu.RLock() + defer s.mu.RUnlock() + var out []*credRecord + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + out = append(out, r) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID + } + return out[i].Origin < out[j].Origin + }) return out } func (s *store) credDigests() []credDigest { s.mu.RLock() defer s.mu.RUnlock() - out := make([]credDigest, 0, len(s.creds)) - for _, r := range s.creds { - out = append(out, digestOfCred(r)) + var out []credDigest + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + out = append(out, digestOfCred(r)) + } } - sort.Slice(out, func(i, j int) bool { return out[i].GroupID < out[j].GroupID }) + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID + } + return out[i].Origin < out[j].Origin + }) return out } @@ -341,8 +414,15 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay } s.mu.Lock() defer s.mu.Unlock() - if cur, ok := s.creds[groupID]; ok && cur.CredHash == h { - return nil, false // identical credential already known — no churn + byOrigin := s.creds[groupID] + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[groupID] = byOrigin + } + for _, cur := range byOrigin { + if cur.CredHash == h { + return nil, false // a known candidate need not be re-authored + } } s.lamport++ r := &credRecord{ @@ -358,7 +438,7 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay UpdatedAt: now, } r.Sig = id.sign(r.signingBytes()) - s.creds[groupID] = r + byOrigin[id.NodeID] = r return r, true } @@ -368,7 +448,12 @@ func (s *store) mergeCred(r *credRecord) bool { s.mu.Lock() defer s.mu.Unlock() s.observe(r.Version) - cur, ok := s.creds[r.GroupID] + byOrigin := s.creds[r.GroupID] + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[r.GroupID] = byOrigin + } + cur, ok := byOrigin[r.Origin] if ok { if cur.CredHash == r.CredHash { return false // idempotent: same credential @@ -378,7 +463,7 @@ func (s *store) mergeCred(r *credRecord) bool { } } cp := *r - s.creds[r.GroupID] = &cp + byOrigin[r.Origin] = &cp return true } @@ -390,8 +475,14 @@ func (s *store) mergeCred(r *credRecord) bool { func (s *store) dropOwnCred(groupID, selfID string) bool { s.mu.Lock() defer s.mu.Unlock() - if r, ok := s.creds[groupID]; ok && r.Origin == selfID { - delete(s.creds, groupID) + if byOrigin := s.creds[groupID]; byOrigin != nil { + if _, ok := byOrigin[selfID]; !ok { + return false + } + delete(byOrigin, selfID) + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } return true } return false @@ -408,10 +499,48 @@ func (s *store) pruneCreds() { for id := range s.creds { if _, ok := live[id]; !ok { delete(s.creds, id) + delete(s.candidateHealth, id) } } } +func (s *store) markCandidateHealthy(groupID, hash string, at int64) { + if groupID == "" || hash == "" || at <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byHash := s.candidateHealth[groupID] + if byHash == nil { + byHash = make(map[string]int64) + s.candidateHealth[groupID] = byHash + } + byHash[hash] = at +} + +func (s *store) forgetCandidateHealth(groupID, hash string) { + if groupID == "" || hash == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byHash := s.candidateHealth[groupID] + delete(byHash, hash) + if len(byHash) == 0 { + delete(s.candidateHealth, groupID) + } +} + +func (s *store) candidateHealthy(groupID, hash string, at int64) bool { + if groupID == "" || hash == "" || at <= 0 { + return false + } + s.mu.RLock() + provenAt := s.candidateHealth[groupID][hash] + s.mu.RUnlock() + return provenAt > 0 && at >= provenAt && at-provenAt <= candidateLeaseSec +} + // ---- inventory ---- func (s *store) setLocalInventory(info *nodeInfo) { @@ -478,8 +607,10 @@ func (s *store) export() persistedState { s.mu.RLock() defer s.mu.RUnlock() ps := persistedState{Lamport: s.lamport, Groups: s.groups} - for _, r := range s.creds { - ps.Creds = append(ps.Creds, r) + for _, byOrigin := range s.creds { + for _, r := range byOrigin { + ps.Creds = append(ps.Creds, r) + } } for _, n := range s.inventory { ps.Inventory = append(ps.Inventory, n) @@ -491,14 +622,37 @@ func (s *store) load(ps persistedState) { s.mu.Lock() defer s.mu.Unlock() s.lamport = ps.Lamport - s.groups = ps.Groups - s.creds = make(map[string]*credRecord, len(ps.Creds)) + if ps.Groups.verify() { + s.groups = ps.Groups + } else { + s.groups = groupDoc{} + } + s.creds = make(map[string]map[string]*credRecord) + s.candidateHealth = make(map[string]map[string]int64) for _, r := range ps.Creds { - s.creds[r.GroupID] = r + if r == nil || r.GroupID == "" || r.Origin == "" || !r.verify() { + continue + } + byOrigin := s.creds[r.GroupID] + if byOrigin == nil { + byOrigin = make(map[string]*credRecord) + s.creds[r.GroupID] = byOrigin + } + if cur, ok := byOrigin[r.Origin]; !ok || r.dominates(cur) { + cp := *r + byOrigin[r.Origin] = &cp + } + if r.Version > s.lamport { + s.lamport = r.Version + } } s.inventory = make(map[string]*nodeInfo, len(ps.Inventory)) for _, n := range ps.Inventory { - s.inventory[n.NodeID] = n + if n == nil || n.NodeID == "" { + continue + } + cp := *n + s.inventory[n.NodeID] = &cp } } diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 545c32c813..32e3aeea5f 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -114,6 +114,23 @@ func TestGroupDocSignAndMerge(t *testing.T) { } } +func TestGroupDocRejectsNonCanonicalVersionZero(t *testing.T) { + if !(&groupDoc{}).verify() { + t.Fatal("the empty bootstrap document must remain valid") + } + id, _ := newIdentity() + forged := &groupDoc{ + Groups: []group{{ID: "g1", Members: []member{{NodeID: id.NodeID, MountPath: "/115"}}}}, + Origin: id.NodeID, + OriginPub: id.Pub, + UpdatedAt: 1, + } + forged.Sig = id.sign(forged.signingBytes()) + if forged.verify() { + t.Fatal("a non-empty version-zero group document must be rejected") + } +} + func TestGroupsForMount(t *testing.T) { id, _ := newIdentity() s := newStore() @@ -169,6 +186,70 @@ func TestCredRecordMergeLWW(t *testing.T) { } } +// A group is a peer set, not a primary/replica pair. A newer credential from +// one healthy member must never erase a different member's candidate: each +// origin owns its current candidate and nodes choose what to activate locally. +func TestCredRecordMergePreservesCandidatesFromIndependentOrigins(t *testing.T) { + id1, _ := newIdentity() + id2, _ := newIdentity() + s := newStore() + + payloadA := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"A"`)} + if _, ok := s.localCredChange(id1, "g1", "115", "/115-a", payloadA, 1); !ok { + t.Fatal("local candidate should be recorded") + } + + payloadB := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"B"`)} + peer := &credRecord{ + GroupID: "g1", + OriginDriver: "115", + Fields: []string{"refresh_token"}, + Payload: payloadB, + Version: 99, // Must not make this origin a group-wide primary. + Origin: id2.NodeID, + OriginPub: id2.Pub, + OriginMount: "/115-b", + CredHash: credHash(payloadB), + } + peer.Sig = id2.sign(peer.signingBytes()) + if !s.mergeCred(peer) { + t.Fatal("peer candidate should merge") + } + + got := s.credSnapshot() + if len(got) != 2 { + t.Fatalf("candidate count = %d, want 2 independent origins; snapshot=%#v", len(got), got) + } + seen := map[string]string{} + for _, candidate := range got { + seen[candidate.Origin] = candidate.CredHash + } + if seen[id1.NodeID] != credHash(payloadA) || seen[id2.NodeID] != credHash(payloadB) { + t.Fatalf("candidate origins were not preserved: %#v", seen) + } +} + +func TestCandidateHealthLeaseExpires(t *testing.T) { + s := newStore() + const groupID = "g1" + const hash = "candidate-hash" + const provenAt int64 = 1_000 + + s.markCandidateHealthy(groupID, hash, provenAt) + if !s.candidateHealthy(groupID, hash, provenAt+candidateLeaseSec) { + t.Fatal("candidate must remain offerable through its health lease") + } + if s.candidateHealthy(groupID, hash, provenAt+candidateLeaseSec+1) { + t.Fatal("idle candidate must stop being offerable after its health lease") + } + + s.markCandidateHealthy(groupID, hash, provenAt) + s.forgetCandidateHealth(groupID, hash) + if s.candidateHealthy(groupID, hash, provenAt) { + t.Fatal("token-invalid must revoke the health lease immediately") + } +} + func TestCredRecordVerifyRejectsForgery(t *testing.T) { id, _ := newIdentity() other, _ := newIdentity() @@ -222,11 +303,9 @@ func TestDropOwnCred(t *testing.T) { } } -// canOfferCred enforces "only share a token proven valid": a peer's record is -// always relayable, but our OWN record is offered only while our token for that -// group is healthy. With no matching group/healthy mount, ownTokenHealthy is -// false so our own record must not be advertised (no poisoning peers with a -// stale/boot token). +// canOfferCred enforces "only share a token proven valid" for every candidate. +// Receiving a signed peer record is not proof that this node can use it, so it +// must not turn this node into a blind relay for an unvalidated credential. func TestCanOfferCred(t *testing.T) { id1, _ := newIdentity() // self id2, _ := newIdentity() // peer @@ -236,10 +315,11 @@ func TestCanOfferCred(t *testing.T) { t.Fatal("a nil record is never offerable") } - // peer-authored record -> always offerable (relay). + // A peer-authored record is not offerable without a matching local healthy + // mount. This manager has neither a group nor a live validated candidate. peer := &credRecord{GroupID: "g1", Origin: id2.NodeID} - if !m.canOfferCred(peer) { - t.Fatal("a peer-authored record must always be offerable") + if m.canOfferCred(peer) { + t.Fatal("a peer-authored record must NOT be relayed before local validation") } // our own record, but no group/healthy mount -> not offerable. @@ -249,6 +329,45 @@ func TestCanOfferCred(t *testing.T) { } } +func TestAbsorbCredsRejectsUnauthorizedOriginMount(t *testing.T) { + admin, _ := newIdentity() + memberID, _ := newIdentity() + localID, _ := newIdentity() + s := newStore() + s.setGroups(admin, []group{{ + ID: "g1", + Members: []member{{NodeID: memberID.NodeID, MountPath: "/115"}}, + }}, 1) + m := &Manager{id: localID, state: s, cfgStore: &configStore{cfg: Config{ApplyRemote: false}}} + + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"candidate"`)} + bad := &credRecord{ + GroupID: "g1", + OriginDriver: "115 Open", + Fields: []string{"refresh_token"}, + Payload: payload, + CredHash: credHash(payload), + Version: 2, + Origin: memberID.NodeID, + OriginPub: memberID.Pub, + OriginMount: "/not-a-group-member", + } + bad.Sig = memberID.sign(bad.signingBytes()) + if got := m.absorbCreds([]*credRecord{bad}); len(got) != 0 { + t.Fatalf("unauthorized origin mount was accepted: %#v", got) + } + if _, ok := s.getCredForOrigin("g1", memberID.NodeID); ok { + t.Fatal("unauthorized credential must not enter replicated state") + } + + good := *bad + good.OriginMount = "/115" + good.Sig = memberID.sign(good.signingBytes()) + if got := m.absorbCreds([]*credRecord{&good}); len(got) != 1 { + t.Fatalf("authorized group member credential rejected: %#v", got) + } +} + // ---- inventory ---- func TestInventoryMergeAndDialTargets(t *testing.T) { diff --git a/internal/op/hook.go b/internal/op/hook.go index 9cb3cd87f8..63656146d6 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -119,9 +119,25 @@ func RegisterStorageHook(hook StorageHook) { // propagating the broken token. Drivers may call this when an API call fails with // an unrecoverable auth error. func NotifyStorageTokenInvalid(storage driver.Driver) { + markStorageTokenInvalidStatus(storage) go callStorageHooks("token-invalid", storage) } +func markStorageTokenInvalidStatus(storage driver.Driver) { + st := storage.GetStorage() + if st == nil || st.Disabled || st.Status != WORK { + return + } + const invalidStatus = "token invalid" + st.SetStatus(invalidStatus) + if st.ID == 0 { + return + } + if err := db.UpdateStorageStatus(st.ID, invalidStatus); err != nil { + log.Errorf("failed mark storage token invalid: %s", err) + } +} + // NotifyStorageTokenValid signals that a storage's credentials were just proven // good by a successful authenticated request. It restores a stale failure status // before firing the storage hook, so listeners — notably cluster sync — can diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go index b9f6088802..22515652a3 100644 --- a/internal/op/hook_test.go +++ b/internal/op/hook_test.go @@ -61,3 +61,28 @@ func TestNotifyStorageTokenValidRestoresStatus(t *testing.T) { t.Fatalf("persisted status = %q, want %q", got.Status, op.WORK) } } +func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { + storage := model.Storage{ + Driver: "TokenInvalidTest", + MountPath: "/token-invalid-status", + Status: op.WORK, + Addition: "{}", + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + + d := &tokenValidStorageDriver{Storage: storage} + op.NotifyStorageTokenInvalid(d) + + if d.GetStorage().Status == op.WORK { + t.Fatal("token-invalid must make the in-memory storage non-work before recovery hooks run") + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status == op.WORK { + t.Fatal("token-invalid must persist a non-work status before peer recovery") + } +} From 5a61c4b50c3a8c6bba80caa3da8f168bd93112c5 Mon Sep 17 00:00:00 2001 From: ironbox Date: Fri, 17 Jul 2026 20:06:30 +0800 Subject: [PATCH 099/107] fix(cluster): bind credential signatures to mount --- internal/cluster/state.go | 2 ++ internal/cluster/state_test.go | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 11f47f236c..b9fde1bf8c 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -131,6 +131,8 @@ func (r *credRecord) signingBytes() []byte { b = append(b, 0) b = append(b, r.OriginDriver...) b = append(b, 0) + b = append(b, r.OriginMount...) + b = append(b, 0) b = append(b, r.CredHash...) b = append(b, 0) v := make([]byte, 8) diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 32e3aeea5f..4bb5230967 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -263,6 +263,29 @@ func TestCredRecordVerifyRejectsForgery(t *testing.T) { } } +func TestCredRecordVerifyBindsOriginMount(t *testing.T) { + id, _ := newIdentity() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`)} + r := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115-a", + } + r.Sig = id.sign(r.signingBytes()) + if !r.verify() { + t.Fatal("freshly signed record must verify") + } + r.OriginMount = "/115-b" + if r.verify() { + t.Fatal("changing origin mount must invalidate the credential signature") + } +} + // dropOwnCred must remove only our own-authored record so a dead/stale local // token can never out-rank (by Lamport version) a peer's genuinely-valid one and // block recovery. A peer-authored record must survive. From 640160357a9669f59a49be0e8a73f2c2580750b1 Mon Sep 17 00:00:00 2001 From: ironbox Date: Fri, 17 Jul 2026 20:37:40 +0800 Subject: [PATCH 100/107] fix(cluster): preserve mount-level credential recovery --- drivers/115_open/driver.go | 4 + internal/cluster/engine.go | 169 ++++++++++++++++++++----------- internal/cluster/state.go | 180 ++++++++++++++++++++++++++------- internal/cluster/state_test.go | 136 ++++++++++++++++++++++--- internal/op/hook.go | 51 ++++++++-- internal/op/hook_test.go | 25 +++++ 6 files changed, 446 insertions(+), 119 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 2919324e86..4d40a39381 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -68,6 +68,10 @@ func (d *Open115) Init(ctx context.Context) error { // proven valid, and pulls a fresh one from a healthy peer when its own // dies. Edge-triggered so the cluster only reacts to real transitions. sdk.WithOnTokenValid(func() { + // A successful authenticated request renews the cluster's in-memory + // proof only. This avoids a periodic refresh/push while preventing a + // working credential from ageing out of recovery advertisement. + op.NotifyStorageTokenHealthy(d) if d.shouldNotifyTokenValid() { op.NotifyStorageTokenValid(d) } diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 5e04beb92b..50644e80d7 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -40,6 +40,9 @@ type Manager struct { persistMu sync.Mutex + recoveryMu sync.Mutex + recoveryLocks map[string]*sync.Mutex // per local mount: serialize credential adoption + dialMu sync.Mutex dialing map[string]bool // peer URLs with an in-flight/live outbound dial @@ -69,14 +72,15 @@ func Init(dataDir string) (*Manager, error) { return nil, err } m := &Manager{ - dir: dir, - id: id, - cfgStore: newConfigStore(dir), - state: newStore(), - replay: newReplayCache(replayWindowSec), - conns: newConnRegistry(), - dialing: make(map[string]bool), - stopCh: make(chan struct{}), + dir: dir, + id: id, + cfgStore: newConfigStore(dir), + state: newStore(), + replay: newReplayCache(replayWindowSec), + conns: newConnRegistry(), + dialing: make(map[string]bool), + recoveryLocks: make(map[string]*sync.Mutex), + stopCh: make(chan struct{}), } if _, err := m.cfgStore.loadOrInit(); err != nil { return nil, err @@ -84,6 +88,7 @@ func Init(dataDir string) (*Manager, error) { m.loadState() op.RegisterStorageHook(m.onStorageHook) + op.RegisterStorageHealthHook(m.onStorageHealthy) Default = m return m, nil } @@ -349,19 +354,25 @@ func (m *Manager) seedLocalCreds() { } if st.Status != op.WORK { // Unhealthy mount (e.g. token dead at boot): drop our own stale cred so - // it can't dominate a peer's valid one, then pull a fresh credential. + // it can't dominate a peer's valid one. Persisted peer candidates are + // tried before a network pull so restart recovery also works offline. for _, g := range groups { - if m.state.dropOwnCred(g.ID, m.id.NodeID) { + if m.state.dropOwnCred(g.ID, m.id.NodeID, st.MountPath) { changed = true } - m.pullGroup(g.ID) + for _, candidate := range m.state.credsForGroup(g.ID) { + m.applyCredRecord(candidate) + } + if d.GetStorage().Status != op.WORK { + m.pullGroup(g.ID) + } } continue } creds := extractCreds(st.Addition) credID := credHash(creds) for _, g := range groups { - if m.state.hasCredHash(g.ID, credID) { + if m.state.hasCredForSource(g.ID, selfID, st.MountPath, credID) { m.state.markCandidateHealthy(g.ID, credID, now()) continue // keep the original peer signature when the token is identical } @@ -418,7 +429,7 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { credID := credHash(creds) var dirty bool for _, g := range groups { - if m.state.hasCredHash(g.ID, credID) { + if m.state.hasCredForSource(g.ID, selfID, st.MountPath, credID) { m.state.markCandidateHealthy(g.ID, credID, now()) continue } @@ -436,6 +447,12 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { // A mount was removed locally. We keep the group's credential record (other // members still rely on it); only our inventory changes (handled above). case "token-invalid": + if st.Status == op.WORK { + // Hooks are asynchronous. A later successful request may already have + // restored this storage, so an older invalid event must not revoke the + // healthy candidate it would otherwise overwrite. + return + } // Our token died. Drop our own (now-stale) credential record so its Lamport // version can't out-rank a peer's valid one, then pull a fresh credential // from a healthy peer. @@ -443,7 +460,7 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { credID := credHash(extractCreds(st.Addition)) for _, g := range groups { m.state.forgetCandidateHealth(g.ID, credID) - if m.state.dropOwnCred(g.ID, m.id.NodeID) { + if m.state.dropOwnCred(g.ID, m.id.NodeID, st.MountPath) { dropped = true m.recordEvent("invalidate", g.ID, fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) @@ -456,6 +473,29 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { } } +// onStorageHealthy renews only the in-memory proof for an already working +// credential. It deliberately does not write the database, broadcast a record, +// or emit a storage lifecycle event; ordinary successful requests therefore keep +// a candidate usable without causing token-refresh or cluster-sync churn. +func (m *Manager) onStorageHealthy(d driver.Driver) { + if !m.cfgStore.get().active() { + return + } + st := d.GetStorage() + if st == nil || st.Status != op.WORK { + return + } + credID := credHash(extractCreds(st.Addition)) + if credID == "" { + return + } + for _, g := range m.state.groupsForMount(m.id.NodeID, st.MountPath) { + if m.state.hasCredHash(g.ID, credID) { + m.state.markCandidateHealthy(g.ID, credID, now()) + } + } +} + // pullGroup asks peers for the latest credential of a group. func (m *Manager) pullGroup(groupID string) { if !m.cfgStore.get().active() { @@ -495,9 +535,17 @@ func (m *Manager) hasHealthyLocalCandidate(r *credRecord) bool { } // canOfferCred reports whether this node may advertise a credential candidate. -// Receiving a record does not turn this node into a blind relay: it must first -// be proven by a matching local working mount within the health lease. +// A remote candidate is authenticated by its signature and the signed group +// document, and must remain relayable through an accept-only/NAT hub. This +// node's own candidate additionally needs a recent local health proof so stale +// credentials from its state.json are never reintroduced after a restart. func (m *Manager) canOfferCred(r *credRecord) bool { + if !m.credentialAuthorized(r) { + return false + } + if r.Origin != m.id.NodeID { + return true + } return m.hasHealthyLocalCandidate(r) } @@ -512,6 +560,20 @@ func (m *Manager) offerableDigests() []credDigest { return out } +func (m *Manager) recoveryLock(mount string) *sync.Mutex { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + if m.recoveryLocks == nil { + m.recoveryLocks = make(map[string]*sync.Mutex) + } + lock := m.recoveryLocks[mount] + if lock == nil { + lock = &sync.Mutex{} + m.recoveryLocks[mount] = lock + } + return lock +} + // applyCredRecord overlays a group's credential onto this node's member mounts. func (m *Manager) applyCredRecord(r *credRecord) { if r == nil { @@ -527,32 +589,37 @@ func (m *Manager) applyCredRecord(r *credRecord) { } ctx := context.Background() for _, mp := range g.mountsForNode(m.id.NodeID) { - d, err := op.GetStorageByMountPath(mp) - if err != nil { - continue - } - st := *d.GetStorage() // copy; preserve ID/Status/local fields - if r.OriginDriver != "" && st.Driver != r.OriginDriver { - utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", - r.GroupID, mp, st.Driver, r.OriginDriver) - continue - } - if st.Status == op.WORK { - // A peer's newer Lamport value is a recovery candidate, not authority - // to overwrite credentials this mount has already proved locally. - continue - } - newAdd, changed := applyCreds(st.Addition, r.Payload) - if !changed { - continue // already has these credentials — no churn, no re-init - } - st.Addition = newAdd - if err := op.UpdateStorage(ctx, st); err != nil { - utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) - continue - } - m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) - utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) + lock := m.recoveryLock(mp) + lock.Lock() + func() { + defer lock.Unlock() + d, err := op.GetStorageByMountPath(mp) + if err != nil { + return + } + st := *d.GetStorage() // copy; preserve ID/Status/local fields + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", + r.GroupID, mp, st.Driver, r.OriginDriver) + return + } + if st.Status == op.WORK { + // A peer's newer Lamport value is a recovery candidate, not authority + // to overwrite credentials this mount has already proved locally. + return + } + newAdd, changed := applyCreds(st.Addition, r.Payload) + if !changed { + return // already has these credentials — no churn, no re-init + } + st.Addition = newAdd + if err := op.UpdateStorage(ctx, st); err != nil { + utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) + return + } + m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) + utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) + }() } } @@ -606,19 +673,7 @@ func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { // the signed group at the exact mount it claims. The signature proves who wrote // the record; the group document proves that writer may supply this group. func (m *Manager) credentialAuthorized(r *credRecord) bool { - if r == nil || r.GroupID == "" || r.Origin == "" || r.OriginMount == "" { - return false - } - g, ok := m.state.groupByID(r.GroupID) - if !ok { - return false - } - for _, member := range g.Members { - if member.NodeID == r.Origin && member.MountPath == r.OriginMount { - return true - } - } - return false + return credentialAuthorizedByGroups(m.state.groupList(), r) } // ---- anti-entropy reply ---- @@ -634,7 +689,7 @@ func (m *Manager) buildReply(in *syncMessage) *syncMessage { reply.Groups = &local } - // Credential anti-entropy is per (group, origin) candidate. A group-level + // Credential anti-entropy is per (group, origin, mount) candidate. A group-level // LWW exchange would silently promote one member into a primary and discard // the independent recovery paths held by the other members. peerHas := make(map[string]credDigest, len(in.CredDigests)) @@ -667,7 +722,7 @@ func (m *Manager) buildReply(in *syncMessage) *syncMessage { } key := credDigestKey(dg) peerHas[key] = dg - cur, ok := m.state.getCredForOrigin(dg.GroupID, dg.Origin) + cur, ok := m.state.getCredForSource(dg.GroupID, dg.Origin, dg.OriginMount) if !ok { appendWant(dg.GroupID) continue diff --git a/internal/cluster/state.go b/internal/cluster/state.go index b9fde1bf8c..ae05e5dcc7 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -120,19 +120,35 @@ type credRecord struct { Version uint64 `json:"version"` // Lamport clock Origin string `json:"origin"` // authoring node id OriginPub []byte `json:"origin_pub"` // origin ed25519 pubkey - OriginMount string `json:"origin_mount"` // informational + OriginMount string `json:"origin_mount"` // authoring mount path UpdatedAt int64 `json:"updated_at"` - Sig []byte `json:"sig"` + Sig []byte `json:"sig"` // v1-compatible record signature + MountSig []byte `json:"mount_sig,omitempty"` // binds OriginMount for upgraded peers } +// signingBytes deliberately preserves the original credential signature format. +// Older nodes ignore MountSig and can therefore keep verifying records from an +// upgraded node during a rolling deployment. func (r *credRecord) signingBytes() []byte { + return r.credentialSigningBytes(false) +} + +// v2SigningBytes was briefly emitted by 5a61c4b5. It is accepted on read so a +// state.json written by that revision is not silently discarded. +func (r *credRecord) v2SigningBytes() []byte { + return r.credentialSigningBytes(true) +} + +func (r *credRecord) credentialSigningBytes(bindMount bool) []byte { var b []byte b = append(b, r.GroupID...) b = append(b, 0) b = append(b, r.OriginDriver...) b = append(b, 0) - b = append(b, r.OriginMount...) - b = append(b, 0) + if bindMount { + b = append(b, r.OriginMount...) + b = append(b, 0) + } b = append(b, r.CredHash...) b = append(b, 0) v := make([]byte, 8) @@ -145,14 +161,43 @@ func (r *credRecord) signingBytes() []byte { return b } -func (r *credRecord) verify() bool { - if nodeIDFromPub(r.OriginPub) != r.Origin { - return false +func (r *credRecord) mountSigningBytes() []byte { + b := append([]byte("openlist/cluster/credential-mount/v1\x00"), r.signingBytes()...) + b = append(b, 0) + b = append(b, r.OriginMount...) + return b +} + +type credentialSignature uint8 + +const ( + credentialSignatureInvalid credentialSignature = iota + credentialSignatureLegacy + credentialSignatureMountBound + credentialSignatureV2 +) + +func (r *credRecord) signatureKind() credentialSignature { + if r == nil || nodeIDFromPub(r.OriginPub) != r.Origin || credHash(r.Payload) != r.CredHash { + return credentialSignatureInvalid } - if credHash(r.Payload) != r.CredHash { - return false + if verifySig(r.OriginPub, r.signingBytes(), r.Sig) { + if len(r.MountSig) == 0 { + return credentialSignatureLegacy + } + if verifySig(r.OriginPub, r.mountSigningBytes(), r.MountSig) { + return credentialSignatureMountBound + } + return credentialSignatureInvalid } - return verifySig(r.OriginPub, r.signingBytes(), r.Sig) + if len(r.MountSig) == 0 && verifySig(r.OriginPub, r.v2SigningBytes(), r.Sig) { + return credentialSignatureV2 + } + return credentialSignatureInvalid +} + +func (r *credRecord) verify() bool { + return r.signatureKind() != credentialSignatureInvalid } func (r *credRecord) dominates(other *credRecord) bool { @@ -167,19 +212,22 @@ func (r *credRecord) dominates(other *credRecord) bool { // credDigest is the compact (no-secret) advert of a held credential record. type credDigest struct { - GroupID string `json:"group_id"` - CredHash string `json:"cred_hash"` - Version uint64 `json:"version"` - Origin string `json:"origin"` + GroupID string `json:"group_id"` + CredHash string `json:"cred_hash"` + Version uint64 `json:"version"` + Origin string `json:"origin"` + OriginMount string `json:"origin_mount,omitempty"` } func digestOfCred(r *credRecord) credDigest { - return credDigest{GroupID: r.GroupID, CredHash: r.CredHash, Version: r.Version, Origin: r.Origin} + return credDigest{GroupID: r.GroupID, CredHash: r.CredHash, Version: r.Version, Origin: r.Origin, OriginMount: r.OriginMount} } -// credDigestKey identifies one origin's current candidate within a group. +func candidateKey(origin, mount string) string { return origin + "\x00" + mount } + +// credDigestKey identifies one mount-level candidate within a group. func credDigestKey(d credDigest) string { - return d.GroupID + "\x00" + d.Origin + return d.GroupID + "\x00" + candidateKey(d.Origin, d.OriginMount) } func credDigestDominates(a, b credDigest) bool { @@ -189,6 +237,9 @@ func credDigestDominates(a, b credDigest) bool { if a.Origin != b.Origin { return a.Origin > b.Origin } + if a.OriginMount != b.OriginMount { + return a.OriginMount > b.OriginMount + } return a.CredHash > b.CredHash } @@ -222,9 +273,9 @@ type store struct { mu sync.RWMutex groups groupDoc // One group is a peer set, not a primary/replica pair. Each member keeps its - // own current candidate, keyed by the signing origin; Lamport ordering only - // resolves successive credentials from the same origin. - creds map[string]map[string]*credRecord // group id -> origin node id -> candidate + // own current candidate, keyed by the signing origin and mount; Lamport + // ordering only resolves successive credentials from the same source mount. + creds map[string]map[string]*credRecord // group id -> (origin, mount) -> candidate // candidateHealth is an in-memory proof that this node has actually used a // credential successfully. It is deliberately not persisted: a restart must // prove the credential again instead of trusting a stale WORK status. @@ -323,6 +374,36 @@ func (s *store) groupByID(id string) (group, bool) { return group{}, false } +// credentialAuthorizedByGroups binds a signed credential to a configured source +// mount. Legacy signatures did not bind OriginMount, so they are safe only when +// that node has exactly one member mount in the group; upgraded records carry a +// separate MountSig and support multiple mounts from the same node. +func credentialAuthorizedByGroups(groups []group, r *credRecord) bool { + if r == nil || r.GroupID == "" || r.Origin == "" || r.OriginMount == "" || !r.verify() { + return false + } + var membersForOrigin int + var exactMount bool + for _, g := range groups { + if g.ID != r.GroupID { + continue + } + for _, member := range g.Members { + if member.NodeID != r.Origin { + continue + } + membersForOrigin++ + if member.MountPath == r.OriginMount { + exactMount = true + } + } + } + if !exactMount { + return false + } + return r.signatureKind() != credentialSignatureLegacy || membersForOrigin == 1 +} + // ---- creds ---- func (s *store) getCred(groupID string) (*credRecord, bool) { @@ -337,10 +418,10 @@ func (s *store) getCred(groupID string) (*credRecord, bool) { return newest, newest != nil } -func (s *store) getCredForOrigin(groupID, origin string) (*credRecord, bool) { +func (s *store) getCredForSource(groupID, origin, mount string) (*credRecord, bool) { s.mu.RLock() defer s.mu.RUnlock() - r, ok := s.creds[groupID][origin] + r, ok := s.creds[groupID][candidateKey(origin, mount)] return r, ok } @@ -352,7 +433,12 @@ func (s *store) credsForGroup(groupID string) []*credRecord { for _, r := range byOrigin { out = append(out, r) } - sort.Slice(out, func(i, j int) bool { return out[i].Origin < out[j].Origin }) + sort.Slice(out, func(i, j int) bool { + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) return out } @@ -370,6 +456,16 @@ func (s *store) hasCredHash(groupID, hash string) bool { return false } +func (s *store) hasCredForSource(groupID, origin, mount, hash string) bool { + if hash == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.creds[groupID][candidateKey(origin, mount)] + return ok && r.CredHash == hash +} + func (s *store) credSnapshot() []*credRecord { s.mu.RLock() defer s.mu.RUnlock() @@ -383,7 +479,10 @@ func (s *store) credSnapshot() []*credRecord { if out[i].GroupID != out[j].GroupID { return out[i].GroupID < out[j].GroupID } - return out[i].Origin < out[j].Origin + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount }) return out } @@ -401,7 +500,10 @@ func (s *store) credDigests() []credDigest { if out[i].GroupID != out[j].GroupID { return out[i].GroupID < out[j].GroupID } - return out[i].Origin < out[j].Origin + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount }) return out } @@ -421,10 +523,8 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay byOrigin = make(map[string]*credRecord) s.creds[groupID] = byOrigin } - for _, cur := range byOrigin { - if cur.CredHash == h { - return nil, false // a known candidate need not be re-authored - } + if cur, ok := byOrigin[candidateKey(id.NodeID, mount)]; ok && cur.CredHash == h && cur.signatureKind() == credentialSignatureMountBound { + return nil, false // this source mount has not changed } s.lamport++ r := &credRecord{ @@ -440,7 +540,8 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay UpdatedAt: now, } r.Sig = id.sign(r.signingBytes()) - byOrigin[id.NodeID] = r + r.MountSig = id.sign(r.mountSigningBytes()) + byOrigin[candidateKey(id.NodeID, mount)] = r return r, true } @@ -455,7 +556,8 @@ func (s *store) mergeCred(r *credRecord) bool { byOrigin = make(map[string]*credRecord) s.creds[r.GroupID] = byOrigin } - cur, ok := byOrigin[r.Origin] + key := candidateKey(r.Origin, r.OriginMount) + cur, ok := byOrigin[key] if ok { if cur.CredHash == r.CredHash { return false // idempotent: same credential @@ -465,7 +567,7 @@ func (s *store) mergeCred(r *credRecord) bool { } } cp := *r - byOrigin[r.Origin] = &cp + byOrigin[key] = &cp return true } @@ -474,14 +576,15 @@ func (s *store) mergeCred(r *credRecord) bool { // as a dominating record (its Lamport version could otherwise out-rank a peer's // genuinely-valid credential and block recovery). Returns true if a record was // removed. Peer-authored records are never touched here. -func (s *store) dropOwnCred(groupID, selfID string) bool { +func (s *store) dropOwnCred(groupID, selfID, mount string) bool { s.mu.Lock() defer s.mu.Unlock() if byOrigin := s.creds[groupID]; byOrigin != nil { - if _, ok := byOrigin[selfID]; !ok { + key := candidateKey(selfID, mount) + if _, ok := byOrigin[key]; !ok { return false } - delete(byOrigin, selfID) + delete(byOrigin, key) if len(byOrigin) == 0 { delete(s.creds, groupID) } @@ -632,7 +735,7 @@ func (s *store) load(ps persistedState) { s.creds = make(map[string]map[string]*credRecord) s.candidateHealth = make(map[string]map[string]int64) for _, r := range ps.Creds { - if r == nil || r.GroupID == "" || r.Origin == "" || !r.verify() { + if !credentialAuthorizedByGroups(s.groups.Groups, r) { continue } byOrigin := s.creds[r.GroupID] @@ -640,9 +743,10 @@ func (s *store) load(ps persistedState) { byOrigin = make(map[string]*credRecord) s.creds[r.GroupID] = byOrigin } - if cur, ok := byOrigin[r.Origin]; !ok || r.dominates(cur) { + key := candidateKey(r.Origin, r.OriginMount) + if cur, ok := byOrigin[key]; !ok || r.dominates(cur) { cp := *r - byOrigin[r.Origin] = &cp + byOrigin[key] = &cp } if r.Version > s.lamport { s.lamport = r.Version diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 4bb5230967..66b48e7a1e 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -256,6 +256,7 @@ func TestCredRecordVerifyRejectsForgery(t *testing.T) { payload := map[string]json.RawMessage{"token": json.RawMessage(`"x"`)} r := &credRecord{GroupID: "g", Payload: payload, CredHash: credHash(payload), Version: 1, Origin: id.NodeID, OriginPub: id.Pub} r.Sig = id.sign(r.signingBytes()) + r.MountSig = id.sign(r.mountSigningBytes()) // claim someone else's origin without their key. r.Origin = other.NodeID if r.verify() { @@ -277,6 +278,7 @@ func TestCredRecordVerifyBindsOriginMount(t *testing.T) { OriginMount: "/115-a", } r.Sig = id.sign(r.signingBytes()) + r.MountSig = id.sign(r.mountSigningBytes()) if !r.verify() { t.Fatal("freshly signed record must verify") } @@ -286,6 +288,97 @@ func TestCredRecordVerifyBindsOriginMount(t *testing.T) { } } +// legacyCredSigningBytes is the credential wire-signature format used before +// OriginMount was bound separately. Keeping this fixture lets an upgrade prove +// it can still read state.json written by the deployed 53bc1db protocol. +func legacyCredSigningBytes(r *credRecord) []byte { + var b []byte + b = append(b, r.GroupID...) + b = append(b, 0) + b = append(b, r.OriginDriver...) + b = append(b, 0) + b = append(b, r.CredHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + return b +} + +func TestCredRecordVerifiesLegacySignature(t *testing.T) { + id, _ := newIdentity() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"legacy"`)} + r := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 7, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115", + } + r.Sig = id.sign(legacyCredSigningBytes(r)) + if !r.verify() { + t.Fatal("a 53bc1db credential must remain verifiable after upgrade") + } +} + +func TestLocalCredChangeMigratesLegacySourceWithoutTokenChange(t *testing.T) { + id, _ := newIdentity() + s := newStore() + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"legacy"`)} + legacy := &credRecord{ + GroupID: "g", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: "/115", + } + legacy.Sig = id.sign(legacyCredSigningBytes(legacy)) + if !s.mergeCred(legacy) { + t.Fatal("legacy credential should be accepted into local state") + } + migrated, ok := s.localCredChange(id, "g", "115 Open", "/115", payload, 2) + if !ok || migrated == nil || len(migrated.MountSig) == 0 { + t.Fatal("a healthy local source must rewrite its legacy record with MountSig") + } + if migrated.signatureKind() != credentialSignatureMountBound { + t.Fatal("migrated record must use the rolling-compatible mount-bound format") + } +} + +func TestSameOriginMultipleMountCandidatesArePreserved(t *testing.T) { + id, _ := newIdentity() + s := newStore() + first := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"first"`)} + second := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"second"`)} + if _, ok := s.localCredChange(id, "g", "115 Open", "/115-a", first, 1); !ok { + t.Fatal("first local mount candidate was not recorded") + } + if _, ok := s.localCredChange(id, "g", "115 Open", "/115-b", second, 2); !ok { + t.Fatal("second local mount candidate was not recorded") + } + got := s.credsForGroup("g") + if len(got) != 2 { + t.Fatalf("candidate count = %d, want separate candidates for both mounts: %#v", len(got), got) + } + if !s.dropOwnCred("g", id.NodeID, "/115-a") { + t.Fatal("invalidating one source mount must drop that source candidate") + } + remaining := s.credsForGroup("g") + if len(remaining) != 1 || remaining[0].OriginMount != "/115-b" { + t.Fatalf("invalidating /115-a removed the wrong candidate: %#v", remaining) + } +} + // dropOwnCred must remove only our own-authored record so a dead/stale local // token can never out-rank (by Lamport version) a peer's genuinely-valid one and // block recovery. A peer-authored record must survive. @@ -295,7 +388,7 @@ func TestDropOwnCred(t *testing.T) { s := newStore() // nothing recorded yet -> nothing to drop. - if s.dropOwnCred("g1", id1.NodeID) { + if s.dropOwnCred("g1", id1.NodeID, "/115") { t.Fatal("dropping a non-existent record must return false") } @@ -304,7 +397,7 @@ func TestDropOwnCred(t *testing.T) { if _, ok := s.localCredChange(id1, "g1", "115", "/115", payloadA, 1); !ok { t.Fatal("own credential should be recorded") } - if !s.dropOwnCred("g1", id1.NodeID) { + if !s.dropOwnCred("g1", id1.NodeID, "/115") { t.Fatal("own credential should be dropped") } if _, ok := s.getCred("g1"); ok { @@ -318,7 +411,7 @@ func TestDropOwnCred(t *testing.T) { if !s.mergeCred(r) { t.Fatal("peer credential should merge") } - if s.dropOwnCred("g2", id1.NodeID) { + if s.dropOwnCred("g2", id1.NodeID, "/115") { t.Fatal("a peer-authored credential must never be dropped as own") } if _, ok := s.getCred("g2"); !ok { @@ -326,23 +419,40 @@ func TestDropOwnCred(t *testing.T) { } } -// canOfferCred enforces "only share a token proven valid" for every candidate. -// Receiving a signed peer record is not proof that this node can use it, so it -// must not turn this node into a blind relay for an unvalidated credential. +// canOfferCred requires a local health proof for this node's own candidates. +// A signed, group-authorized peer candidate remains relayable: the hub need not +// overwrite a healthy local mount merely to forward NAT peers' recovery path. func TestCanOfferCred(t *testing.T) { id1, _ := newIdentity() // self id2, _ := newIdentity() // peer - m := &Manager{id: id1, state: newStore()} + admin, _ := newIdentity() + s := newStore() + s.setGroups(admin, []group{{ + ID: "g1", + Members: []member{{NodeID: id1.NodeID, MountPath: "/local"}, {NodeID: id2.NodeID, MountPath: "/peer"}}, + }}, 1) + m := &Manager{id: id1, state: s} if m.canOfferCred(nil) { t.Fatal("a nil record is never offerable") } - // A peer-authored record is not offerable without a matching local healthy - // mount. This manager has neither a group nor a live validated candidate. - peer := &credRecord{GroupID: "g1", Origin: id2.NodeID} - if m.canOfferCred(peer) { - t.Fatal("a peer-authored record must NOT be relayed before local validation") + // The hub may relay a peer-authored candidate without first applying it to a + // working local mount; its signature and group membership authorize that. + payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"peer"`)} + peer := &credRecord{ + GroupID: "g1", + OriginDriver: "115 Open", + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: id2.NodeID, + OriginPub: id2.Pub, + OriginMount: "/peer", + } + peer.Sig = id2.sign(peer.signingBytes()) + if !m.canOfferCred(peer) { + t.Fatal("an authorized peer candidate must remain relayable") } // our own record, but no group/healthy mount -> not offerable. @@ -379,7 +489,7 @@ func TestAbsorbCredsRejectsUnauthorizedOriginMount(t *testing.T) { if got := m.absorbCreds([]*credRecord{bad}); len(got) != 0 { t.Fatalf("unauthorized origin mount was accepted: %#v", got) } - if _, ok := s.getCredForOrigin("g1", memberID.NodeID); ok { + if _, ok := s.getCredForSource("g1", memberID.NodeID, "/115"); ok { t.Fatal("unauthorized credential must not enter replicated state") } diff --git a/internal/op/hook.go b/internal/op/hook.go index 63656146d6..434c302380 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -4,6 +4,7 @@ import ( "context" "regexp" "strings" + "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -99,8 +100,13 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { // Storage type StorageHook func(typ string, storage driver.Driver) +type StorageHealthHook func(storage driver.Driver) -var storageHooks = make([]StorageHook, 0) +var ( + storageHooks = make([]StorageHook, 0) + storageHealthHooks = make([]StorageHealthHook, 0) + storageTokenStatusMu sync.Mutex +) func callStorageHooks(typ string, storage driver.Driver) { for _, hook := range storageHooks { @@ -112,6 +118,19 @@ func RegisterStorageHook(hook StorageHook) { storageHooks = append(storageHooks, hook) } +// RegisterStorageHealthHook receives a cheap proof that an authenticated +// request succeeded. It is distinct from StorageHook: callers use it for +// in-memory liveness only, never for database writes or lifecycle work. +func RegisterStorageHealthHook(hook StorageHealthHook) { + storageHealthHooks = append(storageHealthHooks, hook) +} + +func NotifyStorageTokenHealthy(storage driver.Driver) { + for _, hook := range storageHealthHooks { + hook(storage) + } +} + // NotifyStorageTokenInvalid signals that a storage's credentials were found to be // invalid/expired while in use (as opposed to successfully refreshed). It fires // the storage hook with the "token-invalid" type so listeners — notably cluster @@ -119,23 +138,28 @@ func RegisterStorageHook(hook StorageHook) { // propagating the broken token. Drivers may call this when an API call fails with // an unrecoverable auth error. func NotifyStorageTokenInvalid(storage driver.Driver) { - markStorageTokenInvalidStatus(storage) - go callStorageHooks("token-invalid", storage) + storageTokenStatusMu.Lock() + changed := markStorageTokenInvalidStatus(storage) + storageTokenStatusMu.Unlock() + if changed { + go callStorageHooks("token-invalid", storage) + } } -func markStorageTokenInvalidStatus(storage driver.Driver) { +func markStorageTokenInvalidStatus(storage driver.Driver) bool { st := storage.GetStorage() if st == nil || st.Disabled || st.Status != WORK { - return + return false } const invalidStatus = "token invalid" st.SetStatus(invalidStatus) if st.ID == 0 { - return + return true } if err := db.UpdateStorageStatus(st.ID, invalidStatus); err != nil { log.Errorf("failed mark storage token invalid: %s", err) } + return true } // NotifyStorageTokenValid signals that a storage's credentials were just proven @@ -144,20 +168,25 @@ func markStorageTokenInvalidStatus(storage driver.Driver) { // (re)share the proven token with peers. Drivers should call this only on a real // transition or stale-status recovery to avoid per-request churn. func NotifyStorageTokenValid(storage driver.Driver) { - restoreStorageTokenValidStatus(storage) - go callStorageHooks("token-valid", storage) + storageTokenStatusMu.Lock() + changed := restoreStorageTokenValidStatus(storage) + storageTokenStatusMu.Unlock() + if changed { + go callStorageHooks("token-valid", storage) + } } -func restoreStorageTokenValidStatus(storage driver.Driver) { +func restoreStorageTokenValidStatus(storage driver.Driver) bool { st := storage.GetStorage() if st == nil || st.Disabled || st.Status == WORK { - return + return false } st.SetStatus(WORK) if st.ID == 0 { - return + return true } if err := db.UpdateStorageStatus(st.ID, WORK); err != nil { log.Errorf("failed mark storage token valid: %s", err) } + return true } diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go index 22515652a3..33ba289d4b 100644 --- a/internal/op/hook_test.go +++ b/internal/op/hook_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "testing" + "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -86,3 +87,27 @@ func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { t.Fatal("token-invalid must persist a non-work status before peer recovery") } } + +func TestNotifyStorageTokenHealthyDoesNotChangeLifecycleStatus(t *testing.T) { + d := &tokenValidStorageDriver{Storage: model.Storage{Status: op.WORK}} + called := make(chan struct{}, 1) + op.RegisterStorageHealthHook(func(got driver.Driver) { + if got != d { + return + } + select { + case called <- struct{}{}: + default: + } + }) + + op.NotifyStorageTokenHealthy(d) + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("health hook was not invoked") + } + if d.GetStorage().Status != op.WORK { + t.Fatal("a healthy proof must not mutate the storage lifecycle status") + } +} From bc58424d0a81ba959e6d89fadb2a49c8ac9d728d Mon Sep 17 00:00:00 2001 From: ironbox Date: Sun, 19 Jul 2026 16:11:19 +0800 Subject: [PATCH 101/107] fix(cluster): recover invalid credential candidates - propagate signed source tombstones so relays cannot replay invalid tokens\n- quarantine failed remote candidates per mount and try bounded alternatives\n- keep invalid recovery events flowing without repeated status writes --- internal/cluster/engine.go | 180 +++++++++++++---- internal/cluster/state.go | 355 +++++++++++++++++++++++++++++++-- internal/cluster/state_test.go | 91 +++++++++ internal/cluster/transport.go | 18 +- internal/op/hook.go | 14 +- internal/op/hook_test.go | 26 +++ 6 files changed, 612 insertions(+), 72 deletions(-) diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 50644e80d7..429357e9fd 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -27,6 +27,10 @@ const ( peerLivenessSec = 130 // maxEvents bounds the in-memory activity log surfaced to the UI. maxEvents = 60 + // candidateRecoveryCooldown prevents an invalid mount from cycling through + // every persisted credential on each caller retry. A new credential push is + // still applied immediately; this only bounds local fallback attempts. + candidateRecoveryCooldown = 5 * time.Minute ) // Manager is the running cluster credential-sync engine for this node. @@ -42,6 +46,7 @@ type Manager struct { recoveryMu sync.Mutex recoveryLocks map[string]*sync.Mutex // per local mount: serialize credential adoption + recoveryLast map[string]time.Time dialMu sync.Mutex dialing map[string]bool // peer URLs with an in-flight/live outbound dial @@ -80,6 +85,7 @@ func Init(dataDir string) (*Manager, error) { conns: newConnRegistry(), dialing: make(map[string]bool), recoveryLocks: make(map[string]*sync.Mutex), + recoveryLast: make(map[string]time.Time), stopCh: make(chan struct{}), } if _, err := m.cfgStore.loadOrInit(); err != nil { @@ -298,6 +304,7 @@ func (m *Manager) helloMessage() *syncMessage { Nodes: m.knownNodesForPEX(), Groups: &gd, GroupsVer: gd.Version, + Revocations: m.state.revocationSnapshot(), CredDigests: m.offerableDigests(), } } @@ -312,6 +319,7 @@ func (m *Manager) announceMessage() *syncMessage { Nodes: m.knownNodesForPEX(), Groups: &gd, GroupsVer: gd.Version, + Revocations: m.state.revocationSnapshot(), CredDigests: m.offerableDigests(), } } @@ -357,8 +365,9 @@ func (m *Manager) seedLocalCreds() { // it can't dominate a peer's valid one. Persisted peer candidates are // tried before a network pull so restart recovery also works offline. for _, g := range groups { - if m.state.dropOwnCred(g.ID, m.id.NodeID, st.MountPath) { + if rev, ok := m.revokeOwnCandidate(g.ID, st.MountPath); ok { changed = true + m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) } for _, candidate := range m.state.credsForGroup(g.ID) { m.applyCredRecord(candidate) @@ -398,6 +407,14 @@ func (m *Manager) seedLocalCreds() { } } +// revokeOwnCandidate removes the local source candidate and returns a signed +// tombstone that every relay can retain. The tombstone's version is newer than +// the removed credential, so an old relay replay cannot re-enter state while a +// future credential rotation from this source remains valid. +func (m *Manager) revokeOwnCandidate(groupID, mount string) (*credRevocation, bool) { + return m.state.revokeOwnCred(m.id, groupID, mount, now()) +} + // onStorageHook reacts to local storage lifecycle/credential changes. func (m *Manager) onStorageHook(typ string, d driver.Driver) { cfg := m.cfgStore.get() @@ -455,19 +472,26 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { } // Our token died. Drop our own (now-stale) credential record so its Lamport // version can't out-rank a peer's valid one, then pull a fresh credential - // from a healthy peer. - var dropped bool + // from a healthy peer. The currently-used credential is quarantined for + // this mount even when it originated on a peer: that prevents a restart + // from immediately retrying the exact 40140125/26 candidate. + var changed bool credID := credHash(extractCreds(st.Addition)) for _, g := range groups { m.state.forgetCandidateHealth(g.ID, credID) - if m.state.dropOwnCred(g.ID, m.id.NodeID, st.MountPath) { - dropped = true + m.state.markCandidateFailed(g.ID, st.MountPath, credID, now()) + changed = true + if rev, ok := m.revokeOwnCandidate(g.ID, st.MountPath); ok { m.recordEvent("invalidate", g.ID, fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) + m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) + } else { + m.recordEvent("invalidate", g.ID, + fmt.Sprintf("%s token invalid — quarantined failed peer candidate", st.MountPath)) } - m.pullGroup(g.ID) + go m.recoverKnownCandidates(g.ID, st.MountPath, credID) } - if dropped { + if changed { m.persist() } } @@ -492,10 +516,52 @@ func (m *Manager) onStorageHealthy(d driver.Driver) { for _, g := range m.state.groupsForMount(m.id.NodeID, st.MountPath) { if m.state.hasCredHash(g.ID, credID) { m.state.markCandidateHealthy(g.ID, credID, now()) + m.state.clearCandidateFailed(g.ID, st.MountPath, credID) } } } +func (m *Manager) beginCandidateRecovery(groupID, mount string) bool { + key := groupID + "\x00" + mount + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + n := time.Now() + if last := m.recoveryLast[key]; !last.IsZero() && n.Sub(last) < candidateRecoveryCooldown { + return false + } + m.recoveryLast[key] = n + return true +} + +// recoverKnownCandidates tries already-held alternatives once, in deterministic +// order, after a token dies. It never refreshes on a timer: each candidate is +// attempted at most once per mount per cooldown window, then a pull waits for a +// peer to publish a genuinely newer credential. +func (m *Manager) recoverKnownCandidates(groupID, mount, failedHash string) { + if !m.beginCandidateRecovery(groupID, mount) { + return + } + changed := false + for _, candidate := range m.state.recoveryCandidates(groupID, mount, failedHash) { + if !m.credentialAuthorized(candidate) { + continue + } + attempted, ok := m.applyCredRecordToMount(candidate, mount) + if !attempted { + continue + } + if ok { + return + } + m.state.markCandidateFailed(groupID, mount, candidate.CredHash, now()) + changed = true + } + if changed { + m.persist() + } + m.pullGroup(groupID) +} + // pullGroup asks peers for the latest credential of a group. func (m *Manager) pullGroup(groupID string) { if !m.cfgStore.get().active() { @@ -587,40 +653,53 @@ func (m *Manager) applyCredRecord(r *credRecord) { if !ok { return } - ctx := context.Background() for _, mp := range g.mountsForNode(m.id.NodeID) { + m.applyCredRecordToMount(r, mp) + } +} + +// applyCredRecordToMount adopts one candidate on one local mount. It returns +// whether a re-initialization was attempted and whether it completed with a +// working storage; recovery uses that distinction to quarantine only candidates +// that actually failed on this node. +func (m *Manager) applyCredRecordToMount(r *credRecord, mp string) (attempted, success bool) { + if r == nil || !m.cfgStore.get().ApplyRemote || m.state.candidateFailed(r.GroupID, mp, r.CredHash) { + return false, false + } + ctx := context.Background() + { lock := m.recoveryLock(mp) lock.Lock() - func() { - defer lock.Unlock() - d, err := op.GetStorageByMountPath(mp) - if err != nil { - return - } - st := *d.GetStorage() // copy; preserve ID/Status/local fields - if r.OriginDriver != "" && st.Driver != r.OriginDriver { - utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", - r.GroupID, mp, st.Driver, r.OriginDriver) - return - } - if st.Status == op.WORK { - // A peer's newer Lamport value is a recovery candidate, not authority - // to overwrite credentials this mount has already proved locally. - return - } - newAdd, changed := applyCreds(st.Addition, r.Payload) - if !changed { - return // already has these credentials — no churn, no re-init - } - st.Addition = newAdd - if err := op.UpdateStorage(ctx, st); err != nil { - utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) - return - } - m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) - utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) - }() + defer lock.Unlock() + d, err := op.GetStorageByMountPath(mp) + if err != nil { + return false, false + } + st := *d.GetStorage() // copy; preserve ID/Status/local fields + if r.OriginDriver != "" && st.Driver != r.OriginDriver { + utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", + r.GroupID, mp, st.Driver, r.OriginDriver) + return false, false + } + if st.Status == op.WORK { + // A peer's newer Lamport value is a recovery candidate, not authority + // to overwrite credentials this mount has already proved locally. + return false, false + } + newAdd, changed := applyCreds(st.Addition, r.Payload) + if !changed { + return false, false // already has these credentials — no churn, no re-init + } + st.Addition = newAdd + if err := op.UpdateStorage(ctx, st); err != nil { + utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) + return true, false + } + m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) + utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) + return true, true } + return false, false } // ---- absorb (merge) helpers ---- @@ -669,6 +748,24 @@ func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { return merged } +func (m *Manager) absorbRevocations(revs []*credRevocation) []*credRevocation { + var merged []*credRevocation + for _, rev := range revs { + if !credentialRevocationAuthorizedByGroups(m.state.groupList(), rev) { + continue + } + if m.state.mergeRevocation(rev) { + merged = append(merged, rev) + m.recordEvent("revoke", rev.GroupID, + fmt.Sprintf("%s revoked candidate from %s", rev.OriginMount, shortNode(rev.Origin))) + } + } + if len(merged) > 0 { + m.persist() + } + return merged +} + // credentialAuthorized verifies that a record signer is an explicit member of // the signed group at the exact mount it claims. The signature proves who wrote // the record; the group document proves that writer may supply this group. @@ -752,8 +849,9 @@ func (m *Manager) buildReply(in *syncMessage) *syncMessage { appendCred(cur) } } + reply.Revocations = m.state.revocationSnapshot() - if reply.Groups == nil && len(reply.Creds) == 0 && len(reply.Wants) == 0 { + if reply.Groups == nil && len(reply.Creds) == 0 && len(reply.Revocations) == 0 && len(reply.Wants) == 0 { return nil } return reply @@ -860,6 +958,12 @@ func (m *Manager) handleFrame(c *peerConn, data []byte) { relay.Groups = &gd relayHas = true } + // Candidate tombstones must converge before credentials: otherwise a relay + // can briefly resurrect an old signed token while a revocation is in flight. + if revoked := m.absorbRevocations(msg.Revocations); len(revoked) > 0 { + relay.Revocations = revoked + relayHas = true + } // Credentials. if merged := m.absorbCreds(msg.Creds); len(merged) > 0 { // A relay must not amplify a peer credential merely because its signature diff --git a/internal/cluster/state.go b/internal/cluster/state.go index ae05e5dcc7..1c7498d69f 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -210,6 +210,53 @@ func (r *credRecord) dominates(other *credRecord) bool { return r.CredHash > other.CredHash } +// credRevocation is a signed tombstone for one source mount's credential +// candidate. It is intentionally separate from a credential record: a relay +// must retain and forward the tombstone after it has discarded the secret, or +// an old signed credential can be replayed back to its source. +type credRevocation struct { + GroupID string `json:"group_id"` + Origin string `json:"origin"` + OriginPub []byte `json:"origin_pub"` + OriginMount string `json:"origin_mount"` + CredHash string `json:"cred_hash"` + Version uint64 `json:"version"` + UpdatedAt int64 `json:"updated_at"` + Sig []byte `json:"sig"` +} + +func (r *credRevocation) signingBytes() []byte { + var b []byte + b = append(b, "openlist/cluster/credential-revocation/v1\x00"...) + b = append(b, r.GroupID...) + b = append(b, 0) + b = append(b, r.OriginMount...) + b = append(b, 0) + b = append(b, r.CredHash...) + b = append(b, 0) + v := make([]byte, 8) + for i := 0; i < 8; i++ { + v[i] = byte(r.Version >> (8 * uint(i))) + } + b = append(b, v...) + b = append(b, 0) + b = append(b, r.Origin...) + return b +} + +func (r *credRevocation) verify() bool { + return r != nil && r.GroupID != "" && r.OriginMount != "" && r.CredHash != "" && + r.Version > 0 && nodeIDFromPub(r.OriginPub) == r.Origin && + verifySig(r.OriginPub, r.signingBytes(), r.Sig) +} + +func (r *credRevocation) dominates(other *credRevocation) bool { + if other == nil || r.Version != other.Version { + return other == nil || r.Version > other.Version + } + return r.CredHash > other.CredHash +} + // credDigest is the compact (no-secret) advert of a held credential record. type credDigest struct { GroupID string `json:"group_id"` @@ -276,6 +323,13 @@ type store struct { // own current candidate, keyed by the signing origin and mount; Lamport // ordering only resolves successive credentials from the same source mount. creds map[string]map[string]*credRecord // group id -> (origin, mount) -> candidate + // revocations are source-signed tombstones. They prevent a relay from + // reintroducing a credential after that source proved it invalid. + revocations map[string]map[string]*credRevocation // group id -> (origin, mount) -> tombstone + // failedCandidates is node-local only: a candidate may be valid on its origin + // yet unusable from this node. Persisting the quarantine avoids a restart + // repeatedly refreshing the exact token that just failed here. + failedCandidates map[string]map[string]map[string]int64 // group id -> local mount -> credential hash -> failed unix second // candidateHealth is an in-memory proof that this node has actually used a // credential successfully. It is deliberately not persisted: a restart must // prove the credential again instead of trusting a stale WORK status. @@ -288,9 +342,11 @@ const candidateLeaseSec int64 = 15 * 60 func newStore() *store { return &store{ - creds: make(map[string]map[string]*credRecord), - candidateHealth: make(map[string]map[string]int64), - inventory: make(map[string]*nodeInfo), + creds: make(map[string]map[string]*credRecord), + revocations: make(map[string]map[string]*credRevocation), + failedCandidates: make(map[string]map[string]map[string]int64), + candidateHealth: make(map[string]map[string]int64), + inventory: make(map[string]*nodeInfo), } } @@ -404,6 +460,23 @@ func credentialAuthorizedByGroups(groups []group, r *credRecord) bool { return r.signatureKind() != credentialSignatureLegacy || membersForOrigin == 1 } +func credentialRevocationAuthorizedByGroups(groups []group, rev *credRevocation) bool { + if !rev.verify() { + return false + } + for _, g := range groups { + if g.ID != rev.GroupID { + continue + } + for _, member := range g.Members { + if member.NodeID == rev.Origin && member.MountPath == rev.OriginMount { + return true + } + } + } + return false +} + // ---- creds ---- func (s *store) getCred(groupID string) (*credRecord, bool) { @@ -557,6 +630,9 @@ func (s *store) mergeCred(r *credRecord) bool { s.creds[r.GroupID] = byOrigin } key := candidateKey(r.Origin, r.OriginMount) + if rev := s.revocations[r.GroupID][key]; rev != nil && rev.Version >= r.Version { + return false // a source-signed tombstone blocks relay replay of this version + } cur, ok := byOrigin[key] if ok { if cur.CredHash == r.CredHash { @@ -571,26 +647,126 @@ func (s *store) mergeCred(r *credRecord) bool { return true } -// dropOwnCred removes our own-authored credential record for a group. It is used -// when the local token is found invalid: a dead/stale credential must not linger -// as a dominating record (its Lamport version could otherwise out-rank a peer's -// genuinely-valid credential and block recovery). Returns true if a record was -// removed. Peer-authored records are never touched here. +// dropOwnCred removes this source mount's local candidate without revoking it. +// It is used for uncertain startup failures (for example a transient TLS +// timeout), where this node must stop advertising the candidate but must not +// globally invalidate a credential that may still work on its origin. func (s *store) dropOwnCred(groupID, selfID, mount string) bool { s.mu.Lock() defer s.mu.Unlock() - if byOrigin := s.creds[groupID]; byOrigin != nil { - key := candidateKey(selfID, mount) - if _, ok := byOrigin[key]; !ok { - return false + byOrigin := s.creds[groupID] + if byOrigin == nil { + return false + } + key := candidateKey(selfID, mount) + if _, ok := byOrigin[key]; !ok { + return false + } + delete(byOrigin, key) + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + return true +} + +// revokeOwnCred removes a source candidate only after the provider has +// explicitly declared it invalid. Unlike dropOwnCred, this emits a signed +// tombstone so relays cannot replay the candidate back to its origin. +func (s *store) revokeOwnCred(id *identity, groupID, mount string, at int64) (*credRevocation, bool) { + if id == nil || at <= 0 { + return nil, false + } + s.mu.Lock() + defer s.mu.Unlock() + byOrigin := s.creds[groupID] + if byOrigin == nil { + return nil, false + } + key := candidateKey(id.NodeID, mount) + current := byOrigin[key] + if current == nil { + return nil, false + } + s.lamport++ + rev := &credRevocation{ + GroupID: groupID, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + CredHash: current.CredHash, + Version: s.lamport, + UpdatedAt: at, + } + rev.Sig = id.sign(rev.signingBytes()) + byRevocation := s.revocations[groupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[groupID] = byRevocation + } + byRevocation[key] = rev + delete(byOrigin, key) + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + cp := *rev + return &cp, true +} + +func (s *store) revocationSnapshot() []*credRevocation { + s.mu.RLock() + defer s.mu.RUnlock() + var out []*credRevocation + for _, bySource := range s.revocations { + for _, rev := range bySource { + if !rev.verify() { + continue + } + cp := *rev + out = append(out, &cp) } - delete(byOrigin, key) - if len(byOrigin) == 0 { - delete(s.creds, groupID) + } + sort.Slice(out, func(i, j int) bool { + if out[i].GroupID != out[j].GroupID { + return out[i].GroupID < out[j].GroupID } - return true + if out[i].Origin != out[j].Origin { + return out[i].Origin < out[j].Origin + } + return out[i].OriginMount < out[j].OriginMount + }) + return out +} + +// mergeRevocation applies a verified tombstone and removes any same-source +// candidate that it supersedes. A later credential from the source remains +// allowed because it must carry a strictly greater Lamport version. +func (s *store) mergeRevocation(rev *credRevocation) bool { + if rev == nil { + return false } - return false + s.mu.Lock() + defer s.mu.Unlock() + s.observe(rev.Version) + key := candidateKey(rev.Origin, rev.OriginMount) + byRevocation := s.revocations[rev.GroupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[rev.GroupID] = byRevocation + } + if current := byRevocation[key]; current != nil && !rev.dominates(current) { + return false + } + cp := *rev + byRevocation[key] = &cp + if byOrigin := s.creds[rev.GroupID]; byOrigin != nil { + if current := byOrigin[key]; current != nil && current.Version <= rev.Version { + delete(byOrigin, key) + if len(byOrigin) == 0 { + delete(s.creds, rev.GroupID) + } + } + } + return true } // pruneCreds drops credential records for groups that no longer exist. @@ -607,6 +783,16 @@ func (s *store) pruneCreds() { delete(s.candidateHealth, id) } } + for id := range s.revocations { + if _, ok := live[id]; !ok { + delete(s.revocations, id) + } + } + for id := range s.failedCandidates { + if _, ok := live[id]; !ok { + delete(s.failedCandidates, id) + } + } } func (s *store) markCandidateHealthy(groupID, hash string, at int64) { @@ -646,6 +832,63 @@ func (s *store) candidateHealthy(groupID, hash string, at int64) bool { return provenAt > 0 && at >= provenAt && at-provenAt <= candidateLeaseSec } +func (s *store) markCandidateFailed(groupID, mount, hash string, at int64) { + if groupID == "" || mount == "" || hash == "" || at <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byMount := s.failedCandidates[groupID] + if byMount == nil { + byMount = make(map[string]map[string]int64) + s.failedCandidates[groupID] = byMount + } + byHash := byMount[mount] + if byHash == nil { + byHash = make(map[string]int64) + byMount[mount] = byHash + } + byHash[hash] = at +} + +func (s *store) clearCandidateFailed(groupID, mount, hash string) { + if groupID == "" || mount == "" || hash == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + byMount := s.failedCandidates[groupID] + byHash := byMount[mount] + delete(byHash, hash) + if len(byHash) == 0 { + delete(byMount, mount) + } + if len(byMount) == 0 { + delete(s.failedCandidates, groupID) + } +} + +func (s *store) candidateFailed(groupID, mount, hash string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.failedCandidates[groupID][mount][hash] > 0 +} + +// recoveryCandidates returns persisted alternatives for one local mount. The +// current failed credential and any locally quarantined candidate are excluded; +// the order is deterministic so a recovery attempt is bounded and reproducible. +func (s *store) recoveryCandidates(groupID, mount, failedHash string) []*credRecord { + candidates := s.credsForGroup(groupID) + out := make([]*credRecord, 0, len(candidates)) + for _, candidate := range candidates { + if candidate.CredHash == failedHash || s.candidateFailed(groupID, mount, candidate.CredHash) { + continue + } + out = append(out, candidate) + } + return out +} + // ---- inventory ---- func (s *store) setLocalInventory(info *nodeInfo) { @@ -702,10 +945,21 @@ func (s *store) dialableAddrs(selfID string) []string { // ---- persistence ---- type persistedState struct { - Lamport uint64 `json:"lamport"` - Groups groupDoc `json:"groups"` - Creds []*credRecord `json:"creds"` - Inventory []*nodeInfo `json:"inventory"` + Lamport uint64 `json:"lamport"` + Groups groupDoc `json:"groups"` + Creds []*credRecord `json:"creds"` + Revocations []*credRevocation `json:"revocations,omitempty"` + FailedCandidates []candidateFailure `json:"failed_candidates,omitempty"` + Inventory []*nodeInfo `json:"inventory"` +} + +// candidateFailure is local-only recovery metadata; it deliberately contains no +// credential payload, only the already-public credential hash. +type candidateFailure struct { + GroupID string `json:"group_id"` + Mount string `json:"mount"` + CredHash string `json:"cred_hash"` + FailedAt int64 `json:"failed_at"` } func (s *store) export() persistedState { @@ -717,6 +971,22 @@ func (s *store) export() persistedState { ps.Creds = append(ps.Creds, r) } } + for _, bySource := range s.revocations { + for _, rev := range bySource { + if rev.verify() { + ps.Revocations = append(ps.Revocations, rev) + } + } + } + for groupID, byMount := range s.failedCandidates { + for mount, byHash := range byMount { + for hash, failedAt := range byHash { + ps.FailedCandidates = append(ps.FailedCandidates, candidateFailure{ + GroupID: groupID, Mount: mount, CredHash: hash, FailedAt: failedAt, + }) + } + } + } for _, n := range s.inventory { ps.Inventory = append(ps.Inventory, n) } @@ -733,17 +1003,40 @@ func (s *store) load(ps persistedState) { s.groups = groupDoc{} } s.creds = make(map[string]map[string]*credRecord) + s.revocations = make(map[string]map[string]*credRevocation) s.candidateHealth = make(map[string]map[string]int64) + s.failedCandidates = make(map[string]map[string]map[string]int64) + for _, rev := range ps.Revocations { + if !credentialRevocationAuthorizedByGroups(s.groups.Groups, rev) { + continue + } + bySource := s.revocations[rev.GroupID] + if bySource == nil { + bySource = make(map[string]*credRevocation) + s.revocations[rev.GroupID] = bySource + } + key := candidateKey(rev.Origin, rev.OriginMount) + if current := bySource[key]; current == nil || rev.dominates(current) { + cp := *rev + bySource[key] = &cp + } + if rev.Version > s.lamport { + s.lamport = rev.Version + } + } for _, r := range ps.Creds { if !credentialAuthorizedByGroups(s.groups.Groups, r) { continue } + key := candidateKey(r.Origin, r.OriginMount) + if rev := s.revocations[r.GroupID][key]; rev != nil && rev.Version >= r.Version { + continue + } byOrigin := s.creds[r.GroupID] if byOrigin == nil { byOrigin = make(map[string]*credRecord) s.creds[r.GroupID] = byOrigin } - key := candidateKey(r.Origin, r.OriginMount) if cur, ok := byOrigin[key]; !ok || r.dominates(cur) { cp := *r byOrigin[key] = &cp @@ -752,6 +1045,24 @@ func (s *store) load(ps persistedState) { s.lamport = r.Version } } + for _, failed := range ps.FailedCandidates { + if failed.GroupID == "" || failed.Mount == "" || failed.CredHash == "" || failed.FailedAt <= 0 { + continue + } + byMount := s.failedCandidates[failed.GroupID] + if byMount == nil { + byMount = make(map[string]map[string]int64) + s.failedCandidates[failed.GroupID] = byMount + } + byHash := byMount[failed.Mount] + if byHash == nil { + byHash = make(map[string]int64) + byMount[failed.Mount] = byHash + } + if failed.FailedAt > byHash[failed.CredHash] { + byHash[failed.CredHash] = failed.FailedAt + } + } s.inventory = make(map[string]*nodeInfo, len(ps.Inventory)) for _, n := range ps.Inventory { if n == nil || n.NodeID == "" { diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 66b48e7a1e..a7041682a3 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -419,6 +419,97 @@ func TestDropOwnCred(t *testing.T) { } } +// A source that has proved one of its credentials invalid must be able to +// prevent an old signed copy held by a relay from entering its state again. +// Without this, the source drops its own candidate, sends a pull, and a hub +// immediately returns the exact same stale candidate as a peer record. +func TestDroppedOwnCredentialRejectsStaleRelayReplay(t *testing.T) { + source, _ := newIdentity() + s := newStore() + payload := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"dead-access"`), + "refresh_token": json.RawMessage(`"dead-refresh"`), + } + record, ok := s.localCredChange(source, "g1", "115 Open", "/storage/115", payload, 1) + if !ok { + t.Fatal("source candidate was not recorded") + } + if rev, ok := s.revokeOwnCred(source, "g1", "/storage/115", 2); !ok || !rev.verify() { + t.Fatal("source candidate was not revoked after invalidation") + } + + if s.mergeCred(record) { + t.Fatal("stale signed candidate from a relay was accepted after source invalidation") + } + if _, ok := s.getCredForSource("g1", source.NodeID, "/storage/115"); ok { + t.Fatal("revoked source candidate re-entered state") + } +} + +func TestRecoveryCandidatesSkipCurrentAndQuarantinedCredentials(t *testing.T) { + idA, _ := newIdentity() + idB, _ := newIdentity() + idC, _ := newIdentity() + s := newStore() + for _, tc := range []struct { + id *identity + mount string + payload map[string]json.RawMessage + }{ + {idA, "/a", map[string]json.RawMessage{"access_token": json.RawMessage(`"A"`)}}, + {idB, "/b", map[string]json.RawMessage{"access_token": json.RawMessage(`"B"`)}}, + {idC, "/c", map[string]json.RawMessage{"access_token": json.RawMessage(`"C"`)}}, + } { + if _, ok := s.localCredChange(tc.id, "g1", "115 Open", tc.mount, tc.payload, 1); !ok { + t.Fatalf("candidate %s was not recorded", tc.mount) + } + } + failedHash := credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"A"`)}) + currentHash := credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"B"`)}) + s.markCandidateFailed("g1", "/local", failedHash, 2) + + candidates := s.recoveryCandidates("g1", "/local", currentHash) + if len(candidates) != 1 || candidates[0].CredHash != credHash(map[string]json.RawMessage{"access_token": json.RawMessage(`"C"`)}) { + t.Fatalf("recovery candidates = %#v, want only remaining C candidate", candidates) + } +} + +func TestSignedRevocationRemovesRelayCopyButAllowsNewerRotation(t *testing.T) { + source, _ := newIdentity() + relay := newStore() + sourceState := newStore() + payloadA := map[string]json.RawMessage{"access_token": json.RawMessage(`"old"`)} + old, ok := sourceState.localCredChange(source, "g1", "115 Open", "/storage/115", payloadA, 1) + if !ok { + t.Fatal("source candidate was not recorded") + } + if !relay.mergeCred(old) { + t.Fatal("relay did not accept initial source candidate") + } + rev, ok := sourceState.revokeOwnCred(source, "g1", "/storage/115", 2) + if !ok || !rev.verify() { + t.Fatal("source revocation was not signed") + } + if !relay.mergeRevocation(rev) { + t.Fatal("relay did not accept source revocation") + } + if _, ok := relay.getCredForSource("g1", source.NodeID, "/storage/115"); ok { + t.Fatal("relay retained revoked candidate") + } + if relay.mergeCred(old) { + t.Fatal("relay accepted revoked candidate replay") + } + + payloadB := map[string]json.RawMessage{"access_token": json.RawMessage(`"rotated"`)} + newer, ok := sourceState.localCredChange(source, "g1", "115 Open", "/storage/115", payloadB, 3) + if !ok || newer.Version <= rev.Version { + t.Fatalf("source rotation did not advance beyond revocation: %#v", newer) + } + if !relay.mergeCred(newer) { + t.Fatal("relay rejected newer credential rotation after revocation") + } +} + // canOfferCred requires a local health proof for this node's own candidates. // A signed, group-authorized peer candidate remains relayable: the hub need not // overwrite a healthy local mount merely to forward NAT peers' recovery path. diff --git a/internal/cluster/transport.go b/internal/cluster/transport.go index f8752414a2..3c6a695ef4 100644 --- a/internal/cluster/transport.go +++ b/internal/cluster/transport.go @@ -19,18 +19,20 @@ import ( // - Groups: the shared sync-group document (LWW). // - GroupsVer: the sender's groups version, for cheap anti-entropy. // - Creds: credential records being offered (push, or reply to a Want). +// - Revocations: source-signed tombstones for invalidated credential records. // - CredDigests: compact, no-secret advert of the credential records the sender // holds, so the receiver can detect what it is missing. // - Wants: group ids the sender wants the receiver to send creds for. type syncMessage struct { - Type string `json:"type"` // "hello" | "push" | "announce" | "pull" | "reply" - Node *nodeInfo `json:"node,omitempty"` - Nodes []*nodeInfo `json:"nodes,omitempty"` - Groups *groupDoc `json:"groups,omitempty"` - GroupsVer uint64 `json:"groups_ver,omitempty"` - Creds []*credRecord `json:"creds,omitempty"` - CredDigests []credDigest `json:"cred_digests,omitempty"` - Wants []string `json:"wants,omitempty"` + Type string `json:"type"` // "hello" | "push" | "announce" | "pull" | "reply" + Node *nodeInfo `json:"node,omitempty"` + Nodes []*nodeInfo `json:"nodes,omitempty"` + Groups *groupDoc `json:"groups,omitempty"` + GroupsVer uint64 `json:"groups_ver,omitempty"` + Creds []*credRecord `json:"creds,omitempty"` + Revocations []*credRevocation `json:"revocations,omitempty"` + CredDigests []credDigest `json:"cred_digests,omitempty"` + Wants []string `json:"wants,omitempty"` } // envelope is the outer, on-the-wire structure. Its Cipher is the syncMessage diff --git a/internal/op/hook.go b/internal/op/hook.go index 434c302380..1f675ff430 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -138,12 +138,18 @@ func NotifyStorageTokenHealthy(storage driver.Driver) { // propagating the broken token. Drivers may call this when an API call fails with // an unrecoverable auth error. func NotifyStorageTokenInvalid(storage driver.Driver) { + st := storage.GetStorage() + if st == nil || st.Disabled { + return + } storageTokenStatusMu.Lock() - changed := markStorageTokenInvalidStatus(storage) + _ = markStorageTokenInvalidStatus(storage) storageTokenStatusMu.Unlock() - if changed { - go callStorageHooks("token-invalid", storage) - } + // Persisting status is edge-triggered, but recovery is not. A newly adopted + // remote candidate can fail while this mount is already marked invalid; that + // failure must still reach cluster recovery, which performs its own per-mount + // cooldown and candidate de-duplication. + go callStorageHooks("token-invalid", storage) } func markStorageTokenInvalidStatus(storage driver.Driver) bool { diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go index 33ba289d4b..823ac80cb4 100644 --- a/internal/op/hook_test.go +++ b/internal/op/hook_test.go @@ -88,6 +88,32 @@ func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { } } +func TestNotifyStorageTokenInvalidDispatchesRecoveryWhenAlreadyInvalid(t *testing.T) { + d := &tokenValidStorageDriver{Storage: model.Storage{ + Driver: "TokenInvalidTest", + MountPath: "/token-invalid-repeat", + Status: "token invalid", + Addition: "{}", + }} + called := make(chan struct{}, 1) + op.RegisterStorageHook(func(typ string, got driver.Driver) { + if typ != "token-invalid" || got != d { + return + } + select { + case called <- struct{}{}: + default: + } + }) + + op.NotifyStorageTokenInvalid(d) + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("an already-invalid storage must still dispatch recovery") + } +} + func TestNotifyStorageTokenHealthyDoesNotChangeLifecycleStatus(t *testing.T) { d := &tokenValidStorageDriver{Storage: model.Storage{Status: op.WORK}} called := make(chan struct{}, 1) From b7bea80b28f8a22e3991c3176fc5e2309687cb7d Mon Sep 17 00:00:00 2001 From: ironbox Date: Sun, 19 Jul 2026 17:12:49 +0800 Subject: [PATCH 102/107] chore(deps): recover 115 credentials from peers on 401 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index df497926db..a5d0d5a5b3 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.12 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.13 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index f4bdb2de9b..279b63d23d 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqm github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/Ironboxplus/115-sdk-go v0.2.12 h1:6WB2L9FG4n+Xv5RvYXQxZBFvbhgYlWoqKYpGTR0coas= github.com/Ironboxplus/115-sdk-go v0.2.12/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.13 h1:B815N2mOq2lF2QDyANnkZW3mpmFM7xrBJrqFeENeYVs= +github.com/Ironboxplus/115-sdk-go v0.2.13/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= From 4998e79eee9578336998138c3bcbea86b0598e54 Mon Sep 17 00:00:00 2001 From: ironbox Date: Sun, 19 Jul 2026 21:36:42 +0800 Subject: [PATCH 103/107] fix(cluster): fence rejected credential pairs - preserve credential-pair provenance instead of reauthoring peer imports\n- revoke every relay copy on a provider 401 and block replays\n- probe candidates with the registered production driver before storage commit --- internal/cluster/engine.go | 139 +++++++++++++++++---------------- internal/cluster/state.go | 115 +++++++++++++++++++++++---- internal/cluster/state_test.go | 133 +++++++++++++++++++++++++++++-- 3 files changed, 296 insertions(+), 91 deletions(-) diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 429357e9fd..7b1e7244d5 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -3,6 +3,7 @@ package cluster import ( "context" "encoding/json" + stderrors "errors" "fmt" "os" "path/filepath" @@ -10,7 +11,9 @@ import ( "sync" "time" + sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -344,67 +347,26 @@ func (m *Manager) announceLoop() { // ---- credential seeding / hooks ---- -// seedLocalCreds records credentials for local healthy mounts that belong to a -// group, pulls for member-groups we have no credential for yet, and re-applies -// any held credential to local mounts (e.g. after a groups change). +// seedLocalCreds only asks peers for catalog candidates after startup or a group +// change. A persisted Storage.Status=WORK is not proof that its credential pair +// is still accepted by the provider, so startup must never mint or broadcast a +// candidate from it. func (m *Manager) seedLocalCreds() { cfg := m.cfgStore.get() if !cfg.active() { return } selfID := m.id.NodeID - var changed bool for _, d := range op.GetAllStorages() { st := d.GetStorage() groups := m.state.groupsForMount(selfID, st.MountPath) if len(groups) == 0 { continue } - if st.Status != op.WORK { - // Unhealthy mount (e.g. token dead at boot): drop our own stale cred so - // it can't dominate a peer's valid one. Persisted peer candidates are - // tried before a network pull so restart recovery also works offline. - for _, g := range groups { - if rev, ok := m.revokeOwnCandidate(g.ID, st.MountPath); ok { - changed = true - m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) - } - for _, candidate := range m.state.credsForGroup(g.ID) { - m.applyCredRecord(candidate) - } - if d.GetStorage().Status != op.WORK { - m.pullGroup(g.ID) - } - } - continue - } - creds := extractCreds(st.Addition) - credID := credHash(creds) for _, g := range groups { - if m.state.hasCredForSource(g.ID, selfID, st.MountPath, credID) { - m.state.markCandidateHealthy(g.ID, credID, now()) - continue // keep the original peer signature when the token is identical - } - if rec, ok := m.state.localCredChange(m.id, g.ID, st.Driver, st.MountPath, creds, now()); ok { - m.state.markCandidateHealthy(g.ID, rec.CredHash, now()) - changed = true - m.recordEvent("share", g.ID, fmt.Sprintf("%s shared %d credential field(s)", st.MountPath, len(rec.Fields))) - m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) - } - } - } - // member-groups we hold no credential for: ask peers. - for _, g := range m.state.groupList() { - if len(g.mountsForNode(selfID)) == 0 { - continue - } - if len(m.state.credsForGroup(g.ID)) == 0 { m.pullGroup(g.ID) } } - if changed { - m.persist() - } } // revokeOwnCandidate removes the local source candidate and returns a signed @@ -431,7 +393,7 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { return } switch typ { - case "add", "update", "token-valid": + case "token-valid": // "token-valid" is fired when an authenticated request just proved the // token good — (re)share it so peers converge on the working credential. if st.Status != op.WORK { @@ -460,6 +422,11 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { if dirty { m.persist() } + case "add", "update": + // Storage lifecycle updates are not provider-authentication evidence. + // In particular, UpdateStorage after staging a peer candidate must never + // turn that pair into a newly authored local credential record. + return case "del": // A mount was removed locally. We keep the group's credential record (other // members still rely on it); only our inventory changes (handled above). @@ -481,7 +448,7 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { m.state.forgetCandidateHealth(g.ID, credID) m.state.markCandidateFailed(g.ID, st.MountPath, credID, now()) changed = true - if rev, ok := m.revokeOwnCandidate(g.ID, st.MountPath); ok { + if rev, ok := m.state.revokePair(m.id, g.ID, st.MountPath, credID, now()); ok { m.recordEvent("invalidate", g.ID, fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) @@ -546,15 +513,21 @@ func (m *Manager) recoverKnownCandidates(groupID, mount, failedHash string) { if !m.credentialAuthorized(candidate) { continue } - attempted, ok := m.applyCredRecordToMount(candidate, mount) + attempted, ok, terminal := m.applyCredRecordToMount(candidate, mount) if !attempted { continue } if ok { return } - m.state.markCandidateFailed(groupID, mount, candidate.CredHash, now()) - changed = true + if terminal { + if rev, revoked := m.state.revokePair(m.id, groupID, mount, candidate.CredHash, now()); revoked { + m.recordEvent("invalidate", groupID, + fmt.Sprintf("%s rejected candidate %s with provider 401", mount, shortHash(candidate.CredHash))) + m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) + changed = true + } + } } if changed { m.persist() @@ -609,9 +582,6 @@ func (m *Manager) canOfferCred(r *credRecord) bool { if !m.credentialAuthorized(r) { return false } - if r.Origin != m.id.NodeID { - return true - } return m.hasHealthyLocalCandidate(r) } @@ -659,12 +629,12 @@ func (m *Manager) applyCredRecord(r *credRecord) { } // applyCredRecordToMount adopts one candidate on one local mount. It returns -// whether a re-initialization was attempted and whether it completed with a -// working storage; recovery uses that distinction to quarantine only candidates -// that actually failed on this node. -func (m *Manager) applyCredRecordToMount(r *credRecord, mp string) (attempted, success bool) { +// whether a probe was attempted, whether activation completed, and whether the +// probe received an exact provider-auth 401. Candidate payloads are verified by +// a temporary real driver before production storage is changed. +func (m *Manager) applyCredRecordToMount(r *credRecord, mp string) (attempted, success, terminal bool) { if r == nil || !m.cfgStore.get().ApplyRemote || m.state.candidateFailed(r.GroupID, mp, r.CredHash) { - return false, false + return false, false, false } ctx := context.Background() { @@ -673,33 +643,69 @@ func (m *Manager) applyCredRecordToMount(r *credRecord, mp string) (attempted, s defer lock.Unlock() d, err := op.GetStorageByMountPath(mp) if err != nil { - return false, false + return false, false, false } - st := *d.GetStorage() // copy; preserve ID/Status/local fields + before := *d.GetStorage() // preserve a local LKG for rollback + st := before if r.OriginDriver != "" && st.Driver != r.OriginDriver { utils.Log.Warnf("[cluster] skip applying %s creds to %s: driver mismatch (%s != %s)", r.GroupID, mp, st.Driver, r.OriginDriver) - return false, false + return false, false, false } if st.Status == op.WORK { // A peer's newer Lamport value is a recovery candidate, not authority // to overwrite credentials this mount has already proved locally. - return false, false + return false, false, false } newAdd, changed := applyCreds(st.Addition, r.Payload) if !changed { - return false, false // already has these credentials — no churn, no re-init + return false, false, false // already has these credentials — no churn, no re-init + } + if err := probeCredential(ctx, st, newAdd); err != nil { + utils.Log.Warnf("[cluster] probe candidate %s for %s failed: %v", shortHash(r.CredHash), mp, err) + return true, false, isProvider401(err) } st.Addition = newAdd if err := op.UpdateStorage(ctx, st); err != nil { utils.Log.Warnf("[cluster] apply creds to %s failed: %v", mp, err) - return true, false + // UpdateStorage persists before it initializes. Restore the complete + // previous storage record rather than leaving an unproven pair behind. + if rollbackErr := op.UpdateStorage(ctx, before); rollbackErr != nil { + utils.Log.Errorf("[cluster] rollback %s after failed candidate commit: %v", mp, rollbackErr) + } + return true, false, false } + m.state.markCandidateHealthy(r.GroupID, r.CredHash, now()) m.recordEvent("apply", r.GroupID, fmt.Sprintf("%s adopted credentials from %s", mp, shortNode(r.Origin))) utils.Log.Infof("[cluster] applied group %s credentials to %s (v%d from %s)", r.GroupID, mp, r.Version, r.Origin) - return true, true + return true, true, false + } + return false, false, false +} + +// probeCredential constructs the registered production driver in isolation. It +// runs that driver's actual Init path (115 Open performs UserInfo) without +// registering it in op's storage map or writing SQLite. A failed candidate thus +// never replaces the mount's active/LKG pair. +func probeCredential(ctx context.Context, storage model.Storage, addition string) error { + constructor, err := op.GetDriver(storage.Driver) + if err != nil { + return err + } + temporary := constructor() + storage.Addition = addition + temporary.SetStorage(storage) + if err := utils.Json.UnmarshalFromString(storage.Addition, temporary.GetAddition()); err != nil { + return err } - return false, false + probeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return temporary.Init(probeCtx) +} + +func isProvider401(err error) bool { + var authErr *sdk.Error + return stderrors.As(err, &authErr) && sdk.Is401Started(authErr.Code) } // ---- absorb (merge) helpers ---- @@ -739,7 +745,6 @@ func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { } if m.state.mergeCred(r) { merged = append(merged, r) - m.applyCredRecord(r) } } if len(merged) > 0 { diff --git a/internal/cluster/state.go b/internal/cluster/state.go index 1c7498d69f..becaa150ae 100644 --- a/internal/cluster/state.go +++ b/internal/cluster/state.go @@ -596,7 +596,24 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay byOrigin = make(map[string]*credRecord) s.creds[groupID] = byOrigin } - if cur, ok := byOrigin[candidateKey(id.NodeID, mount)]; ok && cur.CredHash == h && cur.signatureKind() == credentialSignatureMountBound { + if s.pairRevokedLocked(groupID, h) { + return nil, false + } + key := candidateKey(id.NodeID, mount) + // Pair identity is group-wide. A node that adopted an existing peer pair + // must keep that pair's original provenance; re-signing it under another + // origin would let one 401 tombstone leave duplicate live copies behind. + for existingKey, existing := range byOrigin { + if existingKey == key && existing.CredHash == h && existing.signatureKind() != credentialSignatureMountBound { + // Preserve the rolling-compatible upgrade path for a record that this + // exact source had already authored under a legacy signature. + continue + } + if existing.CredHash == h { + return nil, false + } + } + if cur, ok := byOrigin[key]; ok && cur.CredHash == h && cur.signatureKind() == credentialSignatureMountBound { return nil, false // this source mount has not changed } s.lamport++ @@ -614,7 +631,7 @@ func (s *store) localCredChange(id *identity, groupID, driver, mount string, pay } r.Sig = id.sign(r.signingBytes()) r.MountSig = id.sign(r.mountSigningBytes()) - byOrigin[candidateKey(id.NodeID, mount)] = r + byOrigin[key] = r return r, true } @@ -624,15 +641,22 @@ func (s *store) mergeCred(r *credRecord) bool { s.mu.Lock() defer s.mu.Unlock() s.observe(r.Version) + if s.pairRevokedLocked(r.GroupID, r.CredHash) { + return false // a provider-auth tombstone is terminal for this complete pair + } byOrigin := s.creds[r.GroupID] + for _, existing := range byOrigin { + if existing.CredHash == r.CredHash { + // A signature proves who observed the pair, not that this observer is + // entitled to mint a second authority record for it. + return false + } + } if byOrigin == nil { byOrigin = make(map[string]*credRecord) s.creds[r.GroupID] = byOrigin } key := candidateKey(r.Origin, r.OriginMount) - if rev := s.revocations[r.GroupID][key]; rev != nil && rev.Version >= r.Version { - return false // a source-signed tombstone blocks relay replay of this version - } cur, ok := byOrigin[key] if ok { if cur.CredHash == r.CredHash { @@ -703,8 +727,12 @@ func (s *store) revokeOwnCred(id *identity, groupID, mount string, at int64) (*c byRevocation = make(map[string]*credRevocation) s.revocations[groupID] = byRevocation } - byRevocation[key] = rev - delete(byOrigin, key) + byRevocation[rev.CredHash] = rev + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == rev.CredHash { + delete(byOrigin, sourceKey) + } + } if len(byOrigin) == 0 { delete(s.creds, groupID) } @@ -712,6 +740,58 @@ func (s *store) revokeOwnCred(id *identity, groupID, mount string, at int64) (*c return &cp, true } +// revokePair records a provider-auth rejection for a complete credential pair. +// Unlike revokeOwnCred, the reporter need not be the pair's original publisher: +// a 401 is about the pair itself, and must suppress every re-authored or relayed +// copy in the group. +func (s *store) revokePair(id *identity, groupID, mount, hash string, at int64) (*credRevocation, bool) { + if id == nil || groupID == "" || mount == "" || hash == "" || at <= 0 { + return nil, false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.pairRevokedLocked(groupID, hash) { + return nil, false + } + s.lamport++ + rev := &credRevocation{ + GroupID: groupID, + Origin: id.NodeID, + OriginPub: id.Pub, + OriginMount: mount, + CredHash: hash, + Version: s.lamport, + UpdatedAt: at, + } + rev.Sig = id.sign(rev.signingBytes()) + byRevocation := s.revocations[groupID] + if byRevocation == nil { + byRevocation = make(map[string]*credRevocation) + s.revocations[groupID] = byRevocation + } + byRevocation[hash] = rev + if byOrigin := s.creds[groupID]; byOrigin != nil { + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == hash { + delete(byOrigin, sourceKey) + } + } + if len(byOrigin) == 0 { + delete(s.creds, groupID) + } + } + cp := *rev + return &cp, true +} + +func (s *store) pairRevokedLocked(groupID, hash string) bool { + if hash == "" { + return false + } + _, ok := s.revocations[groupID][hash] + return ok +} + func (s *store) revocationSnapshot() []*credRevocation { s.mu.RLock() defer s.mu.RUnlock() @@ -737,9 +817,9 @@ func (s *store) revocationSnapshot() []*credRevocation { return out } -// mergeRevocation applies a verified tombstone and removes any same-source -// candidate that it supersedes. A later credential from the source remains -// allowed because it must carry a strictly greater Lamport version. +// mergeRevocation applies a verified provider-auth tombstone. A tombstone is +// keyed by complete pair hash, not by its reporting source: the same pair may +// have been observed by several nodes, but one 401 makes every copy unusable. func (s *store) mergeRevocation(rev *credRevocation) bool { if rev == nil { return false @@ -747,24 +827,25 @@ func (s *store) mergeRevocation(rev *credRevocation) bool { s.mu.Lock() defer s.mu.Unlock() s.observe(rev.Version) - key := candidateKey(rev.Origin, rev.OriginMount) byRevocation := s.revocations[rev.GroupID] if byRevocation == nil { byRevocation = make(map[string]*credRevocation) s.revocations[rev.GroupID] = byRevocation } - if current := byRevocation[key]; current != nil && !rev.dominates(current) { + if current := byRevocation[rev.CredHash]; current != nil && !rev.dominates(current) { return false } cp := *rev - byRevocation[key] = &cp + byRevocation[rev.CredHash] = &cp if byOrigin := s.creds[rev.GroupID]; byOrigin != nil { - if current := byOrigin[key]; current != nil && current.Version <= rev.Version { - delete(byOrigin, key) - if len(byOrigin) == 0 { - delete(s.creds, rev.GroupID) + for sourceKey, candidate := range byOrigin { + if candidate.CredHash == rev.CredHash { + delete(byOrigin, sourceKey) } } + if len(byOrigin) == 0 { + delete(s.creds, rev.GroupID) + } } return true } diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index a7041682a3..3f62e5be3c 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -3,6 +3,10 @@ package cluster import ( "encoding/json" "testing" + + open115 "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" ) // ---- credential extraction ---- @@ -60,6 +64,122 @@ func TestExtractAndApplyCreds(t *testing.T) { } } +// A complete credential pair has one group-wide identity. Seeing the same pair +// on another member must not turn it into a newer, independently revocable +// candidate, and a provider-auth rejection must suppress every replay of it. +func TestPairIdentityPreventsReauthoringAndBlocksReplayAfterRejection(t *testing.T) { + a, _ := newIdentity() + b, _ := newIdentity() + s := newStore() + pair := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"access-A"`), + "refresh_token": json.RawMessage(`"refresh-A"`), + } + + first, ok := s.localCredChange(a, "g1", "115 Open", "/storage/115", pair, 1) + if !ok { + t.Fatal("first locally proven pair should be recorded") + } + if _, ok := s.localCredChange(b, "g1", "115 Open", "/storage/115", pair, 2); ok { + t.Fatal("an imported pair must retain its original provenance, not be re-authored by another node") + } + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("same pair must have one catalog entry, got %d", got) + } + + rev, ok := s.revokePair(a, "g1", "/storage/115", first.CredHash, 3) + if !ok { + t.Fatal("provider-auth rejection must create a pair-level tombstone") + } + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("pair tombstone must remove every copy, got %d", got) + } + if s.mergeCred(first) { + t.Fatal("a rejected pair must not be replayed by its original signer") + } + if _, ok := s.localCredChange(b, "g1", "115 Open", "/storage/115", pair, 4); ok { + t.Fatal("a rejected pair must not re-enter under another node identity") + } + if !s.mergeRevocation(rev) { + // A locally created tombstone is already present; repeated delivery must + // be a harmless no-op rather than changing catalog state. + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("replayed tombstone changed catalog to %d entries", got) + } + } +} + +// A legacy peer may already have re-authored a pair before v2 reaches every +// node. Receiving that second signature must not recreate duplicate authority. +func TestMergeCredRejectsPeerReauthoringOfExistingPair(t *testing.T) { + a, _ := newIdentity() + b, _ := newIdentity() + s := newStore() + pair := map[string]json.RawMessage{ + "access_token": json.RawMessage(`"access-A"`), + "refresh_token": json.RawMessage(`"refresh-A"`), + } + first, ok := s.localCredChange(a, "g1", "115 Open", "/storage/115", pair, 1) + if !ok { + t.Fatal("first pair should be recorded") + } + reauthored := &credRecord{ + GroupID: first.GroupID, + OriginDriver: first.OriginDriver, + Fields: first.Fields, + CredHash: first.CredHash, + Payload: first.Payload, + Version: first.Version + 100, + Origin: b.NodeID, + OriginPub: b.Pub, + OriginMount: "/storage/115", + UpdatedAt: 2, + } + reauthored.Sig = b.sign(reauthored.signingBytes()) + reauthored.MountSig = b.sign(reauthored.mountSigningBytes()) + if s.mergeCred(reauthored) { + t.Fatal("same pair from another origin must not be accepted as a newer candidate") + } + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("re-authored pair created %d catalog records, want 1", got) + } +} + +// Storage updates are not credential-validation events. In particular, +// UpdateStorage after receiving an imported pair must not re-sign and publish it +// from the receiving node. This uses the real 115 driver type and real manager +// state; no test driver or callback mock is involved. +func TestOnlyTokenValidMayPublishStoragePair(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.setGroups(id, []group{{ + ID: "g1", + Members: []member{{NodeID: id.NodeID, MountPath: "/storage/115"}}, + }}, 1) + m := &Manager{ + id: id, + state: s, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + d := &open115.Open115{Storage: model.Storage{ + MountPath: "/storage/115", + Driver: "115 Open", + Status: op.WORK, + Addition: `{"access_token":"access-A","refresh_token":"refresh-A"}`, + }} + + m.onStorageHook("update", d) + if got := len(s.credsForGroup("g1")); got != 0 { + t.Fatalf("ordinary update published %d credential record(s), want 0", got) + } + + m.onStorageHook("token-valid", d) + if got := len(s.credsForGroup("g1")); got != 1 { + t.Fatalf("token-valid published %d credential record(s), want 1", got) + } +} + func TestCredHashIdempotent(t *testing.T) { a := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`), "access_token": json.RawMessage(`"y"`)} b := map[string]json.RawMessage{"access_token": json.RawMessage(`"y"`), "refresh_token": json.RawMessage(`"x"`)} @@ -510,9 +630,9 @@ func TestSignedRevocationRemovesRelayCopyButAllowsNewerRotation(t *testing.T) { } } -// canOfferCred requires a local health proof for this node's own candidates. -// A signed, group-authorized peer candidate remains relayable: the hub need not -// overwrite a healthy local mount merely to forward NAT peers' recovery path. +// canOfferCred requires a local health proof for every candidate. A signature +// proves provenance only; an unvalidated relay must not advertise a pair as a +// recoverable credential. func TestCanOfferCred(t *testing.T) { id1, _ := newIdentity() // self id2, _ := newIdentity() // peer @@ -528,8 +648,7 @@ func TestCanOfferCred(t *testing.T) { t.Fatal("a nil record is never offerable") } - // The hub may relay a peer-authored candidate without first applying it to a - // working local mount; its signature and group membership authorize that. + // A peer-authored candidate with no local proof is catalog data only. payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"peer"`)} peer := &credRecord{ GroupID: "g1", @@ -542,8 +661,8 @@ func TestCanOfferCred(t *testing.T) { OriginMount: "/peer", } peer.Sig = id2.sign(peer.signingBytes()) - if !m.canOfferCred(peer) { - t.Fatal("an authorized peer candidate must remain relayable") + if m.canOfferCred(peer) { + t.Fatal("an unvalidated peer candidate must not be advertised for recovery") } // our own record, but no group/healthy mount -> not offerable. From 755d713827b909c5838e687e86b5bb9b7990dfe2 Mon Sep 17 00:00:00 2001 From: ironbox Date: Sun, 19 Jul 2026 21:45:22 +0800 Subject: [PATCH 104/107] fix(cluster): isolate credential probes - keep temporary driver callbacks off persisted storage rows\n- retain candidate payload while suppressing probe lifecycle writes --- internal/cluster/engine.go | 17 ++++++++++++++--- internal/cluster/state_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index 7b1e7244d5..b082c94451 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -693,9 +693,9 @@ func probeCredential(ctx context.Context, storage model.Storage, addition string return err } temporary := constructor() - storage.Addition = addition - temporary.SetStorage(storage) - if err := utils.Json.UnmarshalFromString(storage.Addition, temporary.GetAddition()); err != nil { + probe := newProbeStorage(storage, addition) + temporary.SetStorage(probe) + if err := utils.Json.UnmarshalFromString(probe.Addition, temporary.GetAddition()); err != nil { return err } probeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) @@ -703,6 +703,17 @@ func probeCredential(ctx context.Context, storage model.Storage, addition string return temporary.Init(probeCtx) } +// newProbeStorage gives a temporary driver the candidate fields but no ownership +// of the persisted storage row. Some production drivers report an authenticated +// Init through lifecycle hooks; ID=0 prevents such a callback from changing the +// live row, and WORK prevents a successful probe from emitting token-valid. +func newProbeStorage(live model.Storage, addition string) model.Storage { + live.ID = 0 + live.Status = op.WORK + live.Addition = addition + return live +} + func isProvider401(err error) bool { var authErr *sdk.Error return stderrors.As(err, &authErr) && sdk.Is401Started(authErr.Code) diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 3f62e5be3c..3ce4a50d87 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -180,6 +180,34 @@ func TestOnlyTokenValidMayPublishStoragePair(t *testing.T) { } } +// Candidate verification must be hermetic with respect to the live storage +// record. A temporary driver's successful Init can emit lifecycle callbacks, so +// its storage identity must never point at the persisted mount being recovered. +func TestProbeStorageIsEphemeralAndKeepsCandidateAddition(t *testing.T) { + live := model.Storage{ + ID: 42, + MountPath: "/storage/115", + Driver: "115 Open", + Status: "token invalid", + Addition: `{"access_token":"old","refresh_token":"old"}`, + } + candidate := `{"access_token":"new","refresh_token":"new"}` + + probe := newProbeStorage(live, candidate) + if probe.ID != 0 { + t.Fatalf("probe storage ID = %d, want 0 so callbacks cannot write the live row", probe.ID) + } + if probe.Status != op.WORK { + t.Fatalf("probe storage status = %q, want %q to suppress token-valid lifecycle writes", probe.Status, op.WORK) + } + if probe.Addition != candidate { + t.Fatalf("probe addition = %q, want candidate payload", probe.Addition) + } + if probe.MountPath != live.MountPath || probe.Driver != live.Driver { + t.Fatalf("probe changed mount identity: %#v", probe) + } +} + func TestCredHashIdempotent(t *testing.T) { a := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"x"`), "access_token": json.RawMessage(`"y"`)} b := map[string]json.RawMessage{"access_token": json.RawMessage(`"y"`), "refresh_token": json.RawMessage(`"x"`)} From 26b507e02245a8b367f39fbb738c7c8e5829b4f2 Mon Sep 17 00:00:00 2001 From: ironbox Date: Sun, 19 Jul 2026 23:43:55 +0800 Subject: [PATCH 105/107] fix(cluster): converge peer credential recovery --- drivers/115_open/driver.go | 11 +- internal/cluster/engine.go | 344 +++++++++++++++++++++------------ internal/cluster/state_test.go | 148 +++++++++++++- internal/db/storage.go | 15 ++ internal/op/hook.go | 160 +++++++++++++-- internal/op/hook_test.go | 68 +++++++ 6 files changed, 600 insertions(+), 146 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 4d40a39381..5db3833dff 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -57,6 +57,11 @@ func (d *Open115) GetAddition() driver.Additional { } func (d *Open115) Init(ctx context.Context) error { + // This client generation can finish an old request after cluster recovery has + // replaced the storage. Bind auth callbacks to the Addition that configured + // this client so an old 401 cannot revoke the newly installed pair. + observedAddition := d.Storage.Addition + observedModified := d.Storage.Modified d.client = new115SDKClient(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), sdk.WithOnRefreshToken(func(s1, s2 string) { @@ -71,14 +76,14 @@ func (d *Open115) Init(ctx context.Context) error { // A successful authenticated request renews the cluster's in-memory // proof only. This avoids a periodic refresh/push while preventing a // working credential from ageing out of recovery advertisement. - op.NotifyStorageTokenHealthy(d) + op.NotifyStorageTokenHealthyWithSnapshot(d, observedAddition, observedModified) if d.shouldNotifyTokenValid() { - op.NotifyStorageTokenValid(d) + op.NotifyStorageTokenValidWithSnapshot(d, observedAddition, observedModified) } }), sdk.WithOnTokenInvalid(func() { if d.tokenInvalid.CompareAndSwap(false, true) { - op.NotifyStorageTokenInvalid(d) + op.NotifyStorageTokenInvalidWithSnapshot(d, observedAddition, observedModified) } })) applySDKProxyIfConfigured(d.client) diff --git a/internal/cluster/engine.go b/internal/cluster/engine.go index b082c94451..5d289217d7 100644 --- a/internal/cluster/engine.go +++ b/internal/cluster/engine.go @@ -30,10 +30,6 @@ const ( peerLivenessSec = 130 // maxEvents bounds the in-memory activity log surfaced to the UI. maxEvents = 60 - // candidateRecoveryCooldown prevents an invalid mount from cycling through - // every persisted credential on each caller retry. A new credential push is - // still applied immediately; this only bounds local fallback attempts. - candidateRecoveryCooldown = 5 * time.Minute ) // Manager is the running cluster credential-sync engine for this node. @@ -47,9 +43,9 @@ type Manager struct { persistMu sync.Mutex - recoveryMu sync.Mutex - recoveryLocks map[string]*sync.Mutex // per local mount: serialize credential adoption - recoveryLast map[string]time.Time + recoveryMu sync.Mutex + recoveryLocks map[string]*sync.Mutex // per local mount: serialize credential adoption + recoveryQueues map[string]*recoveryQueue dialMu sync.Mutex dialing map[string]bool // peer URLs with an in-flight/live outbound dial @@ -64,6 +60,15 @@ type Manager struct { once sync.Once } +// recoveryQueue is one mount's event-driven recovery mailbox. It deliberately +// contains no timer: a pair is enqueued by an observed 401, a newly received +// signed candidate, or startup reconstruction. The source of truth remains the +// durable candidate/tombstone state, so a crash can safely rebuild this queue. +type recoveryQueue struct { + running bool + pending map[string]*credRecord // credential hash -> immutable signed candidate +} + // Default is the process-wide manager, set by Init. It is nil when cluster sync // was never initialized. var Default *Manager @@ -80,16 +85,16 @@ func Init(dataDir string) (*Manager, error) { return nil, err } m := &Manager{ - dir: dir, - id: id, - cfgStore: newConfigStore(dir), - state: newStore(), - replay: newReplayCache(replayWindowSec), - conns: newConnRegistry(), - dialing: make(map[string]bool), - recoveryLocks: make(map[string]*sync.Mutex), - recoveryLast: make(map[string]time.Time), - stopCh: make(chan struct{}), + dir: dir, + id: id, + cfgStore: newConfigStore(dir), + state: newStore(), + replay: newReplayCache(replayWindowSec), + conns: newConnRegistry(), + dialing: make(map[string]bool), + recoveryLocks: make(map[string]*sync.Mutex), + recoveryQueues: make(map[string]*recoveryQueue), + stopCh: make(chan struct{}), } if _, err := m.cfgStore.loadOrInit(); err != nil { return nil, err @@ -97,7 +102,8 @@ func Init(dataDir string) (*Manager, error) { m.loadState() op.RegisterStorageHook(m.onStorageHook) - op.RegisterStorageHealthHook(m.onStorageHealthy) + op.RegisterStorageCredentialHook(m.onStorageCredential) + op.RegisterStorageCredentialHealthHook(m.onStorageCredentialHealthy) Default = m return m, nil } @@ -128,23 +134,26 @@ func (m *Manager) loadState() { } } -func (m *Manager) persist() { +func (m *Manager) persist() error { m.persistMu.Lock() defer m.persistMu.Unlock() ps := m.state.export() b, err := json.MarshalIndent(ps, "", " ") if err != nil { - return + return err } if err := os.MkdirAll(m.dir, 0o700); err != nil { - return + return err } tmp := m.statePath() + ".tmp" if err := os.WriteFile(tmp, b, 0o600); err != nil { utils.Log.Warnf("[cluster] failed to persist state: %v", err) - return + return err + } + if err := os.Rename(tmp, m.statePath()); err != nil { + return err } - _ = os.Rename(tmp, m.statePath()) + return nil } // ---- config access ---- @@ -208,7 +217,9 @@ func (m *Manager) SetGroups(specs []GroupSpec) error { } d := m.state.setGroups(m.id, groups, now()) m.state.pruneCreds() - m.persist() + if err := m.persist(); err != nil { + return fmt.Errorf("persist groups: %w", err) + } m.recordEvent("groups", "", fmt.Sprintf("updated to %d group(s)", len(groups))) gd := d m.broadcast(&syncMessage{Type: "push", Groups: &gd}) @@ -347,10 +358,11 @@ func (m *Manager) announceLoop() { // ---- credential seeding / hooks ---- -// seedLocalCreds only asks peers for catalog candidates after startup or a group +// seedLocalCreds reconciles every locally failed mount after startup or a group // change. A persisted Storage.Status=WORK is not proof that its credential pair -// is still accepted by the provider, so startup must never mint or broadcast a -// candidate from it. +// is still accepted by the provider, so startup never mints a fresh candidate +// from it; however, a persisted non-WORK mount must retry durable peer +// candidates even when they arrived before this process started. func (m *Manager) seedLocalCreds() { cfg := m.cfgStore.get() if !cfg.active() { @@ -364,6 +376,9 @@ func (m *Manager) seedLocalCreds() { continue } for _, g := range groups { + if st.Status != op.WORK { + m.enqueueKnownCandidates(g.ID, st.MountPath, credHash(extractCreds(st.Addition))) + } m.pullGroup(g.ID) } } @@ -377,38 +392,51 @@ func (m *Manager) revokeOwnCandidate(groupID, mount string) (*credRevocation, bo return m.state.revokeOwnCred(m.id, groupID, mount, now()) } -// onStorageHook reacts to local storage lifecycle/credential changes. +// onStorageHook only tracks lifecycle/inventory changes. Authentication events +// are intentionally handled by onStorageCredential because they need the +// immutable Addition snapshot of the request that produced them. func (m *Manager) onStorageHook(typ string, d driver.Driver) { - cfg := m.cfgStore.get() - if !cfg.active() { + if !m.cfgStore.get().active() || d == nil || d.GetStorage() == nil { + return + } + if typ == "token-valid" || typ == "token-invalid" { return } - st := d.GetStorage() - // Any storage change may alter our inventory (added/removed mount, status). go m.refreshInventory() +} - selfID := m.id.NodeID - groups := m.state.groupsForMount(selfID, st.MountPath) +// onStorageCredential consumes a real provider result bound to the exact +// Addition used on the wire. It is the sole authority for publishing a pair or +// writing its group-wide 401 tombstone; ordinary storage update hooks are never +// authentication evidence. +func (m *Manager) onStorageCredential(typ string, event op.StorageCredentialEvent) { + if !m.cfgStore.get().active() || event.Storage == nil { + return + } + st := event.Storage.GetStorage() + if st == nil || st.Addition != event.Addition || !st.Modified.Equal(event.Modified) { + return // late result from an old client generation + } + go m.refreshInventory() + groups := m.state.groupsForMount(m.id.NodeID, st.MountPath) if len(groups) == 0 { return } + switch typ { case "token-valid": - // "token-valid" is fired when an authenticated request just proved the - // token good — (re)share it so peers converge on the working credential. if st.Status != op.WORK { - // Health gating: never propagate a broken/expired token. Try to recover - // a good one from peers instead. for _, g := range groups { m.pullGroup(g.ID) } return } - creds := extractCreds(st.Addition) + creds := extractCreds(event.Addition) credID := credHash(creds) var dirty bool + var shares []*credRecord for _, g := range groups { - if m.state.hasCredForSource(g.ID, selfID, st.MountPath, credID) { + if m.state.hasCredForSource(g.ID, m.id.NodeID, st.MountPath, credID) { m.state.markCandidateHealthy(g.ID, credID, now()) continue } @@ -416,67 +444,68 @@ func (m *Manager) onStorageHook(typ string, d driver.Driver) { m.state.markCandidateHealthy(g.ID, rec.CredHash, now()) dirty = true m.recordEvent("share", g.ID, fmt.Sprintf("%s refreshed credentials", st.MountPath)) - m.broadcast(&syncMessage{Type: "push", Creds: []*credRecord{rec}}) + shares = append(shares, rec) } } if dirty { - m.persist() + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not sharing unpersisted credential for %s: %v", st.MountPath, err) + return + } } - case "add", "update": - // Storage lifecycle updates are not provider-authentication evidence. - // In particular, UpdateStorage after staging a peer candidate must never - // turn that pair into a newly authored local credential record. - return - case "del": - // A mount was removed locally. We keep the group's credential record (other - // members still rely on it); only our inventory changes (handled above). + if len(shares) > 0 { + m.broadcast(&syncMessage{Type: "push", Creds: shares}) + } + case "token-invalid": - if st.Status == op.WORK { - // Hooks are asynchronous. A later successful request may already have - // restored this storage, so an older invalid event must not revoke the - // healthy candidate it would otherwise overwrite. + // This event is generation-bound. Any 401xxxxx invalidates the precise + // access/refresh pair used by the request even if another asynchronous + // success would otherwise leave Storage.Status as WORK. + credID := credHash(extractCreds(event.Addition)) + if credID == "" { return } - // Our token died. Drop our own (now-stale) credential record so its Lamport - // version can't out-rank a peer's valid one, then pull a fresh credential - // from a healthy peer. The currently-used credential is quarantined for - // this mount even when it originated on a peer: that prevents a restart - // from immediately retrying the exact 40140125/26 candidate. - var changed bool - credID := credHash(extractCreds(st.Addition)) + var revocations []*credRevocation for _, g := range groups { m.state.forgetCandidateHealth(g.ID, credID) m.state.markCandidateFailed(g.ID, st.MountPath, credID, now()) - changed = true if rev, ok := m.state.revokePair(m.id, g.ID, st.MountPath, credID, now()); ok { m.recordEvent("invalidate", g.ID, fmt.Sprintf("%s token invalid — dropped local cred, pulling from peers", st.MountPath)) - m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) + revocations = append(revocations, rev) } else { m.recordEvent("invalidate", g.ID, fmt.Sprintf("%s token invalid — quarantined failed peer candidate", st.MountPath)) } - go m.recoverKnownCandidates(g.ID, st.MountPath, credID) } - if changed { - m.persist() + // The pair-wide 401 fact must survive a crash before a recovery worker + // can touch any alternative. This makes Revoked(P) durable-before-apply. + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not recovering after unpersisted 401 for %s: %v", st.MountPath, err) + return + } + if len(revocations) > 0 { + m.broadcast(&syncMessage{Type: "push", Revocations: revocations}) + } + for _, g := range groups { + m.enqueueKnownCandidates(g.ID, st.MountPath, credID) + m.pullGroup(g.ID) } } } -// onStorageHealthy renews only the in-memory proof for an already working -// credential. It deliberately does not write the database, broadcast a record, -// or emit a storage lifecycle event; ordinary successful requests therefore keep -// a candidate usable without causing token-refresh or cluster-sync churn. -func (m *Manager) onStorageHealthy(d driver.Driver) { - if !m.cfgStore.get().active() { +// onStorageCredentialHealthy renews in-memory proof only for the exact pair +// that received a successful provider response. It cannot accidentally mark a +// newer pair healthy after an old request returns late. +func (m *Manager) onStorageCredentialHealthy(event op.StorageCredentialEvent) { + if !m.cfgStore.get().active() || event.Storage == nil { return } - st := d.GetStorage() - if st == nil || st.Status != op.WORK { + st := event.Storage.GetStorage() + if st == nil || st.Status != op.WORK || st.Addition != event.Addition || !st.Modified.Equal(event.Modified) { return } - credID := credHash(extractCreds(st.Addition)) + credID := credHash(extractCreds(event.Addition)) if credID == "" { return } @@ -488,51 +517,104 @@ func (m *Manager) onStorageHealthy(d driver.Driver) { } } -func (m *Manager) beginCandidateRecovery(groupID, mount string) bool { - key := groupID + "\x00" + mount - m.recoveryMu.Lock() - defer m.recoveryMu.Unlock() - n := time.Now() - if last := m.recoveryLast[key]; !last.IsZero() && n.Sub(last) < candidateRecoveryCooldown { - return false +// enqueueKnownCandidates rebuilds a failed mount's durable recovery work. It is +// used after a local 401 and after restart; it never creates candidates or +// refreshes a token. +func (m *Manager) enqueueKnownCandidates(groupID, mount, failedHash string) { + for _, candidate := range m.state.recoveryCandidates(groupID, mount, failedHash) { + m.enqueueCandidate(candidate, mount) } - m.recoveryLast[key] = n - return true } -// recoverKnownCandidates tries already-held alternatives once, in deterministic -// order, after a token dies. It never refreshes on a timer: each candidate is -// attempted at most once per mount per cooldown window, then a pull waits for a -// peer to publish a genuinely newer credential. -func (m *Manager) recoverKnownCandidates(groupID, mount, failedHash string) { - if !m.beginCandidateRecovery(groupID, mount) { +// enqueueCandidate queues one newly learned, signed candidate for one local +// mount. A queue is keyed by credential hash, so duplicate frames and relay +// paths cannot create probe storms. The worker is per (group, mount), making +// activation deterministic and serial with storage replacement. +func (m *Manager) enqueueCandidate(candidate *credRecord, mount string) { + if candidate == nil || mount == "" || !m.credentialAuthorized(candidate) { return } - changed := false - for _, candidate := range m.state.recoveryCandidates(groupID, mount, failedHash) { - if !m.credentialAuthorized(candidate) { - continue + key := candidate.GroupID + "\x00" + mount + m.recoveryMu.Lock() + if m.recoveryQueues == nil { + m.recoveryQueues = make(map[string]*recoveryQueue) + } + queue := m.recoveryQueues[key] + if queue == nil { + queue = &recoveryQueue{pending: make(map[string]*credRecord)} + m.recoveryQueues[key] = queue + } + if _, exists := queue.pending[candidate.CredHash]; !exists { + cp := *candidate + queue.pending[candidate.CredHash] = &cp + } + if queue.running { + m.recoveryMu.Unlock() + return + } + queue.running = true + m.recoveryMu.Unlock() + go m.runCandidateRecovery(key, candidate.GroupID, mount) +} + +func (m *Manager) takeCandidate(key string) *credRecord { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + queue := m.recoveryQueues[key] + if queue == nil { + return nil + } + if len(queue.pending) == 0 { + queue.running = false + delete(m.recoveryQueues, key) + return nil + } + hashes := make([]string, 0, len(queue.pending)) + for hash := range queue.pending { + hashes = append(hashes, hash) + } + sort.Strings(hashes) + candidate := queue.pending[hashes[0]] + delete(queue.pending, hashes[0]) + return candidate +} + +func (m *Manager) finishCandidateRecovery(key string) { + m.recoveryMu.Lock() + defer m.recoveryMu.Unlock() + delete(m.recoveryQueues, key) +} + +// runCandidateRecovery is the only place a received candidate may become the +// mount's active storage. A successful probe stops the queue; a provider 401 +// emits a durable group-wide tombstone before the next candidate is attempted. +func (m *Manager) runCandidateRecovery(key, groupID, mount string) { + for { + candidate := m.takeCandidate(key) + if candidate == nil { + return } - attempted, ok, terminal := m.applyCredRecordToMount(candidate, mount) - if !attempted { + if !m.credentialAuthorized(candidate) { continue } - if ok { + attempted, success, terminal := m.applyCredRecordToMount(candidate, mount) + if success { + m.finishCandidateRecovery(key) return } - if terminal { - if rev, revoked := m.state.revokePair(m.id, groupID, mount, candidate.CredHash, now()); revoked { - m.recordEvent("invalidate", groupID, - fmt.Sprintf("%s rejected candidate %s with provider 401", mount, shortHash(candidate.CredHash))) - m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) - changed = true + if !attempted || !terminal { + continue + } + if rev, revoked := m.state.revokePair(m.id, groupID, mount, candidate.CredHash, now()); revoked { + m.recordEvent("invalidate", groupID, + fmt.Sprintf("%s rejected candidate %s with provider 401", mount, shortHash(candidate.CredHash))) + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted candidate rejection for %s: %v", mount, err) + return } + m.broadcast(&syncMessage{Type: "push", Revocations: []*credRevocation{rev}}) } } - if changed { - m.persist() - } - m.pullGroup(groupID) } // pullGroup asks peers for the latest credential of a group. @@ -573,19 +655,20 @@ func (m *Manager) hasHealthyLocalCandidate(r *credRecord) bool { return false } -// canOfferCred reports whether this node may advertise a credential candidate. -// A remote candidate is authenticated by its signature and the signed group -// document, and must remain relayable through an accept-only/NAT hub. This -// node's own candidate additionally needs a recent local health proof so stale -// credentials from its state.json are never reintroduced after a restart. +// canOfferCred reports whether this node may relay a credential candidate. A +// valid member signature is the authority for transport: a relay does not need +// to prove the pair locally before forwarding it. Local proof only controls +// whether this node activates a pair on one of its mounts. Keeping those two +// facts separate is essential for NAT/accept-only topologies, where a healthy +// source and an invalid target may never have a direct connection. func (m *Manager) canOfferCred(r *credRecord) bool { - if !m.credentialAuthorized(r) { - return false - } - return m.hasHealthyLocalCandidate(r) + return m.credentialAuthorized(r) } -// offerableDigests is credDigests() filtered to records this node may advertise. +// offerableDigests is the signed, non-revoked candidate inventory that may be +// relayed during anti-entropy. It is deliberately independent of local mount +// health; otherwise a hub that did not itself mount 115 would black-hole a +// healthy source's credential record. func (m *Manager) offerableDigests() []credDigest { var out []credDigest for _, r := range m.state.credSnapshot() { @@ -740,7 +823,10 @@ func (m *Manager) absorbGroups(d *groupDoc) bool { } if m.state.mergeGroups(d) { m.state.pruneCreds() - m.persist() + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted groups: %v", err) + return false + } m.recordEvent("groups", "", fmt.Sprintf("received %d group(s) from %s", len(d.Groups), shortNode(d.Origin))) go m.seedLocalCreds() return true @@ -759,7 +845,22 @@ func (m *Manager) absorbCreds(recs []*credRecord) []*credRecord { } } if len(merged) > 0 { - m.persist() + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not activating unpersisted received credentials: %v", err) + return nil + } + for _, r := range merged { + // A record newly arriving after a local 401 must wake recovery even if + // the earlier empty-candidate scan already completed. The queue itself + // deduplicates the pair and serializes the real driver probe. + g, ok := m.state.groupByID(r.GroupID) + if !ok { + continue + } + for _, mount := range g.mountsForNode(m.id.NodeID) { + m.enqueueCandidate(r, mount) + } + } } return merged } @@ -777,7 +878,10 @@ func (m *Manager) absorbRevocations(revs []*credRevocation) []*credRevocation { } } if len(merged) > 0 { - m.persist() + if err := m.persist(); err != nil { + utils.Log.Errorf("[cluster] not relaying unpersisted revocations: %v", err) + return nil + } } return merged } diff --git a/internal/cluster/state_test.go b/internal/cluster/state_test.go index 3ce4a50d87..d475fa1871 100644 --- a/internal/cluster/state_test.go +++ b/internal/cluster/state_test.go @@ -1,12 +1,20 @@ package cluster import ( + "context" "encoding/json" + "fmt" "testing" + "time" open115 "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/glebarez/sqlite" + "gorm.io/gorm" ) // ---- credential extraction ---- @@ -157,6 +165,7 @@ func TestOnlyTokenValidMayPublishStoragePair(t *testing.T) { Members: []member{{NodeID: id.NodeID, MountPath: "/storage/115"}}, }}, 1) m := &Manager{ + dir: t.TempDir(), id: id, state: s, conns: newConnRegistry(), @@ -174,12 +183,47 @@ func TestOnlyTokenValidMayPublishStoragePair(t *testing.T) { t.Fatalf("ordinary update published %d credential record(s), want 0", got) } - m.onStorageHook("token-valid", d) + m.onStorageCredential("token-valid", op.StorageCredentialEvent{Storage: d, Addition: d.GetStorage().Addition}) if got := len(s.credsForGroup("g1")); got != 1 { t.Fatalf("token-valid published %d credential record(s), want 1", got) } } +// An authentication result belongs to the pair sent with that request, not to +// whatever happens to be mounted when its goroutine finally runs. This uses the +// real 115 driver type and exercises the manager's generation guard without +// fabricating a driver implementation. +func TestLateCredentialEventCannotRevokeReplacementPair(t *testing.T) { + id, _ := newIdentity() + s := newStore() + s.setGroups(id, []group{{ + ID: "g1", + Members: []member{{NodeID: id.NodeID, MountPath: "/storage/115"}}, + }}, 1) + newPair := `{"access_token":"new-access","refresh_token":"new-refresh"}` + d := &open115.Open115{Storage: model.Storage{ + MountPath: "/storage/115", + Driver: "115 Open", + Status: op.WORK, + Addition: newPair, + }} + m := &Manager{ + dir: t.TempDir(), + id: id, + state: s, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + oldPair := `{"access_token":"old-access","refresh_token":"old-refresh"}` + m.onStorageCredential("token-invalid", op.StorageCredentialEvent{Storage: d, Addition: oldPair}) + if d.GetStorage().Status != op.WORK { + t.Fatalf("late old-pair 401 changed replacement status to %q", d.GetStorage().Status) + } + if got := len(s.revocationSnapshot()); got != 0 { + t.Fatalf("late old-pair 401 wrote %d revocations", got) + } +} + // Candidate verification must be hermetic with respect to the live storage // record. A temporary driver's successful Init can emit lifecycle callbacks, so // its storage identity must never point at the persisted mount being recovered. @@ -658,9 +702,9 @@ func TestSignedRevocationRemovesRelayCopyButAllowsNewerRotation(t *testing.T) { } } -// canOfferCred requires a local health proof for every candidate. A signature -// proves provenance only; an unvalidated relay must not advertise a pair as a -// recoverable credential. +// A signed candidate is relayable even when the relay has no local health proof. +// Otherwise a hub between two NAT'd nodes black-holes a good source's pair and +// recovery cannot converge. Local health remains required only for activation. func TestCanOfferCred(t *testing.T) { id1, _ := newIdentity() // self id2, _ := newIdentity() // peer @@ -676,7 +720,7 @@ func TestCanOfferCred(t *testing.T) { t.Fatal("a nil record is never offerable") } - // A peer-authored candidate with no local proof is catalog data only. + // A peer-authored signed candidate must traverse this node unchanged. payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"peer"`)} peer := &credRecord{ GroupID: "g1", @@ -689,8 +733,8 @@ func TestCanOfferCred(t *testing.T) { OriginMount: "/peer", } peer.Sig = id2.sign(peer.signingBytes()) - if m.canOfferCred(peer) { - t.Fatal("an unvalidated peer candidate must not be advertised for recovery") + if !m.canOfferCred(peer) { + t.Fatal("an authorized peer candidate must be relayed without local proof") } // our own record, but no group/healthy mount -> not offerable. @@ -700,6 +744,94 @@ func TestCanOfferCred(t *testing.T) { } } +// This is an end-to-end backend regression for the exact missed transition: +// a peer candidate may arrive after the local mount is already invalid. It uses +// the registered production Local driver, real op storage persistence, and the +// same isolated probe + UpdateStorage path used in production -- not a mock +// driver or a mocked callback. The 115-specific three-node acceptance remains +// a separate live test because it must use a real operator-provided pair. +func TestReceivedCandidateRecoversInvalidMountWithRealDriver(t *testing.T) { + dbName := fmt.Sprintf("file:cluster-recovery-%d?mode=memory&cache=shared", time.Now().UnixNano()) + database, err := gorm.Open(sqlite.Open(dbName), &gorm.Config{}) + if err != nil { + t.Fatalf("open test database: %v", err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + + oldRoot := t.TempDir() + mount := fmt.Sprintf("/cluster-recovery-%d", time.Now().UnixNano()) + storage := model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: fmt.Sprintf(`{"root_folder_path":%q}`, oldRoot), + } + id, err := op.CreateStorage(context.Background(), storage) + if err != nil { + t.Fatalf("create real local storage: %v", err) + } + t.Cleanup(func() { _ = op.DeleteStorageById(context.Background(), id) }) + + local, _ := newIdentity() + peer, _ := newIdentity() + admin, _ := newIdentity() + state := newStore() + state.setGroups(admin, []group{{ + ID: "g1", + Members: []member{ + {NodeID: local.NodeID, MountPath: mount}, + {NodeID: peer.NodeID, MountPath: "/peer"}, + }, + }}, 1) + manager := &Manager{ + dir: t.TempDir(), + id: local, + state: state, + conns: newConnRegistry(), + cfgStore: &configStore{cfg: Config{Enabled: true, Key: "test-key", ApplyRemote: true}}, + } + + d, err := op.GetStorageByMountPath(mount) + if err != nil { + t.Fatalf("get created storage: %v", err) + } + // Model the persisted outcome of a real 401 before a peer has a replacement. + d.GetStorage().SetStatus("token invalid") + payload := map[string]json.RawMessage{"access_token": json.RawMessage(`"peer-pair"`)} + candidate := &credRecord{ + GroupID: "g1", + OriginDriver: "Local", + Fields: []string{"access_token"}, + Payload: payload, + CredHash: credHash(payload), + Version: 1, + Origin: peer.NodeID, + OriginPub: peer.Pub, + OriginMount: "/peer", + UpdatedAt: now(), + } + candidate.Sig = peer.sign(candidate.signingBytes()) + candidate.MountSig = peer.sign(candidate.mountSigningBytes()) + if merged := manager.absorbCreds([]*credRecord{candidate}); len(merged) != 1 { + t.Fatalf("candidate merge = %#v, want one accepted record", merged) + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + current, getErr := op.GetStorageByMountPath(mount) + if getErr == nil && current.GetStorage().Status == op.WORK && state.candidateHealthy("g1", candidate.CredHash, now()) { + for _, event := range manager.eventList() { + if event.Kind == "apply" && event.GroupID == "g1" { + return + } + } + } + time.Sleep(20 * time.Millisecond) + } + current, _ := op.GetStorageByMountPath(mount) + t.Fatalf("received candidate did not activate invalid mount: %#v", current.GetStorage()) +} + func TestAbsorbCredsRejectsUnauthorizedOriginMount(t *testing.T) { admin, _ := newIdentity() memberID, _ := newIdentity() @@ -709,7 +841,7 @@ func TestAbsorbCredsRejectsUnauthorizedOriginMount(t *testing.T) { ID: "g1", Members: []member{{NodeID: memberID.NodeID, MountPath: "/115"}}, }}, 1) - m := &Manager{id: localID, state: s, cfgStore: &configStore{cfg: Config{ApplyRemote: false}}} + m := &Manager{dir: t.TempDir(), id: localID, state: s, cfgStore: &configStore{cfg: Config{ApplyRemote: false}}} payload := map[string]json.RawMessage{"refresh_token": json.RawMessage(`"candidate"`)} bad := &credRecord{ diff --git a/internal/db/storage.go b/internal/db/storage.go index 5c5d98c1c4..decec5b640 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" @@ -27,6 +28,20 @@ func UpdateStorageStatus(id uint, status string) error { return errors.WithStack(db.Model(&model.Storage{}).Where("id = ?", id).Update("status", status).Error) } +// UpdateStorageStatusIfModified updates a lifecycle status only if the row is +// still the storage generation observed by the caller. This is used for auth +// callbacks: an old in-flight request must not mark a newly reconfigured token +// invalid after UpdateStorage has advanced Modified. +func UpdateStorageStatusIfModified(id uint, modified time.Time, status string) (bool, error) { + tx := db.Model(&model.Storage{}). + Where("id = ? AND modified = ?", id, modified). + Update("status", status) + if tx.Error != nil { + return false, errors.WithStack(tx.Error) + } + return tx.RowsAffected > 0, nil +} + // DeleteStorageById just delete storage from database by id func DeleteStorageById(id uint) error { return errors.WithStack(db.Delete(&model.Storage{}, id).Error) diff --git a/internal/op/hook.go b/internal/op/hook.go index 1f675ff430..15f27bdfe4 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -5,6 +5,7 @@ import ( "regexp" "strings" "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -102,10 +103,25 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { type StorageHook func(typ string, storage driver.Driver) type StorageHealthHook func(storage driver.Driver) +// StorageCredentialEvent binds an authentication result to the exact Addition +// snapshot used by that request. A storage may be reinitialized with a peer +// credential while an earlier request is still in flight; listeners must never +// let that older result change the newer pair's cluster state. +type StorageCredentialEvent struct { + Storage driver.Driver + Addition string + Modified time.Time +} + +type StorageCredentialHook func(typ string, event StorageCredentialEvent) +type StorageCredentialHealthHook func(event StorageCredentialEvent) + var ( - storageHooks = make([]StorageHook, 0) - storageHealthHooks = make([]StorageHealthHook, 0) - storageTokenStatusMu sync.Mutex + storageHooks = make([]StorageHook, 0) + storageHealthHooks = make([]StorageHealthHook, 0) + storageCredentialHooks = make([]StorageCredentialHook, 0) + storageCredentialHealthHooks = make([]StorageCredentialHealthHook, 0) + storageTokenStatusMu sync.Mutex ) func callStorageHooks(typ string, storage driver.Driver) { @@ -125,12 +141,46 @@ func RegisterStorageHealthHook(hook StorageHealthHook) { storageHealthHooks = append(storageHealthHooks, hook) } +// RegisterStorageCredentialHook receives token-valid/token-invalid events +// accompanied by the immutable Addition snapshot used on the wire. +func RegisterStorageCredentialHook(hook StorageCredentialHook) { + storageCredentialHooks = append(storageCredentialHooks, hook) +} + +// RegisterStorageCredentialHealthHook receives authenticated-success proofs +// accompanied by the exact Addition snapshot that succeeded. +func RegisterStorageCredentialHealthHook(hook StorageCredentialHealthHook) { + storageCredentialHealthHooks = append(storageCredentialHealthHooks, hook) +} + func NotifyStorageTokenHealthy(storage driver.Driver) { for _, hook := range storageHealthHooks { hook(storage) } } +// NotifyStorageTokenHealthyWithAddition is the generation-safe version used by +// drivers whose client can outlive a storage reconfiguration. +func NotifyStorageTokenHealthyWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenHealthyWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenHealthyWithSnapshot rejects a late success from an older +// storage generation before it can renew local cluster-health proof. +func NotifyStorageTokenHealthyWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + if !storageSnapshotMatches(storage, addition, modified) { + return + } + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHealthHooks { + hook(event) + } +} + // NotifyStorageTokenInvalid signals that a storage's credentials were found to be // invalid/expired while in use (as opposed to successfully refreshed). It fires // the storage hook with the "token-invalid" type so listeners — notably cluster @@ -139,32 +189,73 @@ func NotifyStorageTokenHealthy(storage driver.Driver) { // an unrecoverable auth error. func NotifyStorageTokenInvalid(storage driver.Driver) { st := storage.GetStorage() - if st == nil || st.Disabled { + if st == nil { + return + } + NotifyStorageTokenInvalidWithAddition(storage, st.Addition) +} + +// NotifyStorageTokenInvalidWithAddition records a 401 result only while the +// driver still serves the exact pair that made the failed request. A late 401 +// from an old driver generation is therefore harmless after peer recovery has +// installed a newer pair. +func NotifyStorageTokenInvalidWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenInvalidWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenInvalidWithSnapshot records a 401 only while both the +// in-memory storage and its durable row still match the request generation. +func NotifyStorageTokenInvalidWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + st := storage.GetStorage() + if st == nil || st.Disabled || !storageSnapshotMatches(storage, addition, modified) { return } storageTokenStatusMu.Lock() - _ = markStorageTokenInvalidStatus(storage) + recorded := markStorageTokenInvalidStatus(storage, addition, modified) storageTokenStatusMu.Unlock() + if !recorded { + return + } // Persisting status is edge-triggered, but recovery is not. A newly adopted // remote candidate can fail while this mount is already marked invalid; that // failure must still reach cluster recovery, which performs its own per-mount - // cooldown and candidate de-duplication. + // queue and candidate de-duplication. go callStorageHooks("token-invalid", storage) + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHooks { + hook := hook + go hook("token-invalid", event) + } } -func markStorageTokenInvalidStatus(storage driver.Driver) bool { +func markStorageTokenInvalidStatus(storage driver.Driver, addition string, modified time.Time) bool { st := storage.GetStorage() - if st == nil || st.Disabled || st.Status != WORK { + if st == nil || st.Disabled || !storageSnapshotMatches(storage, addition, modified) { return false } const invalidStatus = "token invalid" - st.SetStatus(invalidStatus) if st.ID == 0 { + st.SetStatus(invalidStatus) return true } - if err := db.UpdateStorageStatus(st.ID, invalidStatus); err != nil { + updated, err := db.UpdateStorageStatusIfModified(st.ID, modified, invalidStatus) + if err != nil { log.Errorf("failed mark storage token invalid: %s", err) + return false + } + if !updated { + return false + } + // The conditional database update proves the durable row still belongs to + // this request generation. Re-check memory before changing its status too. + if !storageSnapshotMatches(storage, addition, modified) { + return false } + st.SetStatus(invalidStatus) return true } @@ -174,25 +265,64 @@ func markStorageTokenInvalidStatus(storage driver.Driver) bool { // (re)share the proven token with peers. Drivers should call this only on a real // transition or stale-status recovery to avoid per-request churn. func NotifyStorageTokenValid(storage driver.Driver) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenValidWithAddition(storage, st.Addition) +} + +// NotifyStorageTokenValidWithAddition restores status and emits a lifecycle +// event only when the authenticated request proved the pair currently mounted. +func NotifyStorageTokenValidWithAddition(storage driver.Driver, addition string) { + st := storage.GetStorage() + if st == nil { + return + } + NotifyStorageTokenValidWithSnapshot(storage, addition, st.Modified) +} + +// NotifyStorageTokenValidWithSnapshot restores status only for the exact +// request generation that completed successfully. +func NotifyStorageTokenValidWithSnapshot(storage driver.Driver, addition string, modified time.Time) { + if !storageSnapshotMatches(storage, addition, modified) { + return + } storageTokenStatusMu.Lock() - changed := restoreStorageTokenValidStatus(storage) + changed := restoreStorageTokenValidStatus(storage, addition, modified) storageTokenStatusMu.Unlock() if changed { go callStorageHooks("token-valid", storage) + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHooks { + hook := hook + go hook("token-valid", event) + } } } -func restoreStorageTokenValidStatus(storage driver.Driver) bool { +func storageSnapshotMatches(storage driver.Driver, addition string, modified time.Time) bool { + st := storage.GetStorage() + return st != nil && st.Addition == addition && st.Modified.Equal(modified) +} + +func restoreStorageTokenValidStatus(storage driver.Driver, addition string, modified time.Time) bool { st := storage.GetStorage() - if st == nil || st.Disabled || st.Status == WORK { + if st == nil || st.Disabled || st.Status == WORK || !storageSnapshotMatches(storage, addition, modified) { return false } - st.SetStatus(WORK) if st.ID == 0 { + st.SetStatus(WORK) return true } - if err := db.UpdateStorageStatus(st.ID, WORK); err != nil { + updated, err := db.UpdateStorageStatusIfModified(st.ID, modified, WORK) + if err != nil { log.Errorf("failed mark storage token valid: %s", err) + return false } + if !updated || !storageSnapshotMatches(storage, addition, modified) { + return false + } + st.SetStatus(WORK) return true } diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go index 823ac80cb4..64bf7b853c 100644 --- a/internal/op/hook_test.go +++ b/internal/op/hook_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + open115 "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -88,6 +89,73 @@ func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { } } +// An old client's 401 must be ignored after cluster recovery has installed a +// different pair. This uses the real 115 driver storage type plus real SQLite +// persistence; no test driver or callback mock is involved. +func TestNotifyStorageTokenInvalidWithStaleAdditionKeepsReplacementWorking(t *testing.T) { + newAddition := `{"access_token":"new","refresh_token":"new"}` + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/token-invalid-stale-addition", + Status: op.WORK, + Addition: newAddition, + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + d := &open115.Open115{Storage: storage} + op.NotifyStorageTokenInvalidWithAddition(d, `{"access_token":"old","refresh_token":"old"}`) + if d.GetStorage().Status != op.WORK { + t.Fatalf("stale invalidation changed in-memory status to %q", d.GetStorage().Status) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Status != op.WORK { + t.Fatalf("stale invalidation changed persisted status to %q", got.Status) + } +} + +// Addition equality alone is not enough: UpdateStorage can replace a row while +// an old client's callback is racing. The durable Modified generation is the +// compare-and-swap guard that prevents that callback from touching the new row. +func TestNotifyStorageTokenInvalidWithStaleGenerationKeepsReplacementWorking(t *testing.T) { + oldAddition := `{"access_token":"old","refresh_token":"old"}` + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/token-invalid-stale-generation", + Status: op.WORK, + Addition: oldAddition, + Modified: time.Now().Add(-time.Minute), + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + oldGeneration := storage.Modified + newStorage := storage + newStorage.Addition = `{"access_token":"new","refresh_token":"new"}` + newStorage.Modified = time.Now() + if err := db.UpdateStorage(&newStorage); err != nil { + t.Fatalf("UpdateStorage replacement failed: %v", err) + } + + // d deliberately models the old client object whose request began before + // the durable replacement. It is a real 115 driver value, not a fake. + d := &open115.Open115{Storage: storage} + op.NotifyStorageTokenInvalidWithSnapshot(d, oldAddition, oldGeneration) + if d.GetStorage().Status != op.WORK { + t.Fatalf("stale generation changed old in-memory status to %q", d.GetStorage().Status) + } + got, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if got.Addition != newStorage.Addition || got.Status != op.WORK { + t.Fatalf("stale generation changed replacement row: %#v", got) + } +} + func TestNotifyStorageTokenInvalidDispatchesRecoveryWhenAlreadyInvalid(t *testing.T) { d := &tokenValidStorageDriver{Storage: model.Storage{ Driver: "TokenInvalidTest", From d560eedd0c274e5c7f3ae98c5edae8df9aa7dc99 Mon Sep 17 00:00:00 2001 From: ironbox Date: Mon, 20 Jul 2026 13:00:01 +0800 Subject: [PATCH 106/107] fix(cluster): publish refreshed 115 credentials --- drivers/115_open/driver.go | 18 ++++-- drivers/115_open/driver_test.go | 104 ++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 + internal/op/hook.go | 13 ++-- internal/op/hook_test.go | 31 ++++++++++ 6 files changed, 160 insertions(+), 10 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 5db3833dff..deab09dd40 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -62,6 +62,7 @@ func (d *Open115) Init(ctx context.Context) error { // this client so an old 401 cannot revoke the newly installed pair. observedAddition := d.Storage.Addition observedModified := d.Storage.Modified + observedAccessToken := d.Addition.AccessToken d.client = new115SDKClient(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), sdk.WithOnRefreshToken(func(s1, s2 string) { @@ -72,13 +73,22 @@ func (d *Open115) Init(ctx context.Context) error { // Cluster token-validity protocol: a node only shares a token it has // proven valid, and pulls a fresh one from a healthy peer when its own // dies. Edge-triggered so the cluster only reacts to real transitions. - sdk.WithOnTokenValid(func() { + sdk.WithOnAccessTokenValid(func(accessToken string) { + st := d.GetStorage() + if st == nil || d.Addition.AccessToken != accessToken { + return + } + currentAddition := st.Addition + currentModified := st.Modified // A successful authenticated request renews the cluster's in-memory // proof only. This avoids a periodic refresh/push while preventing a // working credential from ageing out of recovery advertisement. - op.NotifyStorageTokenHealthyWithSnapshot(d, observedAddition, observedModified) - if d.shouldNotifyTokenValid() { - op.NotifyStorageTokenValidWithSnapshot(d, observedAddition, observedModified) + op.NotifyStorageTokenHealthyWithSnapshot(d, currentAddition, currentModified) + // A refresh creates a new credential generation even when the mount + // never left WORK. Publish that generation after the retried request + // proves it, and also recover any pre-existing error status. + if accessToken != observedAccessToken || d.shouldNotifyTokenValid() { + op.NotifyStorageTokenValidWithSnapshot(d, currentAddition, currentModified) } }), sdk.WithOnTokenInvalid(func() { diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index b0aa657b11..29ad4ab455 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -16,13 +16,16 @@ import ( sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" driverpkg "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/glebarez/sqlite" "golang.org/x/time/rate" + "gorm.io/gorm" ) type recordedRequest struct { @@ -314,6 +317,107 @@ func TestOpen115InitRateLimitsAuthAndRootInfo(t *testing.T) { } } +func TestOpen115RefreshRestoresErrorStateAndPublishesNewPair(t *testing.T) { + database, err := gorm.Open(sqlite.Open("file:open115-refresh-recovery?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("open test database: %v", err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + + var refreshCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/refreshToken": + refreshCount++ + writeSDKResponse(t, w, map[string]any{ + "state": 1, + "code": 0, + "data": map[string]any{ + "access_token": "fresh-access", + "refresh_token": "fresh-refresh", + "expires_in": 7200, + }, + }) + case "/open/user/info": + if r.Header.Get("Authorization") != "Bearer fresh-access" { + writeSDKError(t, w, 40140125, "access_token invalid") + return + } + writeSDKSuccess(t, w, map[string]any{}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + oldNewClient := new115SDKClient + t.Cleanup(func() { new115SDKClient = oldNewClient }) + new115SDKClient = func(opts ...sdk.Option) *sdk.Client { + client := sdk.New(opts...) + client.SetHttpClient(&http.Client{Transport: &rewriteTransport{target: target, base: http.DefaultTransport}}) + return client + } + + driver := &Open115{Addition: Addition{ + RootID: driverpkg.RootID{RootFolderID: "0"}, + AccessToken: "expired-access", + RefreshToken: "live-refresh", + }} + storage := model.Storage{ + Driver: "115 Open", + MountPath: "/refresh-recovery", + Status: "code: 40140125, message: access_token invalid", + Addition: `{"access_token":"expired-access","refresh_token":"live-refresh"}`, + } + if err := db.CreateStorage(&storage); err != nil { + t.Fatalf("CreateStorage failed: %v", err) + } + driver.Storage = storage + proof := make(chan op.StorageCredentialEvent, 1) + op.RegisterStorageCredentialHook(func(typ string, event op.StorageCredentialEvent) { + if typ != "token-valid" || event.Storage != driver { + return + } + select { + case proof <- event: + default: + } + }) + + if err := driver.Init(context.Background()); err != nil { + t.Fatalf("Init after access-token expiry failed: %v", err) + } + if refreshCount != 1 { + t.Fatalf("refresh count = %d, want exactly 1", refreshCount) + } + if driver.Addition.AccessToken != "fresh-access" || driver.Addition.RefreshToken != "fresh-refresh" { + t.Fatalf("refreshed pair was not installed: access=%q refresh=%q", driver.Addition.AccessToken, driver.Addition.RefreshToken) + } + if driver.Storage.Status != op.WORK { + t.Fatalf("refreshed mount status = %q, want WORK", driver.Storage.Status) + } + persisted, err := db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById failed: %v", err) + } + if persisted.Status != op.WORK || !strings.Contains(persisted.Addition, "fresh-access") { + t.Fatalf("refreshed storage was not durably recovered: %#v", persisted) + } + select { + case event := <-proof: + if !strings.Contains(event.Addition, "fresh-access") || !strings.Contains(event.Addition, "fresh-refresh") { + t.Fatalf("published stale credential generation: %s", event.Addition) + } + case <-time.After(time.Second): + t.Fatal("refreshed and proven pair was not published") + } +} + func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { t.Helper() diff --git a/go.mod b/go.mod index a5d0d5a5b3..008e11f657 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.13 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.14 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 279b63d23d..3ac1cf7c3d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/Ironboxplus/115-sdk-go v0.2.12 h1:6WB2L9FG4n+Xv5RvYXQxZBFvbhgYlWoqKYp github.com/Ironboxplus/115-sdk-go v0.2.12/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/Ironboxplus/115-sdk-go v0.2.13 h1:B815N2mOq2lF2QDyANnkZW3mpmFM7xrBJrqFeENeYVs= github.com/Ironboxplus/115-sdk-go v0.2.13/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.14 h1:BN1FjRDbJ/UQiwWwO6mbfoU52L9/46l5oHbyEcWDhSI= +github.com/Ironboxplus/115-sdk-go v0.2.14/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= diff --git a/internal/op/hook.go b/internal/op/hook.go index 15f27bdfe4..89a3f9b55b 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -293,11 +293,14 @@ func NotifyStorageTokenValidWithSnapshot(storage driver.Driver, addition string, storageTokenStatusMu.Unlock() if changed { go callStorageHooks("token-valid", storage) - event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} - for _, hook := range storageCredentialHooks { - hook := hook - go hook("token-valid", event) - } + } + // Credential proof is distinct from lifecycle status. A successful refresh + // rotates the pair while the mount may remain WORK; cluster sync must still + // receive and publish that newly proven generation. + event := StorageCredentialEvent{Storage: storage, Addition: addition, Modified: modified} + for _, hook := range storageCredentialHooks { + hook := hook + go hook("token-valid", event) } } diff --git a/internal/op/hook_test.go b/internal/op/hook_test.go index 64bf7b853c..36a5716d28 100644 --- a/internal/op/hook_test.go +++ b/internal/op/hook_test.go @@ -63,6 +63,37 @@ func TestNotifyStorageTokenValidRestoresStatus(t *testing.T) { t.Fatalf("persisted status = %q, want %q", got.Status, op.WORK) } } + +func TestNotifyStorageTokenValidPublishesProvenPairWhileAlreadyWork(t *testing.T) { + d := &open115.Open115{Storage: model.Storage{ + Driver: "115 Open", + MountPath: "/token-valid-rotated-pair", + Status: op.WORK, + Addition: `{"access_token":"fresh","refresh_token":"rotated"}`, + }} + proof := make(chan op.StorageCredentialEvent, 1) + op.RegisterStorageCredentialHook(func(typ string, event op.StorageCredentialEvent) { + if typ != "token-valid" || event.Storage != d { + return + } + select { + case proof <- event: + default: + } + }) + + op.NotifyStorageTokenValidWithSnapshot(d, d.Storage.Addition, d.Storage.Modified) + + select { + case event := <-proof: + if event.Addition != d.Storage.Addition || !event.Modified.Equal(d.Storage.Modified) { + t.Fatalf("credential proof used the wrong generation: %#v", event) + } + case <-time.After(time.Second): + t.Fatal("a proven rotated pair was not published while status was already WORK") + } +} + func TestNotifyStorageTokenInvalidMarksStatusAndPersists(t *testing.T) { storage := model.Storage{ Driver: "TokenInvalidTest", From 5bc9a2a64322dcfceb4d2aaa333f5d53fb040e10 Mon Sep 17 00:00:00 2001 From: ironbox Date: Tue, 21 Jul 2026 04:17:53 +0800 Subject: [PATCH 107/107] fix(cluster): invalidate current refreshed generation --- drivers/115_open/driver.go | 14 ++++++++------ drivers/115_open/driver_test.go | 24 ++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 ++ 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index deab09dd40..69b75e3710 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -58,10 +58,8 @@ func (d *Open115) GetAddition() driver.Additional { func (d *Open115) Init(ctx context.Context) error { // This client generation can finish an old request after cluster recovery has - // replaced the storage. Bind auth callbacks to the Addition that configured - // this client so an old 401 cannot revoke the newly installed pair. - observedAddition := d.Storage.Addition - observedModified := d.Storage.Modified + // replaced the storage. The SDK reports the exact access token used by each + // success/failure so callbacks can bind to the current generation. observedAccessToken := d.Addition.AccessToken d.client = new115SDKClient(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), @@ -91,9 +89,13 @@ func (d *Open115) Init(ctx context.Context) error { op.NotifyStorageTokenValidWithSnapshot(d, currentAddition, currentModified) } }), - sdk.WithOnTokenInvalid(func() { + sdk.WithOnAccessTokenInvalid(func(accessToken string) { + st := d.GetStorage() + if st == nil || d.Addition.AccessToken != accessToken { + return + } if d.tokenInvalid.CompareAndSwap(false, true) { - op.NotifyStorageTokenInvalidWithSnapshot(d, observedAddition, observedModified) + op.NotifyStorageTokenInvalidWithSnapshot(d, st.Addition, st.Modified) } })) applySDKProxyIfConfigured(d.client) diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go index 29ad4ab455..f3892ef5e9 100644 --- a/drivers/115_open/driver_test.go +++ b/drivers/115_open/driver_test.go @@ -11,6 +11,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -326,6 +327,7 @@ func TestOpen115RefreshRestoresErrorStateAndPublishesNewPair(t *testing.T) { db.Init(database) var refreshCount int + var rejectFresh atomic.Bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/open/refreshToken": @@ -344,6 +346,10 @@ func TestOpen115RefreshRestoresErrorStateAndPublishesNewPair(t *testing.T) { writeSDKError(t, w, 40140125, "access_token invalid") return } + if rejectFresh.Load() { + writeSDKError(t, w, 40140120, "refresh token error") + return + } writeSDKSuccess(t, w, map[string]any{}) default: t.Fatalf("unexpected path: %s", r.URL.Path) @@ -416,6 +422,24 @@ func TestOpen115RefreshRestoresErrorStateAndPublishesNewPair(t *testing.T) { case <-time.After(time.Second): t.Fatal("refreshed and proven pair was not published") } + + // The same long-lived driver may fail hours after refreshing. The invalid + // callback must bind to fresh-access, not the pre-refresh generation that + // configured the client. + rejectFresh.Store(true) + if _, err := driver.client.UserInfo(context.Background()); err == nil { + t.Fatal("expected the refreshed generation to become invalid") + } + if driver.Storage.Status == op.WORK { + t.Fatal("refreshed generation 401 did not invalidate the mounted storage") + } + persisted, err = db.GetStorageById(storage.ID) + if err != nil { + t.Fatalf("GetStorageById after invalidation failed: %v", err) + } + if persisted.Status == op.WORK { + t.Fatal("refreshed generation 401 was not persisted for peer recovery") + } } func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { diff --git a/go.mod b/go.mod index 008e11f657..436036b5a9 100644 --- a/go.mod +++ b/go.mod @@ -311,6 +311,6 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.14 +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.15 replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index 3ac1cf7c3d..e9790a64a8 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/Ironboxplus/115-sdk-go v0.2.13 h1:B815N2mOq2lF2QDyANnkZW3mpmFM7xrBJrq github.com/Ironboxplus/115-sdk-go v0.2.13/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/Ironboxplus/115-sdk-go v0.2.14 h1:BN1FjRDbJ/UQiwWwO6mbfoU52L9/46l5oHbyEcWDhSI= github.com/Ironboxplus/115-sdk-go v0.2.14/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/Ironboxplus/115-sdk-go v0.2.15 h1:GEFamzPCINsB2aRxjcQyvsqC0SmXuQ6H417Ga7BKUII= +github.com/Ironboxplus/115-sdk-go v0.2.15/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=