Skip to content

Optimize buffering of forwarded SEARCH requests #4116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
roman-khimov merged 1 commit into from
Aug 27, 2026
Merged

Optimize buffering of forwarded SEARCH requests #4116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@ Changelog for NeoFS Node
- SNs no longer sign TTL=1 requests over mutually authenticated inter-node connections (#4100)
- Optimized GRPC read/write bufferring (#4130)
- TLS key of SN is now always read from the node wallet instead of configuration (#4131)
- SN now allocates less to forward SEARCH requests (#4116)

### Removed
- Session token storage migration (#4124)
- Metabase migrations for DB versions 8 and 9 (#4127)

### Updated
- `github.com/nspcc-dev/neo-go` module to `v0.122.1-0.20260807115931-cfee8827ddfd` (#4123)
- `github.com/nspcc-dev/neofs-sdk-go` module to `v1.0.0-rc.21.0.20260824193430-0f1faf21eb76` (#4123, #4144)
- `github.com/nspcc-dev/neofs-sdk-go` module to `v1.0.0-rc.21.0.20260826165415-15812e94109e` (#4123, #4144, #4116)
- Go 1.26+ is required to build now (#3816)
- github.com/cheggaaa/pb dependency from v1.0.29 to v3.2.1 (#4135)
- `github.com/klauspost/compress` dependency from v1.18.4 to v1.19.2 (#4135)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
github.com/nspcc-dev/neo-go v0.122.1-0.20260807115931-cfee8827ddfd
github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240827150555-5ce597aa14ea
github.com/nspcc-dev/neofs-contract v0.26.1
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260824193430-0f1faf21eb76
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260826165415-15812e94109e
github.com/nspcc-dev/tzhash v1.8.4
github.com/panjf2000/ants/v2 v2.12.1
github.com/prometheus/client_golang v1.24.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,8 @@ github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240827150555-5ce597aa14ea h1:mK
github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240827150555-5ce597aa14ea/go.mod h1:YzhD4EZmC9Z/PNyd7ysC7WXgIgURc9uCG1UWDeV027Y=
github.com/nspcc-dev/neofs-contract v0.26.1 h1:7Ii7Q4L3au408LOsIWKiSgfnT1g8G9jo3W7381d41T8=
github.com/nspcc-dev/neofs-contract v0.26.1/go.mod h1:pevVF9OWdEN5bweKxOu6ryZv9muCEtS1ppzYM4RfBIo=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260824193430-0f1faf21eb76 h1:mWHwhsGMfxr0KKW+0q2kSZrWJ23lxl8Yi5M0w9ZJv6M=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260824193430-0f1faf21eb76/go.mod h1:cFjepLvmVg4DKWgdo3LTsSkemtW0GLzKobOVpsbBGTg=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260826165415-15812e94109e h1:63pBaB1crRFkI9+9hK8NIvHqXGlUA7ar7huhzHTclt0=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.21.0.20260826165415-15812e94109e/go.mod h1:cFjepLvmVg4DKWgdo3LTsSkemtW0GLzKobOVpsbBGTg=
github.com/nspcc-dev/rfc6979 v0.2.4 h1:NBgsdCjhLpEPJZqmC9rciMZDcSY297po2smeaRjw57k=
github.com/nspcc-dev/rfc6979 v0.2.4/go.mod h1:86ylDw6Kss+P6v4QAJqo1Sp3mC0/Zr9G97xSjQ9TuFg=
github.com/nspcc-dev/tzhash v1.8.4 h1:lvuPGWsqEo9dVEvo/kdNLKv/Cy0yxRs9z5hJp8VcBuo=
Expand Down
11 changes: 6 additions & 5 deletions internal/crypto/requests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,28 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/network/peerauth"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto"
protoencoding "github.com/nspcc-dev/neofs-sdk-go/proto/encoding"
"github.com/nspcc-dev/neofs-sdk-go/proto/refs"
protosession "github.com/nspcc-dev/neofs-sdk-go/proto/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
)

// VerifyRequestSignatures checks whether all request signatures are set and
// valid. Returns [apistatus.SignatureVerification] otherwise.
func VerifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedRequest[B]) error {
func VerifyRequestSignatures[B protoencoding.Message](req neofscrypto.SignedRequest[B]) error {
return verifyRequestSignatures(req, nil)
}

// VerifyRequestSignaturesWithContext is same as [VerifyRequestSignatures], but
// skips verification for an authenticated inter-node request with TTL equal to one.
func VerifyRequestSignaturesWithContext[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B]) error {
func VerifyRequestSignaturesWithContext[B protoencoding.Message](ctx context.Context, req neofscrypto.SignedRequest[B]) error {
if !requestNeedsSignature(ctx, req) {
return nil
}
return verifyRequestSignatures(req, nil)
}

func verifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedRequest[B], verifyN3 func(data, invocScript, verifScript []byte) error) error {
func verifyRequestSignatures[B protoencoding.Message](req neofscrypto.SignedRequest[B], verifyN3 func(data, invocScript, verifScript []byte) error) error {
err := neofscrypto.VerifyRequestWithBufferN3(req, nil, verifyN3)
if err != nil {
var st apistatus.SignatureVerification
Expand All @@ -42,7 +43,7 @@ func verifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedR

// VerifyRequestSignaturesN3 is same as [VerifyRequestSignatures] but supports
// [neofscrypto.N3] scheme.
func VerifyRequestSignaturesN3[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B], fsChain N3ScriptRunner) error {
func VerifyRequestSignaturesN3[B protoencoding.Message](ctx context.Context, req neofscrypto.SignedRequest[B], fsChain N3ScriptRunner) error {
if !requestNeedsSignature(ctx, req) {
return nil
}
Expand All @@ -54,7 +55,7 @@ func VerifyRequestSignaturesN3[B neofscrypto.ProtoMessage](ctx context.Context,
})
}

func requestNeedsSignature[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B]) bool {
func requestNeedsSignature[B protoencoding.Message](ctx context.Context, req neofscrypto.SignedRequest[B]) bool {
if req.GetVerifyHeader() != nil {
return true
}
Expand Down
12 changes: 8 additions & 4 deletions pkg/services/object/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,17 @@ func forwardServerStreamRequestGRPC(ctx context.Context, req any, respStream grp
}
}

func callUnary(ctx context.Context, conn *grpc.ClientConn, method string, req any) (mem.BufferSlice, error) {
var respBuf mem.BufferSlice

err := conn.Invoke(ctx, method, req, &respBuf,
func callUnaryWithCustomResponse(ctx context.Context, conn *grpc.ClientConn, method string, req any, resp any) error {
return conn.Invoke(ctx, method, req, resp,
grpc.StaticMethod(),
grpc.ForceCodecV2(iprotobuf.BufferedCodec{}),
)
}

func callUnary(ctx context.Context, conn *grpc.ClientConn, method string, req any) (mem.BufferSlice, error) {
var respBuf mem.BufferSlice

err := callUnaryWithCustomResponse(ctx, conn, method, req, &respBuf)
if err != nil {
return nil, fmt.Errorf("sending the request failed: %w", err)
}
Expand Down
117 changes: 22 additions & 95 deletions pkg/services/object/get.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,13 @@ import (
getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
protoobject "github.com/nspcc-dev/neofs-sdk-go/proto/object"
iprotobuf "github.com/nspcc-dev/neofs-sdk-go/proto/protobuf"
"github.com/nspcc-dev/neofs-sdk-go/proto/protobuf/protoscan"
protorefs "github.com/nspcc-dev/neofs-sdk-go/proto/refs"
protostatus "github.com/nspcc-dev/neofs-sdk-go/proto/status"
"github.com/nspcc-dev/neofs-sdk-go/version"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/mem"
Expand Down Expand Up @@ -480,7 +479,7 @@ func (x *getECTransport) CopyLocalECPartRange(ctx context.Context, storage *engi
return ln, nil
}

func (x *getECTransport) initGetPartRequest(remoteServerAPIVersion *protorefs.Version, partInfo iec.PartInfo) error {
func (x *getECTransport) initGetPartRequest(remoteServerAPIVersion version.Version, partInfo iec.PartInfo) error {
if x.getPartRequestInfo == partInfo && x.getPartRequest != nil {
return nil
}
Expand All @@ -503,7 +502,8 @@ func (x *getECTransport) CopyRemoteECPartParentHeaderAndPayload(ctx context.Cont
var partPldLen uint64
var copiedPartPld uint64

connAPIVersion := conn.APIVersion()
connAPIVersionMsg := conn.APIVersion()
connAPIVersion := version.New(connAPIVersionMsg.GetMajor(), connAPIVersionMsg.GetMinor())

err := conn.ForAnyGRPCConn(ctx, func(ctx context.Context, conn *grpc.ClientConn) error {
if !copiedHdr {
Expand Down Expand Up @@ -651,7 +651,7 @@ func (x *getECTransport) copyRemotePart(ctx context.Context, conn *grpc.ClientCo
return copiedHdr, parentPldLen, partPldLen, copiedPartPldLen, nil
}

func (x *getECTransport) copyRemotePartRange(ctx context.Context, conn *grpc.ClientConn, connAPIVersion *protorefs.Version, partInfo iec.PartInfo, off, ln uint64, controlCh <-chan bool) (uint64, error) {
func (x *getECTransport) copyRemotePartRange(ctx context.Context, conn *grpc.ClientConn, connAPIVersion version.Version, partInfo iec.PartInfo, off, ln uint64, controlCh <-chan bool) (uint64, error) {
request, err := x.makeGetECPartRangeRequest(connAPIVersion, partInfo, off, ln)
if err != nil {
return 0, fmt.Errorf("make request: %w", err)
Expand Down Expand Up @@ -835,7 +835,8 @@ func handleGetECPartResponseInit(buffers iprotobuf.BuffersSlice) (iprotobuf.Buff
func (x *getECTransport) CopyRemoteECPartRange(ctx context.Context, conn clientcore.MultiAddressClient, partInfo iec.PartInfo, off uint64, ln uint64, full bool, controlCh <-chan bool) (uint64, error) {
var copiedPld uint64

connAPIVersion := conn.APIVersion()
connAPIVersionMsg := conn.APIVersion()
connAPIVersion := version.New(connAPIVersionMsg.GetMajor(), connAPIVersionMsg.GetMinor())

@roman-khimov roman-khimov Aug 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previous API was made to optimize out version->protorefs conversion, now we have protorefs->version. Maybe we need to change the API then.


err := conn.ForAnyGRPCConn(ctx, func(ctx context.Context, conn *grpc.ClientConn) error {
var reqLen uint64
Expand Down Expand Up @@ -866,7 +867,7 @@ func (x *getECTransport) CopyRemoteECPartRange(ctx context.Context, conn clientc
return copiedPld, nil
}

func (s *Server) makeGetECPartRequest(remoteServerAPIVersion *protorefs.Version, cnr cid.ID, parent oid.ID, partInfo iec.PartInfo) (mem.Buffer, error) {
func (s *Server) makeGetECPartRequest(remoteServerAPIVersion version.Version, cnr cid.ID, parent oid.ID, partInfo iec.PartInfo) (mem.Buffer, error) {
ruleIdxStr := strconv.Itoa(partInfo.RuleIndex)
partIdxStr := strconv.Itoa(partInfo.Index)

Expand All @@ -875,29 +876,24 @@ func (s *Server) makeGetECPartRequest(remoteServerAPIVersion *protorefs.Version,

metaHdrLen := calculateGetECPartRequestMetaHeaderLength(ruleIdxHdrLen, partIdxHdrLen)

verifHdrSigCount := getRequestVerificationSignaturesCount(remoteServerAPIVersion)

verifHdrLen := calculateRequestVerificationHeaderLen(verifHdrSigCount)
verifHdrFldLen := calculateRequestVerificationHeaderFieldLen(remoteServerAPIVersion)

reqLen := 1 + 1 + getByAddressRequestBodyLen + // first 1 for iprotobuf.TagBytes1
1 + protowire.SizeBytes(metaHdrLen) + // 1 for iprotobuf.TagBytes2
1 + protowire.SizeBytes(verifHdrLen) // 1 for iprotobuf.TagBytes3
verifHdrFldLen

// TODO: try with sync.Pool
buf := make([]byte, reqLen)

n, err := s.writeGetECPartRequest(buf, cnr, parent, metaHdrLen, ruleIdxHdrLen, ruleIdxStr, partIdxHdrLen, partIdxStr, verifHdrSigCount)
err := s.writeGetECPartRequest(buf, cnr, parent, metaHdrLen, ruleIdxHdrLen, ruleIdxStr, partIdxHdrLen, partIdxStr, remoteServerAPIVersion)
if err != nil {
return nil, err
}
if n != len(buf) {
return nil, fmt.Errorf("got wrong request length: expected %d, got %d", n, len(buf))
}

return mem.SliceBuffer(buf), nil
}

func (x *getECTransport) makeGetECPartRangeRequest(remoteServerAPIVersion *protorefs.Version, partInfo iec.PartInfo, off, ln uint64) (mem.Buffer, error) {
func (x *getECTransport) makeGetECPartRangeRequest(remoteServerAPIVersion version.Version, partInfo iec.PartInfo, off, ln uint64) (mem.Buffer, error) {
x.getPartRangeRequestsMtx.RLock()
req := x.getPartRangeRequests[partInfo]
x.getPartRangeRequestsMtx.RUnlock()
Expand Down Expand Up @@ -934,7 +930,7 @@ func (x *getECTransport) makeGetECPartRangeRequest(remoteServerAPIVersion *proto
return reqBuf, nil
}

func (s *Server) makeGetECPartRangeRequest(remoteServerAPIVersion *protorefs.Version, cnr cid.ID, parent oid.ID, partInfo iec.PartInfo, off, ln uint64) (mem.Buffer, error) {
func (s *Server) makeGetECPartRangeRequest(remoteServerAPIVersion version.Version, cnr cid.ID, parent oid.ID, partInfo iec.PartInfo, off, ln uint64) (mem.Buffer, error) {
ruleIdxStr := strconv.Itoa(partInfo.RuleIndex)
partIdxStr := strconv.Itoa(partInfo.Index)

Expand All @@ -960,39 +956,25 @@ func (s *Server) makeGetECPartRangeRequest(remoteServerAPIVersion *protorefs.Ver
// payload_only flag
bodyLen += 1 + 1 // 1 for iprotobuf.TagVarint4, 1 for true

verifHdrSigCount := getRequestVerificationSignaturesCount(remoteServerAPIVersion)

verifHdrLen := calculateRequestVerificationHeaderLen(verifHdrSigCount)
verifHdrFldLen := calculateRequestVerificationHeaderFieldLen(remoteServerAPIVersion)

reqLen := 1 + protowire.SizeBytes(bodyLen) + // 1 for iprotobuf.TagBytes1
1 + protowire.SizeBytes(metaHdrLen) + // 1 for iprotobuf.TagBytes2
1 + protowire.SizeBytes(verifHdrLen) // 1 for iprotobuf.TagBytes3
verifHdrFldLen

// TODO: try with sync.Pool
buf := make([]byte, reqLen)

n, err := s.writeGetECPartRangeRequest(buf, bodyLen, cnr, parent, rngLen, off, ln,
metaHdrLen, ruleIdxHdrLen, ruleIdxStr, partIdxHdrLen, partIdxStr, verifHdrSigCount)
err := s.writeGetECPartRangeRequest(buf, bodyLen, cnr, parent, rngLen, off, ln,
metaHdrLen, ruleIdxHdrLen, ruleIdxStr, partIdxHdrLen, partIdxStr, remoteServerAPIVersion)
if err != nil {
return nil, err
}
if n != len(buf) {
return nil, fmt.Errorf("got wrong request length: expected %d, got %d", n, len(buf))
}

return mem.SliceBuffer(buf), nil
}

func (s *Server) writeGetECPartRequest(buf []byte, cnr cid.ID, parent oid.ID, metaHdrLen int, ruleIdxHdrLen int, ruleIdxHdr string, partIdxHdrLen int, partIdxHdr string, verifHdrSigCount int) (int, error) {
var originSig []byte
var err error
if verifHdrSigCount == 3 {
originSig, err = neofsecdsa.Signer(s.signer).Sign(nil)
if err != nil {
return 0, fmt.Errorf("sign empty data: %w", err)
}
}

func (s *Server) writeGetECPartRequest(buf []byte, cnr cid.ID, parent oid.ID, metaHdrLen int, ruleIdxHdrLen int, ruleIdxHdr string, partIdxHdrLen int, partIdxHdr string, apiVersion version.Version) error {
// body
buf[0] = iprotobuf.TagBytes1
buf[1] = getByAddressRequestBodyLen
Expand All @@ -1009,14 +991,6 @@ func (s *Server) writeGetECPartRequest(buf []byte, cnr cid.ID, parent oid.ID, me
buf[43] = oid.Size
copy(buf[44:], parent[:])

var bodySig []byte
if verifHdrSigCount > 1 {
bodySig, err = signECDSAWithSHA512(s.signer, buf[2:76])
if err != nil {
return 0, fmt.Errorf("sign body: %w", err)
}
}

// meta header
buf[76] = iprotobuf.TagBytes2
off := 77 + binary.PutUvarint(buf[77:], uint64(metaHdrLen))
Expand All @@ -1027,37 +1001,12 @@ func (s *Server) writeGetECPartRequest(buf []byte, cnr cid.ID, parent oid.ID, me
off += writeRequestMetaXHeader(buf[off:], ruleIdxHdrLen, iec.AttributeRuleIdx, ruleIdxHdr)
off += writeRequestMetaXHeader(buf[off:], partIdxHdrLen, iec.AttributePartIdx, partIdxHdr)

var metaHdrSig []byte
var reqSig []byte
if verifHdrSigCount > 1 {
metaHdrSig, err = signECDSAWithSHA512(s.signer, buf[from:off])
if err != nil {
return 0, fmt.Errorf("sign meta header: %w", err)
}
} else {
reqSig, err = signECDSAWithSHA512(s.signer, buf[:off])
if err != nil {
return 0, fmt.Errorf("sign body and meta header: %w", err)
}
}

// verification header
off += writeRequestVerificationHeader(buf[off:], verifHdrSigCount, s.pubKeyBytes, bodySig, metaHdrSig, originSig, reqSig)

return off, nil
return s.writeRequestSignatures(buf, off, buf[2:76], buf[from:off], apiVersion)
}

func (s *Server) writeGetECPartRangeRequest(buf []byte, bodyLen int, cnr cid.ID, parent oid.ID, rngLen int, off uint64, ln uint64,
metaHdrLen int, ruleIdxHdrLen int, ruleIdxHdr string, partIdxHdrLen int, partIdxHdr string, verifHdrSigCount int) (int, error) {
var originSig []byte
var err error
if verifHdrSigCount == 3 {
originSig, err = neofsecdsa.Signer(s.signer).Sign(nil)
if err != nil {
return 0, fmt.Errorf("sign empty data: %w", err)
}
}

metaHdrLen int, ruleIdxHdrLen int, ruleIdxHdr string, partIdxHdrLen int, partIdxHdr string, apiVersion version.Version) error {
// body
buf[0] = iprotobuf.TagBytes1
n := 1 + binary.PutUvarint(buf[1:], uint64(bodyLen))
Expand Down Expand Up @@ -1104,13 +1053,7 @@ func (s *Server) writeGetECPartRangeRequest(buf []byte, bodyLen int, cnr cid.ID,
buf[n] = 1 // true
n++

var bodySig []byte
if verifHdrSigCount > 1 {
bodySig, err = signECDSAWithSHA512(s.signer, buf[from:n])
if err != nil {
return 0, fmt.Errorf("sign body: %w", err)
}
}
body := buf[from:n]

// meta header
buf[n] = iprotobuf.TagBytes2
Expand All @@ -1123,24 +1066,8 @@ func (s *Server) writeGetECPartRangeRequest(buf []byte, bodyLen int, cnr cid.ID,
n += writeRequestMetaXHeader(buf[n:], ruleIdxHdrLen, iec.AttributeRuleIdx, ruleIdxHdr)
n += writeRequestMetaXHeader(buf[n:], partIdxHdrLen, iec.AttributePartIdx, partIdxHdr)

var metaHdrSig []byte
var reqSig []byte
if verifHdrSigCount > 1 {
metaHdrSig, err = signECDSAWithSHA512(s.signer, buf[from:off])
if err != nil {
return 0, fmt.Errorf("sign meta header: %w", err)
}
} else {
reqSig, err = signECDSAWithSHA512(s.signer, buf[:n])
if err != nil {
return 0, fmt.Errorf("sign body and meta header: %w", err)
}
}

// verification header
n += writeRequestVerificationHeader(buf[n:], verifHdrSigCount, s.pubKeyBytes, bodySig, metaHdrSig, originSig, reqSig)

return n, nil
return s.writeRequestSignatures(buf, n, body, buf[from:n], apiVersion)
}

func (s *Server) writeInitGetResponseBuffers(respStream grpc.ServerStream, id, sig, hdr iprotobuf.BuffersSlice, signResponse bool) error {
Expand Down
Loading
Loading