fix(profiler): reach the target through its own mount namespace

internal/agent/profiler/jvm/async_profiler.go

@@ -63,6 +63,7 @@ type AsyncProfilerManager interface {
 	linkTmpDirToTargetTmpDir(string) error
 	copyProfilerToTmpDir() error
 	selectProfilerLibrary(string) error
+	chownProfilerToTarget(string) error
 	invoke(*job.ProfilingJob, string) (error, time.Duration)
 	cleanUp(*job.ProfilingJob, string)
 }
@@ -83,42 +84,47 @@ func NewAsyncProfiler(commander executil.Commander, publisher publish.Publisher)
 }
 
 func (j *AsyncProfiler) SetUp(job *job.ProfilingJob) error {
-	targetFs, err := util.ContainerFileSystem(job.ContainerRuntime, job.ContainerID, job.ContainerRuntimePath)
-	if err != nil {
-		return err
+	// PIDs first: everything below is staged through the target's own mount
+	// namespace, which we can only reach via one of its PIDs.
+	if stringUtils.IsNotBlank(job.PID) {
+		j.targetPIDs = []string{job.PID}
+	} else {
+		pids, err := util.GetCandidatePIDs(job)
+		if err != nil {
+			return err
+		}
+		log.DebugLogLn(fmt.Sprintf("The PIDs to be profiled: %s", pids))
+		j.targetPIDs = pids
 	}
+
+	// Every PID of a container shares its mount namespace, so any of them
+	// resolves the same filesystem.
+	targetFs := util.TargetRootFS(j.targetPIDs[0])
 	log.DebugLogLn(fmt.Sprintf("The target filesystem is: %s", targetFs))
 
-	err = j.removeTmpDir()
-	if err != nil {
+	if err := j.removeTmpDir(); err != nil {
 		return err
 	}
 
 	targetTmpDir := filepath.Join(targetFs, "tmp")
 	// remove previous files from a previous profiling
 	file.RemoveAll(targetTmpDir, config.ProfilingPrefix+string(job.OutputType))
 
-	err = j.linkTmpDirToTargetTmpDir(targetTmpDir)
-	if err != nil {
+	if err := j.linkTmpDirToTargetTmpDir(targetTmpDir); err != nil {
 		return err
 	}
 
-	if stringUtils.IsNotBlank(job.PID) {
-		j.targetPIDs = []string{job.PID}
-	} else {
-		pids, err := util.GetCandidatePIDs(job)
-		if err != nil {
-			return err
-		}
-		log.DebugLogLn(fmt.Sprintf("The PIDs to be profiled: %s", pids))
-		j.targetPIDs = pids
+	if err := j.copyProfilerToTmpDir(); err != nil {
+		return err
 	}
 
-	if err := j.copyProfilerToTmpDir(); err != nil {
+	if err := j.selectProfilerLibrary(targetFs); err != nil {
 		return err
 	}
 
-	return j.selectProfilerLibrary(targetFs)
+	// The JVM dlopens the library and writes the profile itself, as whatever
+	// user it runs as — root-owned staging is unreadable/unwritable for it.
+	return j.chownProfilerToTarget(j.targetPIDs[0])
 }
 
 // targetUsesMusl reports whether the target container's root filesystem is
@@ -151,6 +157,23 @@ func (j *asyncProfilerManager) copyProfilerToTmpDir() error {
 	return cmd.Run()
 }
 
+// chownProfilerToTarget hands the staged directory to the user the target runs
+// as. We stage as root; the JVM then has to read libasyncProfiler.so and write
+// its own output file into that directory, and most hardened images do not run
+// as root.
+func (j *asyncProfilerManager) chownProfilerToTarget(pid string) error {
+	uid, gid, err := util.TargetCredentials(pid)
+	if err != nil {
+		return err
+	}
+	if uid == "0" && gid == "0" {
+		return nil
+	}
+	log.DebugLogLn(fmt.Sprintf("Handing the staged profiler to %s:%s", uid, gid))
+	cmd := j.commander.Command("chown", "-R", uid+":"+gid, asyncProfilerDir)
+	return cmd.Run()
+}
+
 // selectProfilerLibrary points libasyncProfiler.so at the build matching the
 // target's libc. The library is dlopen'd by the target JVM rather than by us,
 // so a mismatch fails the attach with "libc.musl-x86_64.so.1: cannot open

internal/agent/profiler/jvm/async_profiler_fake.go

@@ -128,6 +128,19 @@ func (f *fakeAsyncProfilerManager) selectProfilerLibrary(s string) error {
 	return err
 }
 
+func (f *fakeAsyncProfilerManager) chownProfilerToTarget(s string) error {
+	var err error
+	f.fakeMethods["chownProfilerToTarget"].invokes++
+	if f.fakeMethods["chownProfilerToTarget"].fakeReturnValues != nil && len(f.fakeMethods["chownProfilerToTarget"].fakeReturnValues) > 0 {
+		f.fakeMethods["chownProfilerToTarget"].indexExecution++
+		arg0 := f.fakeMethods["chownProfilerToTarget"].fakeReturnValues[f.fakeMethods["chownProfilerToTarget"].indexExecution-1].([]interface{})[0]
+		if arg0 != nil {
+			err = arg0.(error)
+		}
+	}
+	return err
+}
+
 func (f *fakeAsyncProfilerManager) cleanUp(profilingJob *job.ProfilingJob, s string) {
 	f.fakeMethods["cleanUp"].invokes++
 	if f.fakeMethods["cleanUp"].fakeReturnValues != nil && len(f.fakeMethods["cleanUp"].fakeReturnValues) > 0 {

internal/agent/profiler/jvm/async_profiler_test.go

@@ -43,6 +43,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
 				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
+				asyncProfilerManager.On("chownProfilerToTarget").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -75,6 +76,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
 				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
+				asyncProfilerManager.On("chownProfilerToTarget").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -101,13 +103,14 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 			},
 		},
 		{
-			name: "should fail when getting target filesystem fail",
+			name: "should fail when the container runtime is unknown",
 			given: func() (fields, args) {
 				asyncProfilerManager := newFakeAsyncProfilerManager()
 				asyncProfilerManager.On("removeTmpDir").Return(nil)
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
 				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
+				asyncProfilerManager.On("chownProfilerToTarget").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -118,7 +121,6 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 							Duration:         0,
 							ContainerRuntime: "other",
 							ContainerID:      "ContainerID",
-							PID:              "PID_ContainerID",
 						},
 					}
 			},
@@ -197,6 +199,8 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager := newFakeAsyncProfilerManager()
 				asyncProfilerManager.On("removeTmpDir").Return(nil)
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
+				// PIDs are resolved first now — the target's mount namespace is
+				// reached through one of them, so nothing can be staged before.
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -215,8 +219,8 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 			},
 			then: func(t *testing.T, err error, fields fields) {
 				assert.NotNil(t, err)
-				assert.Equal(t, 1, fields.AsyncProfiler.AsyncProfilerManager.(FakeAsyncProfilerManager).On("removeTmpDir").InvokedTimes())
-				assert.Equal(t, 1, fields.AsyncProfiler.AsyncProfilerManager.(FakeAsyncProfilerManager).On("linkTmpDirToTargetTmpDir").InvokedTimes())
+				assert.Equal(t, 0, fields.AsyncProfiler.AsyncProfilerManager.(FakeAsyncProfilerManager).On("removeTmpDir").InvokedTimes())
+				assert.Equal(t, 0, fields.AsyncProfiler.AsyncProfilerManager.(FakeAsyncProfilerManager).On("linkTmpDirToTargetTmpDir").InvokedTimes())
 				assert.Equal(t, 0, fields.AsyncProfiler.AsyncProfilerManager.(FakeAsyncProfilerManager).On("copyProfilerToTmpDir").InvokedTimes())
 			},
 		},
@@ -495,6 +499,54 @@ func Test_asyncProfilerManager_selectProfilerLibrary(t *testing.T) {
 	})
 }
 
+// Test_asyncProfilerManager_chownProfilerToTarget — we stage as root, but the
+// JVM reads the library and writes its own output into that directory as
+// whatever user it runs as, which in hardened images is not root.
+func Test_asyncProfilerManager_chownProfilerToTarget(t *testing.T) {
+	t.Run("a root target needs no chown", func(t *testing.T) {
+		requireProcFS(t)
+		if os.Getuid() != 0 {
+			t.Skip("this test reads our own /proc entry, so it only says root when we are")
+		}
+		commander := executil.NewFakeCommander()
+		commander.On("Command").Return(exec.Command("false"))
+		a := NewAsyncProfiler(commander, publish.NewFakePublisher())
+
+		assert.Nil(t, a.chownProfilerToTarget("self"))
+		assert.Equal(t, 0, commander.On("Command").InvokedTimes())
+	})
+
+	t.Run("a non-root target is handed the staged directory", func(t *testing.T) {
+		requireProcFS(t)
+		if os.Getuid() == 0 {
+			t.Skip("running as root, so our own /proc entry cannot stand in for a non-root target")
+		}
+		commander := executil.NewFakeCommander()
+		commander.On("Command").Return(exec.Command("true"))
+		a := NewAsyncProfiler(commander, publish.NewFakePublisher())
+
+		assert.Nil(t, a.chownProfilerToTarget("self"))
+		assert.Equal(t, 1, commander.On("Command").InvokedTimes())
+	})
+
+	t.Run("an unreadable pid is an error, not a silent skip", func(t *testing.T) {
+		commander := executil.NewFakeCommander()
+		commander.On("Command").Return(exec.Command("true"))
+		a := NewAsyncProfiler(commander, publish.NewFakePublisher())
+
+		assert.NotNil(t, a.chownProfilerToTarget("not-a-pid"))
+	})
+}
+
+// requireProcFS skips on platforms without /proc — the credentials come from
+// /proc/<pid>/status, which only exists on the Linux boxes this runs on.
+func requireProcFS(t *testing.T) {
+	t.Helper()
+	if _, err := os.Stat("/proc/self/status"); err != nil {
+		t.Skip("no procfs on this platform")
+	}
+}
+
 func Test_asyncProfilerManager_invoke(t *testing.T) {
 	type fields struct {
 		AsyncProfiler *AsyncProfiler

internal/agent/profiler/jvm/jcmd.go

@@ -96,26 +96,8 @@ func NewJcmdProfiler(commander executil.Commander, publisher publish.Publisher)
 }
 
 func (j *JcmdProfiler) SetUp(job *job.ProfilingJob) error {
-	targetFs, err := util.ContainerFileSystem(job.ContainerRuntime, job.ContainerID, job.ContainerRuntimePath)
-	if err != nil {
-		return err
-	}
-	log.DebugLogLn(fmt.Sprintf("The target filesystem is: %s", targetFs))
-
-	err = j.removeTmpDir()
-	if err != nil {
-		return err
-	}
-
-	targetTmpDir := filepath.Join(targetFs, "tmp")
-	// remove previous files from a previous profiling
-	file.RemoveAll(targetTmpDir, config.ProfilingPrefix+string(job.OutputType))
-
-	err = j.linkTmpDirToTargetTmpDir(targetTmpDir)
-	if err != nil {
-		return err
-	}
-
+	// PIDs first: the tmp dir below is the target's own, reachable only
+	// through one of its PIDs.
 	if stringUtils.IsNotBlank(job.PID) {
 		j.targetPIDs = []string{job.PID}
 		recordingPIDs = make(chan string, 1)
@@ -129,6 +111,24 @@ func (j *JcmdProfiler) SetUp(job *job.ProfilingJob) error {
 		recordingPIDs = make(chan string, len(pids))
 	}
 
+	// The JVM writes heap dumps and JFR recordings itself, to a path in its own
+	// mount namespace — so we have to agree with it on what /tmp is. Every PID
+	// of a container shares that namespace.
+	targetFs := util.TargetRootFS(j.targetPIDs[0])
+	log.DebugLogLn(fmt.Sprintf("The target filesystem is: %s", targetFs))
+
+	if err := j.removeTmpDir(); err != nil {
+		return err
+	}
+
+	targetTmpDir := filepath.Join(targetFs, "tmp")
+	// remove previous files from a previous profiling
+	file.RemoveAll(targetTmpDir, config.ProfilingPrefix+string(job.OutputType))
+
+	if err := j.linkTmpDirToTargetTmpDir(targetTmpDir); err != nil {
+		return err
+	}
+
 	return j.copyJfrSettingsToTmpDir()
 }
 

internal/agent/profiler/jvm/jcmd_test.go

@@ -210,8 +210,10 @@ func TestJcmdProfiler_SetUp(t *testing.T) {
 			},
 			then: func(t *testing.T, err error, fields fields) {
 				assert.NotNil(t, err)
-				assert.Equal(t, 1, fields.JcmdProfiler.JcmdManager.(FakeJcmdManager).On("removeTmpDir").InvokedTimes())
-				assert.Equal(t, 1, fields.JcmdProfiler.JcmdManager.(FakeJcmdManager).On("linkTmpDirToTargetTmpDir").InvokedTimes())
+				// PIDs are resolved first now — the target's /tmp is reached
+				// through one of them, so nothing is staged before that.
+				assert.Equal(t, 0, fields.JcmdProfiler.JcmdManager.(FakeJcmdManager).On("removeTmpDir").InvokedTimes())
+				assert.Equal(t, 0, fields.JcmdProfiler.JcmdManager.(FakeJcmdManager).On("linkTmpDirToTargetTmpDir").InvokedTimes())
 				assert.Equal(t, 0, fields.JcmdProfiler.JcmdManager.(FakeJcmdManager).On("copyJfrSettingsToTmpDir").InvokedTimes())
 			},
 		},

internal/agent/profiler/node_dummy.go

@@ -45,10 +45,16 @@ func NewNodeDummyProfiler(publisher publish.Publisher) *NodeDummyProfiler {
 }
 
 func (n *NodeDummyProfiler) SetUp(job *job.ProfilingJob) error {
-	targetFs, err := util.ContainerFileSystem(job.ContainerRuntime, job.ContainerID, job.ContainerRuntimePath)
+	rootPID, err := util.GetRootPID(job)
 	if err != nil {
 		return err
 	}
+
+	// The heapsnapshot is written by the Node process into its working
+	// directory, so we have to read it back from the target's own mount
+	// namespace — the runtime's overlay path misses it whenever that
+	// directory is a volume.
+	targetFs := util.TargetRootFS(rootPID)
 	log.DebugLogLn(fmt.Sprintf("The target filesystem is: %s", targetFs))
 
 	cwd, err := util.GetCWD(job)

internal/agent/profiler/node_dummy_test.go

@@ -53,11 +53,13 @@ func TestNodeDummyProfiler_SetUp(t *testing.T) {
 			},
 			then: func(t *testing.T, err error, fields fields) {
 				assert.Nil(t, err)
-				assert.Equal(t, "/root/fs/ContainerID/cwd", fields.NodeDummyProfiler.cwd)
+				// The heapsnapshot is read back through the target's own mount
+				// namespace, not the runtime's overlay directory.
+				assert.Equal(t, "/proc/PID_ContainerID/root/cwd", fields.NodeDummyProfiler.cwd)
 			},
 		},
 		{
-			name: "should fail when get root file system fail",
+			name: "should fail when the container PID is not found",
 			given: func() (fields, args) {
 				return fields{
 						NodeDummyProfiler: &NodeDummyProfiler{
@@ -66,7 +68,7 @@ func TestNodeDummyProfiler_SetUp(t *testing.T) {
 					}, args{
 						job: &job.ProfilingJob{
 							Duration:         0,
-							ContainerRuntime: api.FakeContainerWithRootFileSystemLocationResultError,
+							ContainerRuntime: api.FakeContainerWithPIDResultError,
 							ContainerID:      "ContainerID",
 						},
 					}