Skip to content

guest: unify pod model for V1, virtual pod, and V2 shim support #2662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shreyanshjain7174 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

guest: unify pod model for V1, virtual pod, and V2 shim support #2662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e2b46c
guest: use OCIBundlePath as sandbox root source of truth
Mar 31, 2026
18fb5d8
guest: unify pod model for V1, virtual pod, and V2 shim support
Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 11 additions & 16 deletions cmd/gcs/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,17 +362,16 @@ func main() {
}
defer containersControl.Delete() //nolint:errcheck

// Create virtual-pods cgroup hierarchy for multi-pod support
// This will be the parent for all virtual pod cgroups: /containers/virtual-pods/{virtualSandboxID}
virtualPodsControl, err := cgroup.NewManager("/containers/virtual-pods", &oci.LinuxResources{
// Create /pods cgroup hierarchy for all pods (sandbox, virtual pod, and v2).
podsControl, err := cgroup.NewManager("/pods", &oci.LinuxResources{
Memory: &oci.LinuxMemory{
Limit: &containersLimit, // Share the same limit as containers
Limit: &containersLimit,
},
})
if err != nil {
logrus.WithError(err).Fatal("failed to create containers/virtual-pods cgroup")
logrus.WithError(err).Fatal("failed to create pods cgroup")
}
defer virtualPodsControl.Delete() //nolint:errcheck
defer podsControl.Delete() //nolint:errcheck

gcsControl, err := cgroup.NewManager("/gcs", &oci.LinuxResources{})
if err != nil {
Expand All @@ -394,10 +393,6 @@ func main() {
EnableV4: *v4,
}
h := hcsv2.NewHost(rtime, tport, initialEnforcer, logWriter)
// Initialize virtual pod support in the host
if err := h.InitializeVirtualPodSupport(virtualPodsControl); err != nil {
logrus.WithError(err).Warn("Virtual pod support initialization failed")
}
b.AssignHandlers(mux, h)

var bridgeIn io.ReadCloser
Expand Down Expand Up @@ -433,13 +428,13 @@ func main() {
oomFile := os.NewFile(oom, "cefd")
defer oomFile.Close()

// Setup OOM monitoring for virtual-pods cgroup
virtualPodsOom, err := virtualPodsControl.OOMEventFD()
// Setup OOM monitoring for pods cgroup
podsOom, err := podsControl.OOMEventFD()
if err != nil {
logrus.WithError(err).Fatal("failed to retrieve the virtual-pods cgroups oom eventfd")
logrus.WithError(err).Fatal("failed to retrieve the pods cgroups oom eventfd")
}
virtualPodsOomFile := os.NewFile(virtualPodsOom, "vp-oomfd")
defer virtualPodsOomFile.Close()
podsOomFile := os.NewFile(podsOom, "pods-oomfd")
defer podsOomFile.Close()

// time synchronization service
if !(*disableTimeSync) {
Expand All @@ -450,7 +445,7 @@ func main() {

go readMemoryEvents(startTime, gefdFile, "/gcs", int64(*gcsMemLimitBytes), gcsControl)
go readMemoryEvents(startTime, oomFile, "/containers", containersLimit, containersControl)
go readMemoryEvents(startTime, virtualPodsOomFile, "/containers/virtual-pods", containersLimit, virtualPodsControl)
go readMemoryEvents(startTime, podsOomFile, "/pods", containersLimit, podsControl)
err = b.ListenAndServe(bridgeIn, bridgeOut)
if err != nil {
logrus.WithFields(logrus.Fields{
Expand Down
26 changes: 12 additions & 14 deletions internal/guest/runtime/hcsv2/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/Microsoft/hcsshim/internal/oc"
"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
"github.com/Microsoft/hcsshim/pkg/annotations"
)

// containerStatus has been introduced to enable parallel container creation
Expand All @@ -53,6 +52,7 @@ type Container struct {

spec *oci.Spec
ociBundlePath string
sandboxID string // ID of the sandbox/pod this container belongs to
isSandbox bool

container runtime.Container
Expand All @@ -76,6 +76,10 @@ type Container struct {
// of this container is located. Usually, this is either `/run/gcs/c/<containerID>` or
// `/run/gcs/c/<UVMID>/container_<containerID>` if scratch is shared with UVM scratch.
scratchDirPath string

// sandboxRoot is the root directory of the pod within the guest.
// Used during cleanup to unmount sandbox-specific paths.
sandboxRoot string
}

func (c *Container) Start(ctx context.Context, conSettings stdio.ConnectionSettings) (_ int, err error) {
Expand Down Expand Up @@ -228,25 +232,19 @@ func (c *Container) Kill(ctx context.Context, signal syscall.Signal) error {
func (c *Container) Delete(ctx context.Context) error {
entity := log.G(ctx).WithField(logfields.ContainerID, c.id)
entity.Info("opengcs::Container::Delete")
if c.isSandbox {
// Check if this is a virtual pod
virtualSandboxID := ""
if c.spec != nil && c.spec.Annotations != nil {
virtualSandboxID = c.spec.Annotations[annotations.VirtualPodID]
}

// remove user mounts in sandbox container - use virtual pod aware paths
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareSandboxMountsDir(c.id, virtualSandboxID), true); err != nil {
if c.isSandbox && c.sandboxRoot != "" {
// remove user mounts in sandbox container
if err := storage.UnmountAllInPath(ctx, specGuest.SandboxMountsDirFromRoot(c.sandboxRoot), true); err != nil {
entity.WithError(err).Error("failed to unmount sandbox mounts")
}

// remove user mounts in tmpfs sandbox container - use virtual pod aware paths
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareSandboxTmpfsMountsDir(c.id, virtualSandboxID), true); err != nil {
// remove tmpfs mounts in sandbox container
if err := storage.UnmountAllInPath(ctx, specGuest.SandboxTmpfsMountsDirFromRoot(c.sandboxRoot), true); err != nil {
entity.WithError(err).Error("failed to unmount tmpfs sandbox mounts")
}

// remove hugepages mounts in sandbox container - use virtual pod aware paths
if err := storage.UnmountAllInPath(ctx, specGuest.VirtualPodAwareHugePagesMountsDir(c.id, virtualSandboxID), true); err != nil {
// remove hugepages mounts in sandbox container
if err := storage.UnmountAllInPath(ctx, specGuest.SandboxHugePagesMountsDirFromRoot(c.sandboxRoot), true); err != nil {
entity.WithError(err).Error("failed to unmount hugepages mounts")
}
}
Expand Down
13 changes: 0 additions & 13 deletions internal/guest/runtime/hcsv2/container_stats_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,16 +494,3 @@ func TestConvertV2StatsToV1_NilInput(t *testing.T) {
t.Error("ConvertV2StatsToV1(nil) should return empty metrics with all nil fields")
}
}

func TestHost_InitializeVirtualPodSupport_ErrorCases(t *testing.T) {
host := &Host{}

// Test with nil input
err := host.InitializeVirtualPodSupport(nil)
if err == nil {
t.Error("Expected error for nil input")
}
if err != nil && err.Error() != "no valid cgroup manager provided for virtual pod support" {
t.Errorf("Unexpected error message: %s", err.Error())
}
}
42 changes: 16 additions & 26 deletions internal/guest/runtime/hcsv2/sandbox_container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,35 +20,30 @@ import (
"github.com/Microsoft/hcsshim/pkg/annotations"
)

func getSandboxHostnamePath(id, virtualSandboxID string) string {
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "hostname")
func getSandboxHostnamePath(sandboxRoot string) string {
return filepath.Join(sandboxRoot, "hostname")
}

func getSandboxHostsPath(id, virtualSandboxID string) string {
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "hosts")
func getSandboxHostsPath(sandboxRoot string) string {
return filepath.Join(sandboxRoot, "hosts")
}

func getSandboxResolvPath(id, virtualSandboxID string) string {
return filepath.Join(specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID), "resolv.conf")
func getSandboxResolvPath(sandboxRoot string) string {
return filepath.Join(sandboxRoot, "resolv.conf")
}

func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (err error) {
func setupSandboxContainerSpec(ctx context.Context, id, sandboxRoot string, spec *oci.Spec) (err error) {
ctx, span := oc.StartSpan(ctx, "hcsv2::setupSandboxContainerSpec")
defer span.End()
defer func() { oc.SetSpanStatus(span, err) }()
span.AddAttributes(trace.StringAttribute("cid", id))

// Check if this is a virtual pod to use appropriate root directory
virtualSandboxID := spec.Annotations[annotations.VirtualPodID]

// Generate the sandbox root dir - virtual pod aware
rootDir := specGuest.VirtualPodAwareSandboxRootDir(id, virtualSandboxID)
if err := os.MkdirAll(rootDir, 0755); err != nil {
return errors.Wrapf(err, "failed to create sandbox root directory %q", rootDir)
if err := os.MkdirAll(sandboxRoot, 0755); err != nil {
return errors.Wrapf(err, "failed to create sandbox root directory %q", sandboxRoot)
}
defer func() {
if err != nil {
_ = os.RemoveAll(rootDir)
_ = os.RemoveAll(sandboxRoot)
}
}()

Expand All @@ -62,19 +57,20 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
}
}

sandboxHostnamePath := getSandboxHostnamePath(id, virtualSandboxID)
sandboxHostnamePath := getSandboxHostnamePath(sandboxRoot)
if err := os.WriteFile(sandboxHostnamePath, []byte(hostname+"\n"), 0644); err != nil {
return errors.Wrapf(err, "failed to write hostname to %q", sandboxHostnamePath)
}

// Write the hosts
sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
sandboxHostsPath := getSandboxHostsPath(id, virtualSandboxID)
sandboxHostsPath := getSandboxHostsPath(sandboxRoot)
if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil {
return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath)
}

// Check if this is a virtual pod sandbox container by comparing container ID with virtual pod ID
virtualSandboxID := spec.Annotations[annotations.VirtualPodID]
isVirtualPodSandbox := virtualSandboxID != "" && id == virtualSandboxID
if strings.EqualFold(spec.Annotations[annotations.SkipPodNetworking], "true") || isVirtualPodSandbox {
ns := GetOrAddNetworkNamespace(specGuest.GetNetworkNamespaceID(spec))
Expand All @@ -97,7 +93,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
if err != nil {
return errors.Wrap(err, "failed to generate sandbox resolv.conf content")
}
sandboxResolvPath := getSandboxResolvPath(id, virtualSandboxID)
sandboxResolvPath := getSandboxResolvPath(sandboxRoot)
if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil {
return errors.Wrap(err, "failed to write sandbox resolv.conf")
}
Expand All @@ -123,14 +119,8 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
// also has a concept of a sandbox/shm file when the IPC NamespaceMode !=
// NODE.

// Set cgroup path - check if this is a virtual pod
if virtualSandboxID != "" {
// Virtual pod sandbox gets its own cgroup under /containers/virtual-pods using the virtual pod ID
spec.Linux.CgroupsPath = "/containers/virtual-pods/" + virtualSandboxID
} else {
// Traditional sandbox goes under /containers
spec.Linux.CgroupsPath = "/containers/" + id
}
// Set cgroup path under the pod's cgroup.
spec.Linux.CgroupsPath = "/pods/" + id

// Clear the windows section as we dont want to forward to runc
spec.Windows = nil
Expand Down
Loading
Loading