WIP: oci: buildkit TMPDIR fallback

Author: dtrudg

cmd/singularity-buildkitd/main.go

@@ -11,6 +11,8 @@ import (
 
 	bkdaemon "github.com/sylabs/singularity/v4/internal/pkg/build/buildkit/daemon"
 	"github.com/sylabs/singularity/v4/internal/pkg/buildcfg"
+	fsoverlay "github.com/sylabs/singularity/v4/internal/pkg/util/fs/overlay"
+	"github.com/sylabs/singularity/v4/pkg/syfs"
 	"github.com/sylabs/singularity/v4/pkg/sylog"
 	"github.com/sylabs/singularity/v4/pkg/util/singularityconf"
 )
@@ -33,10 +35,37 @@ func main() {
 	}
 	singularityconf.SetCurrentConfig(config)
 
+	if err := run(bkSocket, bkArch); err != nil {
+		sylog.Fatalf("%s: %v", bkdaemon.DaemonName, err)
+	}
+}
+
+func run(bkSocket, bkArch string) error {
 	daemonOpts := &bkdaemon.Opts{
 		ReqArch: bkArch,
 	}
-	if err := bkdaemon.Run(context.Background(), daemonOpts, bkSocket); err != nil {
-		sylog.Fatalf("%s: %v", bkdaemon.DaemonName, err)
+
+	// Check the user .singularity dir is in a location supporting overlayfs etc. If not, use a tmpdir.
+	if err := fsoverlay.CheckUpper(syfs.ConfigDir()); err != nil {
+		tmpRoot, err := os.MkdirTemp("", bkdaemon.DaemonName)
+		if err != nil {
+			sylog.Fatalf("%s: while creating temporary root dir: %v", bkdaemon.DaemonName, err)
+		}
+		defer func() {
+			sylog.Warningf("%s: removing temporary directory %s", bkdaemon.DaemonName, tmpRoot)
+			err := os.RemoveAll(tmpRoot)
+			if err != nil {
+				sylog.Errorf("%s: while removing temp dir: %v", bkdaemon.DaemonName, err)
+			}
+		}()
+
+		if err := fsoverlay.CheckUpper(tmpRoot); err != nil {
+			sylog.Fatalf("%s: temporary directory does not support buildkit. Please set $TMPDIR to a local filesystem.", bkdaemon.DaemonName)
+		}
+
+		sylog.Warningf("%s: ~/.singularity filesystem does not support buildkit. Using temporary directory %s. Layers will not be cached for future builds.", bkdaemon.DaemonName, tmpRoot)
+		daemonOpts.TempRootDir = tmpRoot
 	}
+
+	return bkdaemon.Run(context.Background(), daemonOpts, bkSocket)
 }

internal/pkg/build/buildkit/client/client.go

@@ -54,7 +54,7 @@ import (
 const (
 	buildTag          = "tag"
 	bkDefaultSocket   = "unix:///run/buildkit/buildkitd.sock"
-	bkLaunchTimeout   = 120 * time.Second
+	bkLaunchTimeout   = 10 * time.Second
 	bkShutdownTimeout = 10 * time.Second
 	bkMinVersion      = "v0.12.3"
 )

internal/pkg/build/buildkit/daemon/daemon.go

@@ -80,6 +80,8 @@ const DaemonName = "singularity-buildkitd"
 type Opts struct {
 	// Requested build architecture
 	ReqArch string
+	// Override the location of the singularity-buildkitd root with a temporary directory (no GC)
+	TempRootDir string
 }
 
 type workerInitializerOpt struct {
@@ -149,7 +151,7 @@ func waitLock(ctx context.Context, lockPath string) (*flock.Flock, error) {
 func Run(ctx context.Context, opts *Opts, socketPath string) error {
 	// If we need to, enter a new cgroup now, to workaround an issue with crun container cgroup creation (#1538).
 	if err := oci.CrunNestCgroup(); err != nil {
-		sylog.Fatalf("%s: while applying crun cgroup workaround: %v", DaemonName, err)
+		return fmt.Errorf("%s: while applying crun cgroup workaround: %v", DaemonName, err)
 	}
 
 	cfg, err := config.LoadFile(defaultConfigPath())
@@ -166,6 +168,14 @@ func Run(ctx context.Context, opts *Opts, socketPath string) error {
 
 	server := grpc.NewServer()
 
+	if opts.TempRootDir != "" {
+		ptr := func(v bool) *bool {
+			return &v
+		}
+		cfg.Root = opts.TempRootDir
+		cfg.Workers.OCI.GC = ptr(false)
+	}
+
 	// relative path does not work with nightlyone/lockfile
 	root, err := filepath.Abs(cfg.Root)
 	if err != nil {
@@ -181,7 +191,7 @@ func Run(ctx context.Context, opts *Opts, socketPath string) error {
 	sylog.Debugf("%s: path for buildkitd lock file: %s", DaemonName, lockPath)
 	lock, err := waitLock(ctx, lockPath)
 	if err != nil {
-		sylog.Fatalf("%s: while creating lock file: %v", DaemonName, err)
+		return fmt.Errorf("%s: while creating lock file: %v", DaemonName, err)
 	}
 	defer func() {
 		lock.Unlock()