Skip to content

Commit

Permalink
low memory mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@sclevine
sclevine committed Feb 28, 2025
1 parent f3d2b45 commit 4d4f3db
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 68 deletions.
91 changes: 62 additions & 29 deletions lib/autoupdate/agent/updater.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import (
"time"

"github.com/gravitational/trace"
"github.com/prometheus/procfs"
"k8s.io/utils/ptr"

"github.com/gravitational/teleport/api/client/webclient"
"github.com/gravitational/teleport/api/constants"
Expand All @@ -54,7 +56,9 @@ const (
// reservedFreeDisk is the minimum required free space left on disk during downloads.
// TODO(sclevine): This value is arbitrary and could be replaced by, e.g., min(1%, 200mb) in the future
// to account for a range of disk sizes.
reservedFreeDisk = 10_000_000
reservedFreeDisk = 10_000_000 // 10 MB
// lowMemoryLimit is the minimum required available memory for teleport-update to exec Go binaries.
lowMemoryLimit = 1024 * 1024 * 1024 // 1 GiB
// debugSocketFileName is the name of Teleport's debug socket in the data dir.
debugSocketFileName = "debug.sock" // 10 MB
)
Expand All @@ -67,11 +71,14 @@ const (
errorKey = "error"
)

// validatorFunc validates a binary.
type validatorFunc func(ctx context.Context, path string) (bool, error)

// NewLocalUpdater returns a new Updater that auto-updates local
// installations of the Teleport agent.
// The AutoUpdater uses an HTTP client with sane defaults for downloads, and
// will not fill disk to within 10 MB of available capacity.
func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) {
func NewLocalUpdater(ctx context.Context, cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) {
certPool, err := x509.SystemCertPool()
if err != nil {
return nil, trace.Wrap(err)
Expand All @@ -95,8 +102,17 @@ func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) {
cfg.SystemDir = packageSystemDir
}
validator := Validator{Log: cfg.Log}
validate := validator.IsValidBinary
lowMem, err := hasLowMemory()
if err != nil {
cfg.Log.WarnContext(ctx, "Unable to determine if memory usage will exceed limits.", errorKey, err)
}
if lowMem {
cfg.Log.WarnContext(ctx, "Running in low memory mode. Additional safety checks disabled, and configuration may drift.", errorKey, err)
validate = validator.IsBinary
}
debugClient := debug.NewClient(filepath.Join(ns.dataDir, debugSocketFileName))
return &Updater{
u := &Updater{
Log: cfg.Log,
Pool: certPool,
InsecureSkipVerify: cfg.InsecureSkipVerify,
Expand All @@ -114,7 +130,7 @@ func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) {
ReservedFreeTmpDisk: reservedFreeDisk,
ReservedFreeInstallDisk: reservedFreeDisk,
TransformService: ns.ReplaceTeleportService,
ValidateBinary: validator.IsBinary,
ValidateBinary: validate,
Template: autoupdate.DefaultCDNURITemplate,
},
Process: &SystemdService{
Expand All @@ -123,34 +139,51 @@ func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) {
Ready: debugClient,
Log: cfg.Log,
},
ReexecSetup: func(ctx context.Context, pathDir string, reload bool) error {
name := filepath.Join(pathDir, BinaryName)
if cfg.SelfSetup && runtime.GOOS == constants.LinuxOS {
name = "/proc/self/exe"
}
args := []string{
"--install-dir", ns.installDir,
"--install-suffix", ns.name,
"--log-format", cfg.LogFormat,
}
if cfg.Debug {
args = append(args, "--debug")
}
args = append(args, "setup", "--path", pathDir)
if reload {
args = append(args, "--reload")
}
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
cfg.Log.InfoContext(ctx, "Executing new teleport-update binary to update configuration.")
defer cfg.Log.InfoContext(ctx, "Finished executing new teleport-update binary.")
return trace.Wrap(cmd.Run())
},
SetupNamespace: ns.Setup,
TeardownNamespace: ns.Teardown,
LogConfigWarnings: ns.LogWarnings,
}, nil
}
u.ReexecSetup = func(ctx context.Context, pathDir string, reload bool) error {
if lowMem {
return trace.Wrap(u.Setup(ctx, pathDir, reload))
}
name := filepath.Join(pathDir, BinaryName)
if cfg.SelfSetup && runtime.GOOS == constants.LinuxOS {
name = "/proc/self/exe"
}
args := []string{
"--install-dir", ns.installDir,
"--install-suffix", ns.name,
"--log-format", cfg.LogFormat,
}
if cfg.Debug {
args = append(args, "--debug")
}
args = append(args, "setup", "--path", pathDir)
if reload {
args = append(args, "--reload")
}
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
cfg.Log.InfoContext(ctx, "Executing new teleport-update binary to update configuration.")
defer cfg.Log.InfoContext(ctx, "Finished executing new teleport-update binary.")
return trace.Wrap(cmd.Run())
}
return u, nil
}

// hasLowMemory returns true if the system cannot support the full update workflow.
func hasLowMemory() (bool, error) {
pfs, err := procfs.NewDefaultFS()
if err != nil {
return false, trace.Wrap(err)
}
meminfo, err := pfs.Meminfo()
if err != nil {
return false, trace.Wrap(err)
}
return ptr.Deref(meminfo.MemAvailableBytes, lowMemoryLimit) < lowMemoryLimit, nil
}

// LocalUpdaterConfig specifies configuration for managing local agent auto-updates.
Expand Down
31 changes: 14 additions & 17 deletions lib/autoupdate/agent/updater_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ func TestUpdater_Disable(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)
updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -176,8 +177,8 @@ func TestUpdater_Unpin(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)

updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -705,8 +706,8 @@ func TestUpdater_Update(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)

updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -794,7 +795,6 @@ func TestUpdater_Update(t *testing.T) {
return nil
}

ctx := context.Background()
err = updater.Update(ctx, tt.now)
if tt.errMatch != "" {
require.Error(t, err)
Expand Down Expand Up @@ -961,8 +961,8 @@ func TestUpdater_LinkPackage(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)

updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -993,7 +993,6 @@ func TestUpdater_LinkPackage(t *testing.T) {
},
}

ctx := context.Background()
err = updater.LinkPackage(ctx)
if tt.errMatch != "" {
require.Error(t, err)
Expand Down Expand Up @@ -1226,8 +1225,8 @@ func TestUpdater_Remove(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)

updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -1282,7 +1281,6 @@ func TestUpdater_Remove(t *testing.T) {
return nil
}

ctx := context.Background()
err = updater.Remove(ctx, tt.force)
if tt.errMatch != "" {
require.Error(t, err)
Expand Down Expand Up @@ -1584,8 +1582,8 @@ func TestUpdater_Install(t *testing.T) {
_, err := ns.Init()
require.NoError(t, err)
cfgPath := filepath.Join(ns.Dir(), updateConfigName)

updater, err := NewLocalUpdater(LocalUpdaterConfig{
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{
InsecureSkipVerify: true,
}, ns)
require.NoError(t, err)
Expand Down Expand Up @@ -1683,7 +1681,6 @@ func TestUpdater_Install(t *testing.T) {
return nil
}

ctx := context.Background()
err = updater.Install(ctx, tt.userCfg)
if tt.errMatch != "" {
require.Error(t, err)
Expand Down Expand Up @@ -1857,7 +1854,8 @@ func TestUpdater_Setup(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ns := &Namespace{}
updater, err := NewLocalUpdater(LocalUpdaterConfig{}, ns)
ctx := context.Background()
updater, err := NewLocalUpdater(ctx, LocalUpdaterConfig{}, ns)
require.NoError(t, err)

updater.Process = &testProcess{
Expand All @@ -1873,7 +1871,6 @@ func TestUpdater_Setup(t *testing.T) {
return tt.setupErr
}

ctx := context.Background()
err = updater.Setup(ctx, "test", tt.restart)
if tt.errMatch != "" {
require.Error(t, err)
Expand Down
42 changes: 25 additions & 17 deletions lib/autoupdate/agent/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,37 @@ type Validator struct {
Log *slog.Logger
}

// IsBinary returns true for working binaries that are executable by all users.
// If the file is irregular, non-executable, or a shell script, IsBinary returns false and logs a warning.
// IsBinary errors if lstat fails, a regular file is unreadable, or an executable file does not execute.
// IsValidBinary returns true for working binaries that are executable by all users.
// If the file is irregular, non-executable, or a shell script, IsValidBinary returns false and logs a warning.
// IsValidBinary errors if lstat fails, a regular file is unreadable, or an executable file does not execute.
func (v *Validator) IsValidBinary(ctx context.Context, path string) (bool, error) {
if exec, err := v.IsBinary(ctx, path); err != nil || !exec {
return exec, trace.Wrap(err)
}
name := filepath.Base(path)
v.Log.InfoContext(ctx, "Validating binary", "name", name)
r := localExec{
Log: v.Log,
ErrLevel: slog.LevelDebug,
OutLevel: slog.LevelInfo, // always show version
}
code, err := r.Run(ctx, path, "version")
if code < 0 {
return false, trace.Wrap(err, "error validating binary %s", name)
}
if code > 0 {
v.Log.InfoContext(ctx, "Binary does not support version command", "name", name)
}
return true, nil
}

// IsBinary returns true if IsExecutable returns true and the path is not a shellscript.
func (v *Validator) IsBinary(ctx context.Context, path string) (bool, error) {
// The behavior of this method is intended to protect against executable files
// being adding to the Teleport tgz that should not be made available on PATH,
// and additionally, should not cause installation to fail.
// While known copies of these files (e.g., "install") are excluded during extraction,
// it is safer to assume others could be present in past or future tgzs.

if exec, err := v.IsExecutable(ctx, path); err != nil || !exec {
return exec, trace.Wrap(err)
}
Expand All @@ -65,19 +86,6 @@ func (v *Validator) IsBinary(ctx context.Context, path string) (bool, error) {
v.Log.WarnContext(ctx, "Found unexpected shell script", "name", name)
return false, nil
}
v.Log.InfoContext(ctx, "Validating binary", "name", name)
r := localExec{
Log: v.Log,
ErrLevel: slog.LevelDebug,
OutLevel: slog.LevelInfo, // always show version
}
code, err := r.Run(ctx, path, "version")
if code < 0 {
return false, trace.Wrap(err, "error validating binary %s", name)
}
if code > 0 {
v.Log.InfoContext(ctx, "Binary does not support version command", "name", name)
}
return true, nil
}

Expand Down
4 changes: 2 additions & 2 deletions lib/autoupdate/agent/validate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"github.com/stretchr/testify/require"
)

func TestValidator_IsBinary(t *testing.T) {
func TestValidator_IsValidBinary(t *testing.T) {
for _, tt := range []struct {
name string
mode os.FileMode
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestValidator_IsBinary(t *testing.T) {
if tt.contents != "" {
os.WriteFile(path, []byte(tt.contents), tt.mode)
}
val, err := v.IsBinary(ctx, path)
val, err := v.IsValidBinary(ctx, path)
if tt.logMatch != "" {
require.Contains(t, buf.String(), tt.logMatch)
}
Expand Down
6 changes: 3 additions & 3 deletions tool/teleport-update/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ func initConfig(ctx context.Context, ccfg *cliConfig) (updater *autoupdate.Updat
if err != nil {
return nil, "", trace.Wrap(err)
}
updater, err = autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{
updater, err = autoupdate.NewLocalUpdater(ctx, autoupdate.LocalUpdaterConfig{
SelfSetup: ccfg.SelfSetup,
Log: plog,
LogFormat: ccfg.LogFormat,
Expand All @@ -261,7 +261,7 @@ func statusConfig(ctx context.Context, ccfg *cliConfig) (*autoupdate.Updater, er
if err != nil {
return nil, trace.Wrap(err)
}
updater, err := autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{
updater, err := autoupdate.NewLocalUpdater(ctx, autoupdate.LocalUpdaterConfig{
SelfSetup: ccfg.SelfSetup,
Log: plog,
LogFormat: ccfg.LogFormat,
Expand Down Expand Up @@ -426,7 +426,7 @@ func cmdSetup(ctx context.Context, ccfg *cliConfig) error {
if err != nil {
return trace.Wrap(err)
}
updater, err := autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{
updater, err := autoupdate.NewLocalUpdater(ctx, autoupdate.LocalUpdaterConfig{
SelfSetup: ccfg.SelfSetup,
Log: plog,
LogFormat: ccfg.LogFormat,
Expand Down

0 comments on commit 4d4f3db

Please sign in to comment.