|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "errors" |
| 6 | + "os" |
| 7 | + "path/filepath" |
| 8 | + "testing" |
| 9 | + "time" |
| 10 | + |
| 11 | + "github.com/codefly-dev/core/agents/services" |
| 12 | + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" |
| 13 | + "github.com/codefly-dev/core/resources" |
| 14 | + runners "github.com/codefly-dev/core/runners/base" |
| 15 | + "github.com/stretchr/testify/require" |
| 16 | +) |
| 17 | + |
| 18 | +// destroyRuntime is a Runtime holding a real, running native process — the |
| 19 | +// shape `codefly` shutdown reaches when it calls Destroy with no preceding |
| 20 | +// Stop. The process touches a file in cacheLocation every 50ms, so a cache |
| 21 | +// wiped while it was still alive gets repopulated and the ordering is |
| 22 | +// observable rather than assumed. |
| 23 | +func destroyRuntime(t *testing.T) (*Runtime, runners.Proc, string) { |
| 24 | + t.Helper() |
| 25 | + ctx := context.Background() |
| 26 | + |
| 27 | + runtime := NewRuntime(NewService()) |
| 28 | + runtime.Location = t.TempDir() |
| 29 | + runtime.Logger = runtime.Wool |
| 30 | + runtime.Base.Runtime.RuntimeContext = resources.NewRuntimeContextNative() |
| 31 | + |
| 32 | + cache := t.TempDir() |
| 33 | + require.NoError(t, os.WriteFile(filepath.Join(cache, "uv.json"), []byte("{}"), 0o600)) |
| 34 | + runtime.cacheLocation = cache |
| 35 | + |
| 36 | + env, err := runners.NewNativeEnvironment(ctx, t.TempDir()) |
| 37 | + require.NoError(t, err) |
| 38 | + proc, err := env.NewProcess("sh", "-c", |
| 39 | + "while true; do touch "+filepath.Join(cache, "heartbeat")+"; sleep 0.05; done") |
| 40 | + require.NoError(t, err) |
| 41 | + require.NoError(t, proc.Start(ctx)) |
| 42 | + t.Cleanup(func() { _ = proc.Stop(context.Background()) }) |
| 43 | + |
| 44 | + runtime.runner = &runnerHandle{proc: proc, cancel: func() {}} |
| 45 | + runtime.runnerEnvironment = env |
| 46 | + return runtime, proc, cache |
| 47 | +} |
| 48 | + |
| 49 | +func TestDestroyEndsExecutionBeforeClearingCache(t *testing.T) { |
| 50 | + ctx := context.Background() |
| 51 | + runtime, proc, cache := destroyRuntime(t) |
| 52 | + |
| 53 | + resp, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 54 | + require.NoError(t, err) |
| 55 | + require.Equal(t, runtimev0.DestroyStatus_SUCCESS, resp.GetStatus().GetState(), resp.GetStatus().GetMessage()) |
| 56 | + |
| 57 | + alive, err := proc.IsRunning(ctx) |
| 58 | + require.NoError(t, err) |
| 59 | + require.False(t, alive, "Destroy must stop the service process without a preceding Stop") |
| 60 | + |
| 61 | + require.Nil(t, currentRunner(runtime)) |
| 62 | + require.Nil(t, runtime.runnerEnvironment, "the runner environment must be shut down and released") |
| 63 | + |
| 64 | + entries, err := os.ReadDir(cache) |
| 65 | + require.NoError(t, err) |
| 66 | + require.Empty(t, entries, "the owned cache must be cleared") |
| 67 | + |
| 68 | + // Several heartbeat periods: a process that outlived the wipe repopulates. |
| 69 | + time.Sleep(500 * time.Millisecond) |
| 70 | + entries, err = os.ReadDir(cache) |
| 71 | + require.NoError(t, err) |
| 72 | + require.Empty(t, entries, "the cache must be cleared after execution ended, not before") |
| 73 | +} |
| 74 | + |
| 75 | +func TestDestroyStopsWatcher(t *testing.T) { |
| 76 | + ctx := context.Background() |
| 77 | + runtime, _, _ := destroyRuntime(t) |
| 78 | + require.NoError(t, os.MkdirAll(filepath.Join(runtime.Location, "code/src"), 0o700)) |
| 79 | + |
| 80 | + require.NoError(t, runtime.SetupWatcher(ctx, services.NewWatchConfiguration(requirements), runtime.EventHandler)) |
| 81 | + events := runtime.Events |
| 82 | + |
| 83 | + _, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 84 | + require.NoError(t, err) |
| 85 | + |
| 86 | + select { |
| 87 | + case _, open := <-events: |
| 88 | + require.False(t, open, "the watcher must be stopped, not still delivering events") |
| 89 | + case <-time.After(5 * time.Second): |
| 90 | + t.Fatal("watcher still running after Destroy") |
| 91 | + } |
| 92 | +} |
| 93 | + |
| 94 | +func TestDestroyReportsCacheFailureAfterEndingExecution(t *testing.T) { |
| 95 | + if os.Geteuid() == 0 { |
| 96 | + t.Skip("root ignores directory permissions, so the wipe cannot be made to fail") |
| 97 | + } |
| 98 | + ctx := context.Background() |
| 99 | + runtime, proc, cache := destroyRuntime(t) |
| 100 | + |
| 101 | + // Removing an entry needs write permission on its parent, so the wipe fails |
| 102 | + // while every other resource stays reachable. |
| 103 | + require.NoError(t, os.Chmod(cache, 0o500)) |
| 104 | + t.Cleanup(func() { _ = os.Chmod(cache, 0o700) }) |
| 105 | + |
| 106 | + resp, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 107 | + require.NoError(t, err) |
| 108 | + require.Equal(t, runtimev0.DestroyStatus_ERROR, resp.GetStatus().GetState(), |
| 109 | + "a failed cache wipe must be observable, not merely logged") |
| 110 | + require.Contains(t, resp.GetStatus().GetMessage(), "cannot remove cache") |
| 111 | + |
| 112 | + alive, err := proc.IsRunning(ctx) |
| 113 | + require.NoError(t, err) |
| 114 | + require.False(t, alive, "execution cleanup must complete even when the cache wipe fails") |
| 115 | + require.Nil(t, currentRunner(runtime)) |
| 116 | + require.Nil(t, runtime.runnerEnvironment) |
| 117 | +} |
| 118 | + |
| 119 | +// stubbornProc fails every Stop, standing in for a process the agent cannot |
| 120 | +// reap. Only Stop is ever reached on this path. |
| 121 | +type stubbornProc struct{ runners.Proc } |
| 122 | + |
| 123 | +func (stubbornProc) Stop(context.Context) error { return errors.New("process will not die") } |
| 124 | + |
| 125 | +func TestDestroyReportsRunnerFailureAndStillClearsCache(t *testing.T) { |
| 126 | + ctx := context.Background() |
| 127 | + runtime, _, cache := destroyRuntime(t) |
| 128 | + runtime.runner = &runnerHandle{proc: stubbornProc{}, cancel: func() {}} |
| 129 | + |
| 130 | + resp, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 131 | + require.NoError(t, err) |
| 132 | + require.Equal(t, runtimev0.DestroyStatus_ERROR, resp.GetStatus().GetState(), |
| 133 | + "Destroy must not report success after only logging a failed process stop") |
| 134 | + require.Contains(t, resp.GetStatus().GetMessage(), "cannot stop the fastapi app") |
| 135 | + |
| 136 | + require.Nil(t, runtime.runnerEnvironment, "a failed stop must not strand the remaining cleanup") |
| 137 | + entries, err := os.ReadDir(cache) |
| 138 | + require.NoError(t, err) |
| 139 | + require.Empty(t, entries) |
| 140 | +} |
| 141 | + |
| 142 | +func TestDestroyIsIdempotent(t *testing.T) { |
| 143 | + ctx := context.Background() |
| 144 | + |
| 145 | + // Before Init: no runner, no environment, no cache location claimed. |
| 146 | + fresh := NewRuntime(NewService()) |
| 147 | + fresh.Location = t.TempDir() |
| 148 | + resp, err := fresh.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 149 | + require.NoError(t, err) |
| 150 | + require.Equal(t, runtimev0.DestroyStatus_SUCCESS, resp.GetStatus().GetState(), resp.GetStatus().GetMessage()) |
| 151 | + |
| 152 | + runtime, _, _ := destroyRuntime(t) |
| 153 | + for range 3 { |
| 154 | + resp, err = runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 155 | + require.NoError(t, err) |
| 156 | + require.Equal(t, runtimev0.DestroyStatus_SUCCESS, resp.GetStatus().GetState(), resp.GetStatus().GetMessage()) |
| 157 | + } |
| 158 | + |
| 159 | + // After Stop, Destroy still succeeds and has nothing left to release. |
| 160 | + stopped, _, _ := destroyRuntime(t) |
| 161 | + stop, err := stopped.Stop(ctx, &runtimev0.StopRequest{}) |
| 162 | + require.NoError(t, err) |
| 163 | + require.Equal(t, runtimev0.StopStatus_SUCCESS, stop.GetStatus().GetState(), stop.GetStatus().GetMessage()) |
| 164 | + resp, err = stopped.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 165 | + require.NoError(t, err) |
| 166 | + require.Equal(t, runtimev0.DestroyStatus_SUCCESS, resp.GetStatus().GetState(), resp.GetStatus().GetMessage()) |
| 167 | +} |
| 168 | + |
| 169 | +// TestStartAfterDestroyCannotLeaveAReplacement covers the race `codefly` |
| 170 | +// shutdown actually runs: a Start already in flight lands after Destroy has |
| 171 | +// released the mutex. Destroy releases the execution environment, so that Start |
| 172 | +// has nothing to launch into and fails loudly instead of leaving a uvicorn |
| 173 | +// running against a wiped cache. |
| 174 | +func TestStartAfterDestroyCannotLeaveAReplacement(t *testing.T) { |
| 175 | + ctx := context.Background() |
| 176 | + runtime, _, _ := destroyRuntime(t) |
| 177 | + |
| 178 | + _, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 179 | + require.NoError(t, err) |
| 180 | + |
| 181 | + start, err := runtime.Start(ctx, &runtimev0.StartRequest{}) |
| 182 | + require.NoError(t, err) |
| 183 | + require.Equal(t, runtimev0.StartStatus_ERROR, start.GetStatus().GetState()) |
| 184 | + require.Nil(t, currentRunner(runtime), "no process may survive Destroy") |
| 185 | +} |
| 186 | + |
| 187 | +// TestDestroyStopsRealUvicornWithoutStop is the acceptance case against the |
| 188 | +// actual server: a listener that answers requests is gone before Destroy |
| 189 | +// reports success, with no Stop anywhere in the path. |
| 190 | +func TestDestroyStopsRealUvicornWithoutStop(t *testing.T) { |
| 191 | + ctx := context.Background() |
| 192 | + source := uvProject(t, servingApp) |
| 193 | + runtime := uvicornRuntime(t, source, false) |
| 194 | + runtime.cacheLocation = t.TempDir() |
| 195 | + |
| 196 | + _, err := runtime.Start(ctx, &runtimev0.StartRequest{}) |
| 197 | + require.NoError(t, err) |
| 198 | + require.Eventually(t, func() bool { return serves(runtime.port) }, 30*time.Second, 200*time.Millisecond, |
| 199 | + "the fixture must be serving before Destroy means anything") |
| 200 | + |
| 201 | + handle := currentRunner(runtime) |
| 202 | + resp, err := runtime.Destroy(ctx, &runtimev0.DestroyRequest{}) |
| 203 | + require.NoError(t, err) |
| 204 | + require.Equal(t, runtimev0.DestroyStatus_SUCCESS, resp.GetStatus().GetState(), resp.GetStatus().GetMessage()) |
| 205 | + |
| 206 | + running, err := handle.proc.IsRunning(ctx) |
| 207 | + require.NoError(t, err) |
| 208 | + require.False(t, running, "uvicorn must be gone before Destroy reports success") |
| 209 | + require.False(t, serves(runtime.port), "the listener must be gone before Destroy reports success") |
| 210 | +} |
0 commit comments