diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5fe88805b49..0900b1963aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,26 @@ on: - cron: '0 0 * * *' # run ci every night jobs: + graphql-resume: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + database: [mysql, postgres] + steps: + - uses: actions/checkout@v7 + - name: GraphQL real database crash and resume + env: + DEVLAKE_CRASH_DRIVER: ${{ matrix.database }} + TMPDIR: ${{ runner.temp }} + run: bash backend/test/graphql-resume-crash.sh + - name: Preserve failure and resume logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: graphql-resume-${{ matrix.database }} + path: ${{ runner.temp }}/devlake-graphql-crash.*/*.log test: strategy: matrix: diff --git a/backend/core/models/graphql_collector_state.go b/backend/core/models/graphql_collector_state.go new file mode 100644 index 00000000000..f079c3bee53 --- /dev/null +++ b/backend/core/models/graphql_collector_state.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "time" + +type GraphqlCollectorState struct { + ScopeHash string `gorm:"primaryKey;type:char(64)"` + InputHash string `gorm:"primaryKey;type:varchar(64)"` + TaskID uint64 + RawDataTable string `gorm:"type:varchar(255)"` + RawDataParams string `gorm:"type:text"` + Completed bool + SkipCursor string `gorm:"type:text"` + UpdatedAt time.Time + Contract string `gorm:"type:varchar(64)"` + InputDigest string `gorm:"type:varchar(64)"` + RawCount *int64 +} + +func (GraphqlCollectorState) TableName() string { + return "_devlake_graphql_collector_states" +} diff --git a/backend/core/models/migrationscripts/20260916_add_graphql_collector_states.go b/backend/core/models/migrationscripts/20260916_add_graphql_collector_states.go new file mode 100644 index 00000000000..f3c661cb23f --- /dev/null +++ b/backend/core/models/migrationscripts/20260916_add_graphql_collector_states.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/devlake/core/context" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" +) + +type graphqlCollectorState20260916 struct { + ScopeHash string `gorm:"primaryKey;type:char(64)"` + InputHash string `gorm:"primaryKey;type:varchar(64)"` + TaskID uint64 + RawDataTable string `gorm:"type:varchar(255)"` + RawDataParams string `gorm:"type:text"` + Completed bool + SkipCursor string `gorm:"type:text"` + UpdatedAt time.Time + Contract string `gorm:"type:varchar(64)"` + InputDigest string `gorm:"type:varchar(64)"` + RawCount *int64 +} + +func (graphqlCollectorState20260916) TableName() string { return "_devlake_graphql_collector_states" } + +type addGraphqlCollectorStates struct{} + +var _ plugin.MigrationScript = (*addGraphqlCollectorStates)(nil) + +func (*addGraphqlCollectorStates) Up(res context.BasicRes) errors.Error { + return res.GetDal().AutoMigrate(&graphqlCollectorState20260916{}) +} +func (*addGraphqlCollectorStates) Version() uint64 { return 20260916000001 } +func (*addGraphqlCollectorStates) Name() string { return "add graphql collector checkpoints" } diff --git a/backend/core/models/migrationscripts/graphql_checkpoint_test.go b/backend/core/models/migrationscripts/graphql_checkpoint_test.go new file mode 100644 index 00000000000..679fbb96910 --- /dev/null +++ b/backend/core/models/migrationscripts/graphql_checkpoint_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/devlake/core/models" + "github.com/stretchr/testify/require" + "reflect" + "testing" +) + +func TestGraphqlCheckpointMigrationSnapshot(t *testing.T) { + snapshot := reflect.TypeOf(graphqlCollectorState20260916{}) + runtime := reflect.TypeOf(models.GraphqlCollectorState{}) + require.Equal(t, snapshot.NumField(), runtime.NumField()) + for i := 0; i < snapshot.NumField(); i++ { + a, b := snapshot.Field(i), runtime.Field(i) + require.Equal(t, a.Name, b.Name) + require.Equal(t, a.Type, b.Type) + require.Equal(t, a.Tag, b.Tag) + } + require.Equal(t, (graphqlCollectorState20260916{}).TableName(), (models.GraphqlCollectorState{}).TableName()) +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 620a02af218..c5785bc372c 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -152,5 +152,6 @@ func All() []plugin.MigrationScript { new(addSprintVelocityFields), new(addBlueprintIdIndexToPipelines), new(expandDomainTextColumns), + new(addGraphqlCollectorStates), } } diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go index 6529be9ea9c..a8566cbf565 100644 --- a/backend/core/plugin/plugin_task.go +++ b/backend/core/plugin/plugin_task.go @@ -19,6 +19,8 @@ package plugin import ( "context" + "sync" + "time" corecontext "github.com/apache/devlake/core/context" "github.com/apache/devlake/core/errors" @@ -27,6 +29,49 @@ import ( type ProgressType int +type taskIDContextKey struct{} +type taskStartContextKey struct{} +type taskCodeContextKey struct{} + +var loadedPluginCode sync.Map + +// RegisterPluginCode records the loaded shared object's digest, not its path. +func RegisterPluginCode(name, digest string) { loadedPluginCode.Store(name, digest) } + +func WithTaskCode(ctx context.Context, name string) context.Context { + digest, _ := loadedPluginCode.Load(name) + if digest == nil { + digest = "" + } + return context.WithValue(ctx, taskCodeContextKey{}, digest) +} + +func TaskCode(ctx context.Context) string { + digest, _ := ctx.Value(taskCodeContextKey{}).(string) + return digest +} + +func WithTaskStartedAt(ctx context.Context, started time.Time) context.Context { + return context.WithValue(ctx, taskStartContextKey{}, started) +} + +func TaskStartedAt(ctx context.Context) (time.Time, bool) { + started, ok := ctx.Value(taskStartContextKey{}).(time.Time) + return started, ok +} + +// WithTaskID identifies the persisted task across process restarts, without +// changing the TaskContext interface implemented by plugins and test clients. +func WithTaskID(ctx context.Context, id uint64) context.Context { + return context.WithValue(ctx, taskIDContextKey{}, id) +} + +// TaskID returns zero for callers outside the persisted task runner. +func TaskID(ctx context.Context) uint64 { + id, _ := ctx.Value(taskIDContextKey{}).(uint64) + return id +} + const ( TaskSetProgress ProgressType = iota TaskIncProgress diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go index 9c6a32af9c5..f0c14fe10b7 100644 --- a/backend/core/runner/loader.go +++ b/backend/core/runner/loader.go @@ -18,8 +18,12 @@ limitations under the License. package runner import ( + "crypto/sha256" + "encoding/hex" "fmt" + "io" "io/fs" + "os" "path/filepath" goplugin "plugin" "strings" @@ -70,6 +74,20 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName)) } + file, readErr := os.Open(path) + if readErr != nil { + return readErr + } + hash := sha256.New() + _, readErr = io.Copy(hash, file) + closeErr := file.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + plugin.RegisterPluginCode(pluginName, hex.EncodeToString(hash.Sum(nil))) wg.Add(1) go func(pluginName string, pluginMeta plugin.PluginMeta) { defer func() { diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go index cde7d07ab45..77045a38173 100644 --- a/backend/core/runner/run_task.go +++ b/backend/core/runner/run_task.go @@ -134,6 +134,7 @@ func RunTask( } // start execution + task.BeganAt = &beganAt logger.Info("start executing task: %d", task.ID) dbe := db.UpdateColumns(task, []dal.DalSet{ {ColumnName: "status", Value: models.TASK_RUNNING}, @@ -248,6 +249,11 @@ func RunPluginSubTasks( } } + ctx = plugin.WithTaskID(ctx, task.ID) + ctx = plugin.WithTaskCode(ctx, task.Plugin) + if task.BeganAt != nil { + ctx = plugin.WithTaskStartedAt(ctx, *task.BeganAt) + } taskCtx := contextimpl.NewDefaultTaskContext(ctx, basicRes, task.Plugin, subtasksFlag, progress) if closeablePlugin, ok := pluginTask.(plugin.CloseablePluginTask); ok { defer closeablePlugin.Close(taskCtx) @@ -294,8 +300,16 @@ func RunPluginSubTasks( } subtask = append(subtask, s) } - if err := basicRes.GetDal().CreateOrUpdate(subtask); err != nil { - basicRes.GetLogger().Error(err, "error writing subtask list to DB") + for i := range subtask { + count, err := basicRes.GetDal().Count(dal.From(&models.Subtask{}), dal.Where("task_id = ? AND name = ?", task.ID, subtask[i].Name)) + if err != nil { + return err + } + if count == 0 { + if err := basicRes.GetDal().Create(&subtask[i]); err != nil { + return err + } + } } // execute subtasks in order @@ -324,7 +338,7 @@ func RunPluginSubTasks( if !subtaskMeta.ForceRunOnResume { if task.ID > 0 { sfc := errors.Must1(basicRes.GetDal().Count( - dal.From(&models.Subtask{}), dal.Where("task_id = ? AND name = ? AND finished_at IS NOT NULL", task.ID, subtaskMeta.Name), + dal.From(&models.Subtask{}), dal.Where("task_id = ? AND name = ? AND finished_at IS NOT NULL AND is_failed = ?", task.ID, subtaskMeta.Name, false), ), ) subtaskFinished = sfc > 0 @@ -421,29 +435,40 @@ func runSubtask( Number: subtaskNumber, BeganAt: &beginAt, } - recordSubtask(basicRes, subtask) - // defer to record subtask status - defer func() { - finishedAt := time.Now() + if err := recordSubtask(basicRes, subtask); err != nil { + return err + } + // A panic leaves this subtask unfinished. Only successful execution may + // create the marker used to skip work after a process restart. + err := entryPoint(ctx) + finishedAt := time.Now() + subtask.IsFailed = err != nil + if err == nil { subtask.FinishedAt = &finishedAt - subtask.SpentSeconds = finishedAt.Unix() - beginAt.Unix() - - recordSubtask(basicRes, subtask) - }() - return entryPoint(ctx) + } else { + subtask.Message = err.Error() + } + subtask.SpentSeconds = finishedAt.Unix() - beginAt.Unix() + if recordErr := recordSubtask(basicRes, subtask); recordErr != nil { + if err == nil { + return recordErr + } + return errors.Default.Combine([]error{err, recordErr}) + } + return err } -func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) { +func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) errors.Error { where := dal.Where("task_id = ? and name = ?", subtask.TaskID, subtask.Name) - if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ + return basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ {ColumnName: "began_at", Value: subtask.BeganAt}, {ColumnName: "finished_at", Value: subtask.FinishedAt}, {ColumnName: "spent_seconds", Value: subtask.SpentSeconds}, //{ColumnName: "finished_records", Value: subtask.FinishedRecords}, // FinishedRecords is zero always. {ColumnName: "number", Value: subtask.Number}, - }, where); err != nil { - basicRes.GetLogger().Error(err, "error writing subtask %d status to DB: %v", subtask.ID) - } + {ColumnName: "is_failed", Value: subtask.IsFailed}, + {ColumnName: "message", Value: subtask.Message}, + }, where) } func getTaskLogger(parentLogger log.Logger, task *models.Task) (log.Logger, errors.Error) { diff --git a/backend/core/runner/run_task_resume_test.go b/backend/core/runner/run_task_resume_test.go new file mode 100644 index 00000000000..561da68966f --- /dev/null +++ b/backend/core/runner/run_task_resume_test.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runner + +import ( + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "testing" +) + +func TestRunSubtaskCompletionMarker(t *testing.T) { + for _, mode := range []string{"success", "error", "panic", "start-write-error", "end-write-error"} { + t.Run(mode, func(t *testing.T) { + var records []models.Subtask + var db *mockdal.Dal + failure := errors.Default.New("injected failure") + res := unithelper.DummyBasicRes(func(d *mockdal.Dal) { + db = d + var startErr errors.Error + if mode == "start-write-error" { + startErr = failure + } + d.On("UpdateColumns", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + records = append(records, *args.Get(0).(*models.Subtask)) + }).Return(startErr).Once() + if mode != "start-write-error" && mode != "panic" { + var endErr errors.Error + if mode == "end-write-error" { + endErr = failure + } + d.On("UpdateColumns", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + records = append(records, *args.Get(0).(*models.Subtask)) + }).Return(endErr).Once() + } + }) + for _, expectation := range db.ExpectedCalls { + if expectation.Method == "AllTables" { + expectation.Maybe() + } + } + called := false + invoke := func() errors.Error { + return runSubtask(res, unithelper.DummySubTaskContext(db), 1, 1, func(plugin.SubTaskContext) errors.Error { + called = true + if mode == "panic" { + panic("injected panic") + } + if mode == "error" { + return failure + } + return nil + }) + } + if mode == "panic" { + require.Panics(t, func() { _ = invoke() }) + } else { + err := invoke() + require.Equal(t, mode != "success", err != nil) + } + require.Equal(t, mode != "start-write-error", called) + require.Nil(t, records[0].FinishedAt) + if mode == "error" { + require.Nil(t, records[1].FinishedAt) + require.True(t, records[1].IsFailed) + } + if mode == "success" { + require.NotNil(t, records[1].FinishedAt) + require.False(t, records[1].IsFailed) + } + db.AssertExpectations(t) + }) + } +} diff --git a/backend/helpers/pluginhelper/api/api_collector_stateful.go b/backend/helpers/pluginhelper/api/api_collector_stateful.go index d2b9e824508..c0d288652fa 100644 --- a/backend/helpers/pluginhelper/api/api_collector_stateful.go +++ b/backend/helpers/pluginhelper/api/api_collector_stateful.go @@ -70,6 +70,7 @@ func (m *StatefulApiCollector) InitCollector(args ApiCollectorArgs) errors.Error // InitGraphQLCollector appends a new GraphQL collector to the list func (m *StatefulApiCollector) InitGraphQLCollector(args GraphqlCollectorArgs) errors.Error { args.RawDataSubTaskArgs = m.RawDataSubTaskArgs + args.checkpointIndex = len(m.nestedCollectors) // highest priority: caller may hardcode the incremental flag. // e.g. github graphql pr_collector need to refetch OPENing PRs existing in the database if !args.Incremental { @@ -82,6 +83,11 @@ func (m *StatefulApiCollector) InitGraphQLCollector(args GraphqlCollectorArgs) e args.Incremental = m.CollectorStateManager.IsIncremental() } } + // A shared raw scope is flushed only once, by the first collector. A later + // collector must not erase data whose completed marker will survive restart. + if args.checkpointIndex > 0 { + args.Incremental = true + } graphqlCollector, err := NewGraphqlCollector(args) if err != nil { @@ -93,8 +99,23 @@ func (m *StatefulApiCollector) InitGraphQLCollector(args GraphqlCollectorArgs) e // Execute all nested collectors and save the state if all collectors succeed func (m *StatefulApiCollector) Execute() errors.Error { + raw, err := NewRawDataSubTask(m.RawDataSubTaskArgs) + if err != nil { + return err + } + release, err := acquireGraphqlScope(raw.table, raw.params) + if err != nil { + return err + } + defer release() for _, subtask := range m.nestedCollectors { + if graphql, ok := subtask.(*GraphqlCollector); ok { + graphql.args.scopeLocked = true + } err := subtask.Execute() + if graphql, ok := subtask.(*GraphqlCollector); ok { + graphql.args.scopeLocked = false + } if err != nil { return err } diff --git a/backend/helpers/pluginhelper/api/collector_state_manager.go b/backend/helpers/pluginhelper/api/collector_state_manager.go index 690cf1dd5c2..688e5473731 100644 --- a/backend/helpers/pluginhelper/api/collector_state_manager.go +++ b/backend/helpers/pluginhelper/api/collector_state_manager.go @@ -18,12 +18,14 @@ limitations under the License. package api import ( + gocontext "context" "time" "github.com/apache/devlake/core/context" "github.com/apache/devlake/core/dal" "github.com/apache/devlake/core/errors" "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" ) // CollectorStateManager manages the state of the collector. It is used to determine whether @@ -66,6 +68,11 @@ func NewCollectorStateManager(basicRes context.BasicRes, syncPolicy *models.Sync // fullsync by default now := time.Now() + if exec, ok := basicRes.(interface{ GetContext() gocontext.Context }); ok { + if started, found := plugin.TaskStartedAt(exec.GetContext()); found { + now = started + } + } stateManager = &CollectorStateManager{ db: db, state: state, diff --git a/backend/helpers/pluginhelper/api/graphql_async_client.go b/backend/helpers/pluginhelper/api/graphql_async_client.go index d67e78d05dc..0b9e6072dc6 100644 --- a/backend/helpers/pluginhelper/api/graphql_async_client.go +++ b/backend/helpers/pluginhelper/api/graphql_async_client.go @@ -134,10 +134,12 @@ func (apiClient *GraphqlAsyncClient) SetMaxRetry( // updateRateRemaining call getRateRemaining to update rateRemaining periodically func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, resetAt *time.Time) { + apiClient.rateExhaustCond.L.Lock() apiClient.rateRemaining = rateRemaining if rateRemaining > 0 { apiClient.rateExhaustCond.Signal() } + apiClient.rateExhaustCond.L.Unlock() go func() { if apiClient.getRateRemaining == nil { return @@ -157,7 +159,9 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese apiClient.logger.Info("failed to update graphql rate limit, will retry next cycle: %v", err) // Floor the reused value so Signal() always fires; prevents deadlock when // rateRemaining is 0 and the rate-limit endpoint keeps erroring (e.g. GHE). + apiClient.rateExhaustCond.L.Lock() fallback := apiClient.rateRemaining + apiClient.rateExhaustCond.L.Unlock() if fallback < defaultRateLimitConst { fallback = defaultRateLimitConst } @@ -179,6 +183,9 @@ func (apiClient *GraphqlAsyncClient) SetGetRateCost(getRateCost func(q interface // []graphql.DataError are the errors returned in response body // errors.Error is other error func (apiClient *GraphqlAsyncClient) Query(q interface{}, variables map[string]interface{}) ([]graphql.DataError, error) { + if apiClient.maxRetry <= 0 { + return nil, errors.BadInput.New("GraphQL retry attempts must be positive") + } apiClient.waitGroup.Add(1) defer apiClient.waitGroup.Done() apiClient.mu.Lock() @@ -186,7 +193,16 @@ func (apiClient *GraphqlAsyncClient) Query(q interface{}, variables map[string]i apiClient.rateExhaustCond.L.Lock() defer apiClient.rateExhaustCond.L.Unlock() + stopWake := context.AfterFunc(apiClient.ctx, func() { + apiClient.rateExhaustCond.L.Lock() + apiClient.rateExhaustCond.Broadcast() + apiClient.rateExhaustCond.L.Unlock() + }) + defer stopWake() for apiClient.rateRemaining <= 0 { + if err := apiClient.ctx.Err(); err != nil { + return nil, err + } apiClient.logger.Info(`rate limit remaining exhausted, waiting for next period.`) apiClient.rateExhaustCond.Wait() } @@ -197,17 +213,26 @@ func (apiClient *GraphqlAsyncClient) Query(q interface{}, variables map[string]i for retryTime < apiClient.maxRetry { select { case <-apiClient.ctx.Done(): - return nil, nil + return nil, apiClient.ctx.Err() default: var dataErrors []graphql.DataError - dataErrors, err := apiClient.client.Query(apiClient.ctx, q, variables) + dataErrors, err = apiClient.client.Query(apiClient.ctx, q, variables) if err == context.Canceled { return nil, err } if err != nil { apiClient.logger.Warn(err, "retry #%d graphql calling after %ds", retryTime, apiClient.waitBeforeRetry/time.Second) retryTime++ - <-time.After(apiClient.waitBeforeRetry) + if retryTime >= apiClient.maxRetry { + break + } + timer := time.NewTimer(apiClient.waitBeforeRetry) + select { + case <-apiClient.ctx.Done(): + timer.Stop() + return nil, apiClient.ctx.Err() + case <-timer.C: + } continue } if dataErrors != nil { @@ -230,17 +255,13 @@ func (apiClient *GraphqlAsyncClient) NextTick(task func() errors.Error, taskErro // to make sure task will be enqueued apiClient.waitGroup.Add(1) go func() { + defer apiClient.waitGroup.Done() select { case <-apiClient.ctx.Done(): + taskErrorChecker(apiClient.ctx.Err()) return default: - go func() { - // if set waitGroup done here, a serial of goroutine will block until sub-goroutine finish. - // But if done out of this go func, so task will run after waitGroup finish - // I have no idea about this now... - defer apiClient.waitGroup.Done() - taskErrorChecker(task()) - }() + taskErrorChecker(task()) } }() } diff --git a/backend/helpers/pluginhelper/api/graphql_checkpoint_guard.go b/backend/helpers/pluginhelper/api/graphql_checkpoint_guard.go new file mode 100644 index 00000000000..6f0b25a69e4 --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_checkpoint_guard.go @@ -0,0 +1,191 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "reflect" + "sync" + + "github.com/apache/devlake/core/errors" +) + +// The server holds an exclusive DB lock across its lifetime. This additional +// process-local lock rejects overlapping collectors using the same raw scope. +var graphqlScopes = struct { + sync.Mutex + active map[string]bool +}{active: map[string]bool{}} + +func acquireGraphqlScope(table, params string) (func(), errors.Error) { + key := table + "\x00" + params + graphqlScopes.Lock() + defer graphqlScopes.Unlock() + if graphqlScopes.active[key] { + return nil, errors.Default.New("GraphQL raw scope is already being collected") + } + graphqlScopes.active[key] = true + return func() { graphqlScopes.Lock(); delete(graphqlScopes.active, key); graphqlScopes.Unlock() }, nil +} + +var graphqlBinaryIdentity = sync.OnceValues(func() (string, error) { + path, err := os.Executable() + if err != nil { + return "", err + } + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +}) + +// Stage inputs to disk before fetching. The digest makes changed ordering, +// batching or input content an explicit error instead of duplicating old data. +// SQL cursors are closed before collection. Payloads stay on disk; only types +// and fixed-size hashes for duplicate detection are retained in memory. +type graphqlStagedInput struct { + file *os.File + decoder *json.Decoder + types []reflect.Type + pending *graphqlInputRecord + err error +} +type graphqlInputRecord struct { + Type int + Data json.RawMessage +} + +func stageGraphqlInput(ctx context.Context, source Iterator) (staged *graphqlStagedInput, digest string, resultErr errors.Error) { + defer func() { + if closeErr := source.Close(); resultErr == nil && closeErr != nil { + resultErr = closeErr + } + if resultErr != nil && staged != nil { + _ = staged.Close() + staged = nil + } + }() + file, err := os.CreateTemp("", "devlake-graphql-input-*") + if err != nil { + return nil, "", errors.Convert(err) + } + staged = &graphqlStagedInput{file: file} + hash := sha256.New() + encoder := json.NewEncoder(io.MultiWriter(file, hash)) + typeIDs := map[reflect.Type]int{} + seen := map[[32]byte]bool{} + for source.HasNext() { + if err := ctx.Err(); err != nil { + return staged, "", errors.Convert(err) + } + value, err := source.Fetch() + if err != nil { + return staged, "", err + } + typ := reflect.TypeOf(value) + id, found := typeIDs[typ] + if !found { + id = len(staged.types) + typeIDs[typ] = id + staged.types = append(staged.types, typ) + // A type/schema change must not silently reuse an old manifest. + _, _ = fmt.Fprintf(hash, "%v\n", typ) + } + data, marshalErr := json.Marshal(value) + if marshalErr != nil { + return staged, "", errors.Convert(marshalErr) + } + key := sha256.Sum256(data) + if seen[key] { + return staged, "", errors.BadInput.New("duplicate GraphQL input; provide unique inputs") + } + seen[key] = true + if err := encoder.Encode(graphqlInputRecord{Type: id, Data: data}); err != nil { + return staged, "", errors.Convert(err) + } + } + if sourceErr, ok := source.(interface{ Err() error }); ok { + if err := sourceErr.Err(); err != nil { + return staged, "", errors.Convert(err) + } + } + if err := ctx.Err(); err != nil { + return staged, "", errors.Convert(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return staged, "", errors.Convert(err) + } + staged.decoder = json.NewDecoder(file) + return staged, hex.EncodeToString(hash.Sum(nil)), nil +} +func (s *graphqlStagedInput) HasNext() bool { + if s.err != nil { + return false + } + var record graphqlInputRecord + if err := s.decoder.Decode(&record); err != nil { + if err != io.EOF { + s.err = err + } + return false + } + s.pending = &record + return true +} +func (s *graphqlStagedInput) Fetch() (interface{}, errors.Error) { + if s.pending == nil { + return nil, errors.Default.New("input Fetch without HasNext") + } + record := s.pending + s.pending = nil + typ := s.types[record.Type] + if typ == nil { + return nil, nil + } + value := reflect.New(typ) + if err := json.Unmarshal(record.Data, value.Interface()); err != nil { + return nil, errors.Convert(err) + } + return value.Elem().Interface(), nil +} +func (s *graphqlStagedInput) Err() error { return s.err } +func (s *graphqlStagedInput) Close() errors.Error { + if s.file == nil { + return nil + } + file := s.file + s.file = nil + closeErr := file.Close() + removeErr := os.Remove(file.Name()) + if closeErr != nil { + return errors.Convert(closeErr) + } + return errors.Convert(removeErr) +} diff --git a/backend/helpers/pluginhelper/api/graphql_checkpoint_guard_test.go b/backend/helpers/pluginhelper/api/graphql_checkpoint_guard_test.go new file mode 100644 index 00000000000..ba60b3e3c9d --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_checkpoint_guard_test.go @@ -0,0 +1,306 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "reflect" + "sync" + "testing" + "time" + + "github.com/apache/devlake/core/dal" + lakeerrors "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type failingGraphqlInput struct { + values []interface{} + position int + pending bool + readErr error + fetchErr lakeerrors.Error + closeErr lakeerrors.Error + closes int +} + +func (s *failingGraphqlInput) HasNext() bool { + if s.position >= len(s.values) { + return false + } + s.position++ + s.pending = true + return true +} +func (s *failingGraphqlInput) Fetch() (interface{}, lakeerrors.Error) { + if !s.pending { + return nil, lakeerrors.Default.New("Fetch without Next") + } + s.pending = false + return s.values[s.position-1], s.fetchErr +} +func (s *failingGraphqlInput) Err() error { return s.readErr } +func (s *failingGraphqlInput) Close() lakeerrors.Error { s.closes++; return s.closeErr } + +func TestGraphqlInvalidCollectionConfiguration(t *testing.T) { + for _, field := range []string{"build", "input", "page", "batch", "parser", "both-parsers", "retry"} { + t.Run(field, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("invalid configuration must not send HTTP") }) + args := *c.args + switch field { + case "build": + args.BuildQuery = nil + case "input": + args.InputStep = -1 + case "page": + args.PageSize = -1 + case "batch": + args.BatchSize = -1 + case "parser": + args.ResponseParser = nil + case "both-parsers": + args.ResponseParserWithDal = func(interface{}, dal.Dal) ([]json.RawMessage, lakeerrors.Error) { return nil, nil } + case "retry": + c.args.GraphqlClient.maxRetry = 0 + _, err := c.args.GraphqlClient.Query(&resumeTestQuery{}, nil) + require.Error(t, err) + return + } + _, err := NewGraphqlCollector(args) + require.Error(t, err) + }) + } +} + +func TestGraphqlStageInput(t *testing.T) { + type record struct{ ID int } + values := []interface{}{&record{1}, &record{2}, "three", nil, map[string]int{"a": 4}} + for _, failure := range []string{"none", "read", "fetch", "close", "marshal", "cancel", "duplicate"} { + t.Run(failure, func(t *testing.T) { + input := &failingGraphqlInput{values: values} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch failure { + case "read": + input.readErr = errors.New("DB cursor disconnected") + case "fetch": + input.fetchErr = lakeerrors.Default.New("scan failed") + case "close": + input.closeErr = lakeerrors.Default.New("close failed") + case "marshal": + input.values = []interface{}{make(chan int)} + case "cancel": + cancel() + case "duplicate": + input.values = []interface{}{"same", "same"} + } + staged, digest, err := stageGraphqlInput(ctx, input) + require.Equal(t, 1, input.closes) + if failure != "none" { + require.Error(t, err) + require.Nil(t, staged) + return + } + require.NoError(t, err) + require.Len(t, digest, 64) + defer staged.Close() + var got []interface{} + for staged.HasNext() { + value, err := staged.Fetch() + require.NoError(t, err) + got = append(got, value) + } + require.NoError(t, staged.Err()) + require.Equal(t, values, got) + require.NoError(t, staged.Close()) + require.NoError(t, staged.Close()) + }) + } +} + +func TestGraphqlInputFailureCannotComplete(t *testing.T) { + for _, failure := range []string{"read", "fetch", "close"} { + t.Run(failure, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("must not fetch incomplete input list") }) + input := &failingGraphqlInput{values: []interface{}{"a"}} + switch failure { + case "read": + input.readErr = errors.New("read failed") + case "fetch": + input.fetchErr = lakeerrors.Default.New("fetch failed") + case "close": + input.closeErr = lakeerrors.Default.New("close failed") + } + c.args.Input = input + require.Error(t, c.Execute()) + require.Equal(t, 1, input.closes) + // No migration/deletion/checkpoint expectations: all must remain untouched. + }) + } +} + +func TestGraphqlScopeLock(t *testing.T) { + release, err := acquireGraphqlScope("table", "repo") + require.NoError(t, err) + _, err = acquireGraphqlScope("table", "repo") + require.Error(t, err) + other, err := acquireGraphqlScope("table", "other") + require.NoError(t, err) + other() + release() + releaseAgain, err := acquireGraphqlScope("table", "repo") + require.NoError(t, err) + releaseAgain() +} + +type graphqlContextOverride struct { + plugin.SubTaskContext + ctx context.Context +} + +func (s graphqlContextOverride) GetContext() context.Context { return s.ctx } + +func TestGraphqlCancelledEmptyInput(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("unexpected HTTP") }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c.args.Ctx = graphqlContextOverride{c.args.Ctx, ctx} + c.args.Input = &failingGraphqlInput{} + require.ErrorIs(t, c.Execute(), context.Canceled) +} + +func TestGraphqlCancellationDuringRateWait(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + client := &GraphqlAsyncClient{ctx: ctx, logger: unithelper.DummyLogger(), rateExhaustCond: sync.NewCond(&sync.Mutex{}), maxRetry: 1} + done := make(chan error, 1) + go func() { _, err := client.Query(&resumeTestQuery{}, nil); done <- err }() + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("rate-limit wait ignored cancellation") + } + client.Wait() +} + +func TestGraphqlCancellationDuringRetry(t *testing.T) { + db := mockdal.NewDal(t) + requested := make(chan struct{}, 1) + c := newResumeTestCollector(t, db, func(w http.ResponseWriter, _ *http.Request) { + requested <- struct{}{} + http.Error(w, "retry", 503) + }) + client := c.args.GraphqlClient + client.maxRetry, client.waitBeforeRetry = 3, time.Hour + done := make(chan error, 1) + go func() { _, err := client.Query(&resumeTestQuery{}, nil); done <- err }() + <-requested + client.cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("retry wait ignored cancellation") + } +} + +func TestGraphqlInvalidPageDoesNotAdvanceCheckpoint(t *testing.T) { + for _, mode := range []string{"nil", "error", "empty-cursor", "same-cursor"} { + t.Run(mode, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(w http.ResponseWriter, _ *http.Request) { resumeTestResponse(w, true) }) + expectInput(db, "null", &GraphqlCollectorState{TaskID: 1, SkipCursor: "saved"}, nil) + c.args.GetPageInfo = func(interface{}, *GraphqlCollectorArgs) (*GraphqlQueryPageInfo, error) { + switch mode { + case "nil": + return nil, nil + case "error": + return nil, errors.New("invalid page") + case "empty-cursor": + return &GraphqlQueryPageInfo{HasNextPage: true}, nil + default: + return &GraphqlQueryPageInfo{HasNextPage: true, EndCursor: "saved"}, nil + } + } + tx := &checkpointTestTx{Dal: db} + db.On("Begin").Return(tx).Once() + c.exec(nil) + c.args.GraphqlClient.Wait() + require.True(t, c.HasError()) + require.True(t, tx.rolledBack) + }) + } +} + +func TestGraphqlPartialErrorCannotComplete(t *testing.T) { + for _, ignore := range []bool{false, true} { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"items":{"nodes":[{"id":1}],"pageInfo":{"hasNextPage":false}}},"errors":[{"message":"upstream unavailable"}]}`)) + }) + c.args.IgnoreQueryErrors = ignore + expectInput(db, "null", &GraphqlCollectorState{TaskID: 1}, nil) + c.exec(nil) + c.args.GraphqlClient.Wait() + require.True(t, c.HasError()) + } +} + +func TestGraphqlNestedCollectorRegistration(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) {}) + manager := &StatefulApiCollector{RawDataSubTaskArgs: c.args.RawDataSubTaskArgs} + for i := 0; i < 2; i++ { + args := *c.args + args.Incremental = true + require.NoError(t, manager.InitGraphQLCollector(args)) + nested := manager.nestedCollectors[i].(*GraphqlCollector) + require.Equal(t, i, nested.args.checkpointIndex) + require.True(t, nested.args.Incremental) + } +} + +func TestGraphqlParserUsesPageTransaction(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(w http.ResponseWriter, _ *http.Request) { resumeTestResponse(w, false) }) + expectInput(db, "null", &GraphqlCollectorState{TaskID: 1}, nil) + tx := &checkpointTestTx{Dal: db} + db.On("Begin").Return(tx).Once() + c.args.ResponseParser = nil + c.args.ResponseParserWithDal = func(_ interface{}, actual dal.Dal) ([]json.RawMessage, lakeerrors.Error) { + require.Equal(t, reflect.ValueOf(tx).Pointer(), reflect.ValueOf(actual).Pointer()) + return nil, lakeerrors.Default.New("side effect failed") + } + c.exec(nil) + c.args.GraphqlClient.Wait() + require.True(t, c.HasError()) + require.True(t, tx.rolledBack) + db.AssertNotCalled(t, "CreateOrUpdate", mock.Anything) +} diff --git a/backend/helpers/pluginhelper/api/graphql_collector.go b/backend/helpers/pluginhelper/api/graphql_collector.go index 2601e4e4f51..2e0f4e14574 100644 --- a/backend/helpers/pluginhelper/api/graphql_collector.go +++ b/backend/helpers/pluginhelper/api/graphql_collector.go @@ -23,6 +23,7 @@ import ( "net/http" "reflect" "strings" + "sync" "time" "github.com/apache/devlake/core/dal" @@ -78,17 +79,30 @@ type GraphqlCollectorArgs struct { // GetPageInfo is to tell `GraphqlCollector` is page information GetPageInfo func(query interface{}, args *GraphqlCollectorArgs) (*GraphqlQueryPageInfo, error) BatchSize int - // one of ResponseParser and ResponseParserEvenWhenDataErrors is required to parse response - ResponseParser func(queryWrapper interface{}) ([]json.RawMessage, errors.Error) - IgnoreQueryErrors bool + // Exactly one of ResponseParser and ResponseParserWithDal is required. + ResponseParser func(queryWrapper interface{}) ([]json.RawMessage, errors.Error) + // ResponseParserWithDal must use the supplied transaction for all DB side + // effects. They commit atomically with raw rows and the input checkpoint. + ResponseParserWithDal func(queryWrapper interface{}, db dal.Dal) ([]json.RawMessage, errors.Error) + IgnoreQueryErrors bool + // Stable position when a subtask contains multiple collectors sharing a table. + checkpointIndex int + scopeLocked bool + checkpointContract string + inputDigest string } // GraphqlCollector help you collect data from Graphql services type GraphqlCollector struct { *RawDataSubTask - args *GraphqlCollectorArgs - workerErrors []error - batchSave *BatchSave + args *GraphqlCollectorArgs + workerErrors []error + batchSave *BatchSave + taskID uint64 + scopeHash string + errorsMu sync.Mutex + pageMu sync.Mutex + expectedRawCount *int64 } // ErrFinishCollect is an error which will finish this collector @@ -99,6 +113,9 @@ var ErrFinishCollect = errors.Default.New("finish collect") // of response we want to save, GraphqlCollector will collect them from remote server and store them into database. func NewGraphqlCollector(args GraphqlCollectorArgs) (*GraphqlCollector, errors.Error) { // process args + if args.BuildQuery == nil || args.InputStep < 0 || args.PageSize < 0 || args.BatchSize < 0 { + return nil, errors.BadInput.New("GraphQL BuildQuery is required and collection sizes must not be negative") + } rawDataSubTask, err := NewRawDataSubTask(args.RawDataSubTaskArgs) if err != nil { return nil, err @@ -106,8 +123,11 @@ func NewGraphqlCollector(args GraphqlCollectorArgs) (*GraphqlCollector, errors.E if args.GraphqlClient == nil { return nil, errors.Default.New("ApiClient is required") } - if args.ResponseParser == nil { - return nil, errors.Default.New("one of ResponseParser and ResponseParserWithDataErrors is required") + if args.ResponseParser == nil && args.ResponseParserWithDal == nil { + return nil, errors.Default.New("one of ResponseParser and ResponseParserWithDal is required") + } + if args.ResponseParser != nil && args.ResponseParserWithDal != nil { + return nil, errors.BadInput.New("only one GraphQL response parser may be supplied") } if args.BatchSize == 0 { args.BatchSize = 100 @@ -118,6 +138,7 @@ func NewGraphqlCollector(args GraphqlCollectorArgs) (*GraphqlCollector, errors.E apiCollector := &GraphqlCollector{ RawDataSubTask: rawDataSubTask, args: &args, + taskID: plugin.TaskID(args.Ctx.GetContext()), batchSave: errors.Must1(NewBatchSave( args.Ctx, reflect.TypeOf(&RawData{}), @@ -130,6 +151,45 @@ func NewGraphqlCollector(args GraphqlCollectorArgs) (*GraphqlCollector, errors.E // Execute api collection func (collector *GraphqlCollector) Execute() errors.Error { + inputClosed := false + if collector.args.Input != nil { + defer func() { + if !inputClosed { + _ = collector.args.Input.Close() + } + }() + } + if err := collector.args.Ctx.GetContext().Err(); err != nil { + return errors.Convert(err) + } + if !collector.args.scopeLocked { + release, err := acquireGraphqlScope(collector.table, collector.params) + if err != nil { + return err + } + defer release() + } + if collector.taskID != 0 { + binary, err := graphqlBinaryIdentity() + if err != nil { + return errors.Convert(err) + } + contract, err := json.Marshal([]interface{}{binary, plugin.TaskCode(collector.args.Ctx.GetContext()), collector.args.InputStep, collector.args.PageSize, collector.args.GetPageInfo != nil}) + if err != nil { + return errors.Convert(err) + } + collector.args.checkpointContract = graphqlCollectorInputHash(string(contract)) + if collector.args.Input != nil { + inputClosed = true + staged, digest, err := stageGraphqlInput(collector.args.Ctx.GetContext(), collector.args.Input) + if err != nil { + return err + } + collector.args.Input = staged + collector.args.inputDigest = digest + inputClosed = false + } + } logger := collector.args.Ctx.GetLogger() logger.Info("start graphql collection") @@ -139,18 +199,21 @@ func (collector *GraphqlCollector) Execute() errors.Error { if err != nil { return errors.Default.Wrap(err, "error running auto-migrate") } - // flush data if not incremental collection - if !collector.args.Incremental { - err = db.Delete(&RawData{}, dal.From(collector.table), dal.Where("params = ?", collector.params)) - if err != nil { - return errors.Default.Wrap(err, "error deleting data from collector") + completed, err := collector.prepareCheckpoint() + if err != nil { + return err + } + if completed { + if collector.args.Input != nil { + inputClosed = true + return collector.args.Input.Close() } + return nil } collector.args.Ctx.SetProgress(0, -1) if collector.args.Input != nil { iterator := collector.args.Input - defer iterator.Close() // the comment about difference is written at GraphqlCollectorArgs.InputStep if collector.args.InputStep == 1 { for iterator.HasNext() && !collector.HasError() { @@ -185,6 +248,14 @@ func (collector *GraphqlCollector) Execute() errors.Error { logger.Debug("wait for all async api to finished") collector.args.GraphqlClient.Wait() + collector.checkError(collector.args.Ctx.GetContext().Err()) + if collector.args.Input != nil { + if source, ok := collector.args.Input.(interface{ Err() error }); ok { + collector.checkError(source.Err()) + } + collector.checkError(collector.args.Input.Close()) + inputClosed = true + } if collector.HasError() { err = errors.Default.Combine(collector.workerErrors) @@ -196,13 +267,17 @@ func (collector *GraphqlCollector) Execute() errors.Error { } err = collector.batchSave.Close() - return err + if err != nil { + return err + } + return collector.completeCheckpoint() } func (collector *GraphqlCollector) exec(input interface{}) { inputJson, err := json.Marshal(input) if err != nil { collector.checkError(errors.Default.Wrap(err, `input can not be marshal to json`)) + return } reqData := new(GraphqlRequestData) reqData.Input = input @@ -211,45 +286,28 @@ func (collector *GraphqlCollector) exec(input interface{}) { SkipCursor: nil, Size: collector.args.PageSize, } - if collector.args.GetPageInfo != nil { - collector.fetchOneByOne(reqData) - } else { - collector.fetchAsync(reqData, nil) - } -} - -// fetchOneByOne fetches data of all pages for APIs that return paging information -func (collector *GraphqlCollector) fetchOneByOne(reqData *GraphqlRequestData) { - // fetch first page - var fetchNextPage func(query interface{}) errors.Error - fetchNextPage = func(query interface{}) errors.Error { - pageInfo, err := collector.args.GetPageInfo(query, collector.args) + if collector.taskID != 0 { + state, err := loadGraphqlCollectorState(collector.args.Ctx.GetDal(), collector.scopeHash, graphqlCollectorInputHash(string(inputJson))) if err != nil { - return errors.Default.Wrap(err, "fetchPagesDetermined get totalPages failed") - } - if pageInfo == nil { - return errors.Default.New("fetchPagesDetermined got pageInfo is nil") + collector.checkError(err) + return } - if pageInfo.HasNextPage { - collector.args.GraphqlClient.NextTick(func() errors.Error { - reqDataTemp := &GraphqlRequestData{ - Pager: &CursorPager{ - SkipCursor: &pageInfo.EndCursor, - Size: collector.args.PageSize, - }, - Input: reqData.Input, - InputJSON: reqData.InputJSON, - } - collector.fetchAsync(reqDataTemp, fetchNextPage) - return nil - }, collector.checkError) + if state != nil && state.TaskID == collector.taskID { + if state.Completed { + return + } + if state.SkipCursor != "" { + reqData.Pager.SkipCursor = &state.SkipCursor + } } - return nil } - collector.fetchAsync(reqData, fetchNextPage) + collector.fetchAsync(reqData) } -func (collector *GraphqlCollector) fetchAsync(reqData *GraphqlRequestData, handler func(query interface{}) errors.Error) { +func (collector *GraphqlCollector) fetchAsync(reqData *GraphqlRequestData) { + if collector.HasError() { + return + } if reqData.Pager == nil { reqData.Pager = &CursorPager{ SkipCursor: nil, @@ -275,7 +333,7 @@ func (collector *GraphqlCollector) fetchAsync(reqData *GraphqlRequestData, handl return } if len(dataErrors) > 0 { - if !collector.args.IgnoreQueryErrors { + if !collector.args.IgnoreQueryErrors || collector.taskID != 0 { hasNonIgnorableDataErrors := false for _, dataError := range dataErrors { if isIgnorableGraphqlQueryError(dataError) { @@ -289,7 +347,7 @@ func (collector *GraphqlCollector) fetchAsync(reqData *GraphqlRequestData, handl return } } - // else: error will deal by ResponseParserWithDataErrors + // Task-less legacy callers may explicitly opt into partial responses. } defer logger.Debug("fetchAsync >>> done for %v %v", query, variables) @@ -300,39 +358,97 @@ func (collector *GraphqlCollector) fetchAsync(reqData *GraphqlRequestData, handl return } - results, err := collector.args.ResponseParser(query) - for _, result := range results { - row := &RawData{ - Params: collector.params, - Data: result, - Url: queryStr, - Input: variablesJson, + var pageInfo *GraphqlQueryPageInfo + var insertedRows int64 + // Parse within the transaction too: parsers may delete stale source data. + savePage := func(db dal.Dal) errors.Error { + var results []json.RawMessage + var parseErr errors.Error + if collector.args.ResponseParserWithDal != nil { + results, parseErr = collector.args.ResponseParserWithDal(query, db) + } else { + results, parseErr = collector.args.ResponseParser(query) } - // collector.batchSave.Add(row) - err = db.Create(row, dal.From(collector.table)) - if err != nil { - collector.checkError(errors.Default.Wrap(err, `not created row table in graphql collector`)) - return + finished := errors.Is(parseErr, ErrFinishCollect) + if parseErr != nil && !finished { + return errors.Default.Wrap(parseErr, "graphql response parser failed") + } + if !finished && collector.args.GetPageInfo != nil { + var pageErr error + pageInfo, pageErr = collector.args.GetPageInfo(query, collector.args) + if pageErr != nil { + return errors.Convert(pageErr) + } + if pageInfo == nil { + return errors.Default.New("graphql pageInfo is nil") + } + if pageInfo.HasNextPage && (pageInfo.EndCursor == "" || (reqData.Pager.SkipCursor != nil && pageInfo.EndCursor == *reqData.Pager.SkipCursor)) { + return errors.Default.New("graphql cursor did not advance") + } + } + if err := collector.args.Ctx.GetContext().Err(); err != nil { + return errors.Convert(err) + } + for _, result := range results { + if err := db.Create(&RawData{Params: collector.params, Data: result, Url: queryStr, Input: variablesJson}, dal.From(collector.table)); err != nil { + return err + } + insertedRows++ } + if collector.taskID != 0 { + state := collector.checkpoint(graphqlCollectorInputHash(string(reqData.InputJSON))) + state.Completed = pageInfo == nil || !pageInfo.HasNextPage + if !state.Completed { + state.SkipCursor = pageInfo.EndCursor + } + return saveGraphqlCollectorState(db, state) + } + return nil } - if err != nil { - if errors.Is(err, ErrFinishCollect) { - logger.Info("collector finish by parser") - handler = nil - } else { - collector.checkError(errors.Default.Wrap(err, `not parsed response in graphql collector`)) - return + var saveErr errors.Error + collector.pageMu.Lock() + defer collector.pageMu.Unlock() + if collector.taskID != 0 || collector.args.ResponseParserWithDal != nil { + tx := db.Begin() + defer tx.Rollback() + saveErr = savePage(tx) + var rawCount int64 + if saveErr == nil && collector.args.checkpointContract != "" { + // Pure parsers only append rows. Recounting the entire growing raw + // scope on every page would make a long collection quadratic. + if collector.args.ResponseParserWithDal == nil && collector.expectedRawCount != nil { + rawCount = *collector.expectedRawCount + insertedRows + } else { + // Transactional parsers may also delete raw rows. + rawCount, saveErr = tx.Count(dal.From(collector.table), dal.Where("params = ?", collector.params)) + } + if saveErr == nil { + marker := collector.rawCheckpoint() + marker.RawCount = &rawCount + saveErr = saveGraphqlCollectorState(tx, marker) + } } + if saveErr == nil { + saveErr = tx.Commit() + if saveErr == nil && collector.args.checkpointContract != "" { + collector.expectedRawCount = &rawCount + } + } + } else { + saveErr = savePage(db) + } + if saveErr != nil { + collector.checkError(saveErr) + return } - collector.args.Ctx.IncProgress(1) - if handler != nil { - // trigger next fetch, but return if ErrFinishCollect got from ResponseParser - err = handler(query) - if err != nil { - collector.checkError(errors.Default.Wrap(err, `handle failed in graphql collector`)) - return - } + if pageInfo != nil && pageInfo.HasNextPage { + next := *reqData + next.Pager = &CursorPager{SkipCursor: &pageInfo.EndCursor, Size: collector.args.PageSize} + collector.args.GraphqlClient.NextTick(func() errors.Error { + collector.fetchAsync(&next) + return nil + }, collector.checkError) } } @@ -340,11 +456,15 @@ func (collector *GraphqlCollector) checkError(err error) { if err == nil { return } + collector.errorsMu.Lock() + defer collector.errorsMu.Unlock() collector.workerErrors = append(collector.workerErrors, err) } // HasError return if any error occurred func (collector *GraphqlCollector) HasError() bool { + collector.errorsMu.Lock() + defer collector.errorsMu.Unlock() return len(collector.workerErrors) > 0 } diff --git a/backend/helpers/pluginhelper/api/graphql_collector_crash_test.go b/backend/helpers/pluginhelper/api/graphql_collector_crash_test.go new file mode 100644 index 00000000000..2a6a4689c3b --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_collector_crash_test.go @@ -0,0 +1,410 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + + deverrors "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + "github.com/apache/devlake/impls/dalgorm" + "github.com/merico-ai/graphql" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +type crashInputs struct{ values []string } + +func openGraphqlCrashDB(dsn string) (*gorm.DB, error) { + if os.Getenv("DEVLAKE_CRASH_DRIVER") == "postgres" { + return gorm.Open(postgres.Open(dsn), &gorm.Config{}) + } + return gorm.Open(mysql.Open(dsn), &gorm.Config{}) +} + +type graphqlCrashRequest struct { + ID uint64 `gorm:"primaryKey;autoIncrement"` + Scenario string + Phase string + InputID string + CursorValue string + InputMembers string +} + +func (graphqlCrashRequest) TableName() string { return "crash_requests" } + +func (i *crashInputs) HasNext() bool { return len(i.values) > 0 } +func (i *crashInputs) Fetch() (interface{}, deverrors.Error) { + value := i.values[0] + i.values = i.values[1:] + return value, nil +} +func (i *crashInputs) Close() deverrors.Error { return nil } + +// Real rollback and retry assertions complement the mocked commit-error tests. +func TestGraphqlCollectorMySQLLifecycleRecovery(t *testing.T) { + dsn := os.Getenv("DEVLAKE_CRASH_DSN") + if dsn == "" { + t.Skip("dedicated crash-test database not configured") + } + for _, operation := range []string{"prepare", "complete"} { + for _, stage := range []string{"delete", "save"} { + t.Run(operation+"/"+stage, func(t *testing.T) { + db, err := openGraphqlCrashDB(dsn) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + defer sqlDB.Close() + ctx := unithelper.DummySubTaskContext(dalgorm.NewDalgorm(db)) + args := RawDataSubTaskArgs{Ctx: ctx, Table: "lifecycle_" + operation + "_" + stage, Params: "repo"} + raw, lakeErr := NewRawDataSubTask(args) + require.NoError(t, lakeErr) + c := &GraphqlCollector{RawDataSubTask: raw, args: &GraphqlCollectorArgs{RawDataSubTaskArgs: args}, taskID: 1} + require.NoError(t, db.Table(c.table).AutoMigrate(&RawData{})) + _, lakeErr = c.prepareCheckpoint() + require.NoError(t, lakeErr) + require.NoError(t, db.Table(c.table).Create(&RawData{Params: c.params, Data: json.RawMessage(`{"ID":99}`)}).Error) + input := c.checkpoint("input") + input.Completed = true + require.NoError(t, db.Create(input).Error) + if operation == "prepare" { + c.taskID = 2 + } + injected := errors.New("injected lifecycle SQL failure") + if stage == "save" { + require.NoError(t, db.Callback().Create().After("gorm:create").Register("fail_lifecycle", func(tx *gorm.DB) { + if _, ok := tx.Statement.Dest.(*GraphqlCollectorState); ok { + tx.AddError(injected) + } + })) + } else { + require.NoError(t, db.Callback().Delete().After("gorm:delete").Register("fail_lifecycle", func(tx *gorm.DB) { + if _, ok := tx.Statement.Dest.(*GraphqlCollectorState); ok { + tx.AddError(injected) + } + })) + } + run := func() error { + if operation == "prepare" { + _, err := c.prepareCheckpoint() + return err + } + return c.completeCheckpoint() + } + require.Error(t, run()) + var marker GraphqlCollectorState + require.NoError(t, db.Where("scope_hash = ? AND input_hash = ?", c.scopeHash, "").First(&marker).Error) + require.EqualValues(t, 1, marker.TaskID) + require.False(t, marker.Completed) + var count int64 + require.NoError(t, db.Table(c.table).Count(&count).Error) + require.EqualValues(t, 1, count, "failed initialization must not lose raw data") + require.NoError(t, db.Model(&GraphqlCollectorState{}).Where("scope_hash = ?", c.scopeHash).Count(&count).Error) + require.EqualValues(t, 2, count, "failed completion must retain per-input progress") + if stage == "save" { + require.NoError(t, db.Callback().Create().Remove("fail_lifecycle")) + } else { + require.NoError(t, db.Callback().Delete().Remove("fail_lifecycle")) + } + require.NoError(t, run(), "retry must recover") + marker = GraphqlCollectorState{} + require.NoError(t, db.Where("scope_hash = ? AND input_hash = ?", c.scopeHash, "").First(&marker).Error) + require.Equal(t, c.taskID, marker.TaskID) + require.Equal(t, operation == "complete", marker.Completed) + require.NoError(t, db.Model(&GraphqlCollectorState{}).Where("scope_hash = ?", c.scopeHash).Count(&count).Error) + require.EqualValues(t, 1, count) + require.NoError(t, db.Table(c.table).Count(&count).Error) + if operation == "prepare" { + require.Zero(t, count) + } else { + require.EqualValues(t, 1, count) + } + }) + } + } +} + +func TestGraphqlCollectorMySQLScopeIsolation(t *testing.T) { + dsn := os.Getenv("DEVLAKE_CRASH_DSN") + if dsn == "" { + t.Skip("dedicated crash-test database not configured") + } + db, err := openGraphqlCrashDB(dsn) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + defer sqlDB.Close() + var collectors []*GraphqlCollector + for _, dimension := range []string{"base", "table", "params", "subtask", "index"} { + var ctx plugin.SubTaskContext = unithelper.DummySubTaskContext(dalgorm.NewDalgorm(db)) + args := GraphqlCollectorArgs{RawDataSubTaskArgs: RawDataSubTaskArgs{Ctx: ctx, Table: "scope_isolation", Params: "repo"}, Incremental: true} + switch dimension { + case "table": + args.Table += "_other" + case "params": + args.Params = "other-repository" + case "subtask": + args.Ctx = namedCheckpointContext{ctx, "other-subtask"} + case "index": + args.checkpointIndex = 1 + } + raw, err := NewRawDataSubTask(args.RawDataSubTaskArgs) + require.NoError(t, err) + c := &GraphqlCollector{RawDataSubTask: raw, args: &args, taskID: 1} + completed, err := c.prepareCheckpoint() + require.NoError(t, err) + require.False(t, completed) + input := c.checkpoint("same-input") + input.SkipCursor = dimension + require.NoError(t, saveGraphqlCollectorState(ctx.GetDal(), input)) + collectors = append(collectors, c) + } + // Completing and starting a new run for one scope must not clear any other. + require.NoError(t, collectors[0].completeCheckpoint()) + collectors[0].taskID = 2 + _, lakeErr := collectors[0].prepareCheckpoint() + require.NoError(t, lakeErr) + for i, dimension := range []string{"table", "params", "subtask", "index"} { + c := collectors[i+1] + state, err := loadGraphqlCollectorState(c.args.Ctx.GetDal(), c.scopeHash, "same-input") + require.NoError(t, err) + require.NotNil(t, state) + require.Equal(t, dimension, state.SkipCursor) + require.EqualValues(t, 1, state.TaskID) + completed, err := c.prepareCheckpoint() + require.NoError(t, err) + require.False(t, completed) + } +} + +// Opt-in destructive-process test. Use a dedicated disposable MySQL database. +// Run with DEVLAKE_CRASH_PHASE=crash, wait for CRASH_READY, SIGKILL the +// container, then rerun the same binary with DEVLAKE_CRASH_PHASE=resume. +// DEVLAKE_CRASH_SCENARIO is single, multi, transaction (kill before page +// commit), batch (non-paginated inputs), or incremental (preserve existing raw +// rows). The DB survives both runs. +// Afterwards repeat verifies the completed task makes no requests, and fresh +// uses a new task ID to verify full sync fetches all records again. +func TestGraphqlCollectorMySQLCrash(t *testing.T) { + dsn := os.Getenv("DEVLAKE_CRASH_DSN") + if dsn == "" { + t.Skip("dedicated crash-test database not configured") + } + phase, scenario := os.Getenv("DEVLAKE_CRASH_PHASE"), os.Getenv("DEVLAKE_CRASH_SCENARIO") + require.Contains(t, []string{"crash", "resume", "repeat", "fresh"}, phase) + require.Contains(t, []string{"single", "multi", "transaction", "batch", "incremental", "initialize", "final-page", "complete"}, scenario) + db, err := openGraphqlCrashDB(dsn) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + defer sqlDB.Close() + require.NoError(t, db.AutoMigrate(&graphqlCrashRequest{})) + if phase == "crash" && (scenario == "transaction" || scenario == "initialize" || scenario == "final-page" || scenario == "complete") { + require.NoError(t, db.Callback().Create().Before("gorm:create").Register("crash_before_checkpoint", func(tx *gorm.DB) { + state, ok := tx.Statement.Dest.(*GraphqlCollectorState) + if ok && ((scenario == "transaction" && state.InputHash != "" && state.SkipCursor == "page-1") || + (scenario == "initialize" && state.InputHash == "" && !state.Completed) || + (scenario == "final-page" && state.InputHash != "" && state.InputHash != "__raw__" && state.Completed) || + (scenario == "complete" && state.InputHash == "" && state.Completed)) { + fmt.Println("CRASH_READY: raw rows written but page transaction NOT committed") + select {} + } + })) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Variables struct { + After *string + Input string + Members []string + } + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + cursor := "" + if req.Variables.After != nil { + cursor = *req.Variables.After + } + members, _ := json.Marshal(req.Variables.Members) + if err := db.Exec("INSERT INTO crash_requests (scenario, phase, input_id, cursor_value, input_members) VALUES (?, ?, ?, ?, ?)", scenario, phase, req.Variables.Input, cursor, string(members)).Error; err != nil { + t.Error(err) + return + } + if phase == "crash" && req.Variables.Input == "b" && (((scenario == "single" || scenario == "multi" || scenario == "incremental") && cursor == "page-1") || scenario == "batch") { + fmt.Println("CRASH_READY: checkpoint committed; next response withheld") + <-r.Context().Done() + return + } + id, next := 1, false + if req.Variables.Input == "b" { + id = 2 + next = cursor == "" + if !next { + id = 3 + } + } + if scenario == "batch" { + next = false + } + nodes := []map[string]int{{"id": id}} + if scenario == "batch" { + nodes = nil + for _, member := range req.Variables.Members { + nodes = append(nodes, map[string]int{"id": map[string]int{"a": 1, "a2": 10, "b": 2, "b2": 20}[member]}) + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]interface{}{"items": map[string]interface{}{ + "nodes": nodes, "pageInfo": GraphqlQueryPageInfo{EndCursor: "page-1", HasNextPage: next}, + }}}) + })) + defer server.Close() + taskID := uint64(9142) + if phase == "fresh" { + taskID++ + } + ctx, cancel := context.WithCancel(plugin.WithTaskID(context.Background(), taskID)) + defer cancel() + client := &GraphqlAsyncClient{ctx: ctx, cancel: cancel, client: graphql.NewClient(server.URL, server.Client()), logger: unithelper.DummyLogger(), rateExhaustCond: sync.NewCond(&sync.Mutex{}), rateRemaining: 1000, maxRetry: 1} + inputs := []string{"b"} + if scenario == "multi" { + inputs = []string{"a", "b"} + } + inputStep := 1 + if scenario == "batch" { + inputs = []string{"a", "a2", "b", "b2"} + inputStep = 2 + } + taskCtx := unithelper.DummySubTaskContext(dalgorm.NewDalgorm(db)) + taskCtx.On("GetContext").Return(ctx) + collector, err := NewGraphqlCollector(GraphqlCollectorArgs{ + RawDataSubTaskArgs: RawDataSubTaskArgs{Ctx: taskCtx, Table: "crash_" + scenario, Params: "repo"}, + GraphqlClient: client, Input: &crashInputs{values: inputs}, InputStep: inputStep, PageSize: 1, + BuildQuery: func(req *GraphqlRequestData) (interface{}, map[string]interface{}, error) { + input := req.Input + var members []string + if scenario == "batch" { + input = req.Input.([]interface{})[0] + for _, member := range req.Input.([]interface{}) { + members = append(members, member.(string)) + } + } + return &resumeTestQuery{}, map[string]interface{}{"after": (*graphql.String)(req.Pager.SkipCursor), "input": graphql.String(input.(string)), "members": members}, nil + }, + GetPageInfo: func(q interface{}, _ *GraphqlCollectorArgs) (*GraphqlQueryPageInfo, error) { + return &q.(*resumeTestQuery).Items.PageInfo, nil + }, + ResponseParser: func(q interface{}) ([]json.RawMessage, deverrors.Error) { + var rows []json.RawMessage + for _, node := range q.(*resumeTestQuery).Items.Nodes { + data, err := json.Marshal(node) + if err != nil { + return nil, deverrors.Convert(err) + } + rows = append(rows, data) + } + return rows, nil + }, + }) + require.NoError(t, err) + if scenario == "incremental" { + collector.args.Incremental = true + if phase == "crash" { + require.NoError(t, db.Table(collector.table).AutoMigrate(&RawData{})) + require.NoError(t, db.Table(collector.table).Create(&RawData{Params: collector.params, Data: json.RawMessage(`{"ID":99}`)}).Error) + } + } + if scenario == "batch" { + collector.args.GetPageInfo = nil + } + require.NoError(t, collector.Execute()) + require.NotEqual(t, "crash", phase, "crash phase must be killed before completing") + var rows []RawData + require.NoError(t, db.Table(collector.table).Order("id").Find(&rows).Error) + var ids []int + for _, row := range rows { + var node struct{ ID int } + require.NoError(t, json.Unmarshal(row.Data, &node)) + ids = append(ids, node.ID) + } + expected := []int{2, 3} + if scenario == "incremental" { + expected = []int{99, 2, 3} + if phase == "fresh" { + expected = append(expected, 2, 3) + } + } + if scenario == "multi" { + expected = []int{1, 2, 3} + } + if scenario == "batch" { + expected = []int{1, 10, 2, 20} + } + require.Equal(t, expected, ids, "raw records must survive restart without duplication or loss") + var stateCount int64 + require.NoError(t, db.Model(&GraphqlCollectorState{}).Where("raw_data_table = ? AND completed = ? AND input_hash = ?", collector.table, true, "").Count(&stateCount).Error) + require.EqualValues(t, 1, stateCount, "completed run marker must survive until the next task") + var requests []struct { + InputID string + CursorValue string + InputMembers string + } + require.NoError(t, db.Table("crash_requests").Where("scenario = ? AND phase = ?", scenario, phase).Order("id").Find(&requests).Error) + if phase == "repeat" { + require.Empty(t, requests) + return + } + if phase == "fresh" { + count := 2 + if scenario == "multi" { + count = 3 + } + require.Len(t, requests, count, "new task must collect all pages") + require.Empty(t, requests[0].CursorValue) + return + } + if scenario == "complete" { + require.Empty(t, requests, "all inputs already committed; only completion marker needs retry") + } else if scenario == "transaction" || scenario == "initialize" { + require.Len(t, requests, 2, "uncommitted page must be fetched again") + require.Empty(t, requests[0].CursorValue) + require.Equal(t, "page-1", requests[1].CursorValue) + } else { + require.Len(t, requests, 1, "only unfinished input/page should be fetched") + require.Equal(t, "b", requests[0].InputID) + if scenario == "batch" { + require.Empty(t, requests[0].CursorValue) + require.JSONEq(t, `["b","b2"]`, requests[0].InputMembers) + } else { + require.Equal(t, "page-1", requests[0].CursorValue) + } + } +} diff --git a/backend/helpers/pluginhelper/api/graphql_collector_state.go b/backend/helpers/pluginhelper/api/graphql_collector_state.go new file mode 100644 index 00000000000..d29b96e6f7f --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_collector_state.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models" +) + +// GraphqlCollectorState tracks a collector run (empty InputHash) or an input. +// Completed input rows survive until the entire collector finishes. The run row +// survives completion until the next task, closing the crash window between +// finishing collection and the runner recording subtask success. +type GraphqlCollectorState = models.GraphqlCollectorState + +func graphqlCollectorInputHash(inputJSON string) string { + hash := sha256.Sum256([]byte(inputJSON)) + return hex.EncodeToString(hash[:]) +} + +func loadGraphqlCollectorState(db dal.Dal, scopeHash, inputHash string) (*GraphqlCollectorState, errors.Error) { + state := &GraphqlCollectorState{} + err := db.First(state, dal.Where("scope_hash = ? AND input_hash = ?", scopeHash, inputHash)) + if err != nil { + if db.IsErrorNotFound(err) { + return nil, nil + } + return nil, errors.Default.Wrap(err, "failed to load graphql collector state") + } + return state, nil +} + +func saveGraphqlCollectorState(db dal.Dal, state *GraphqlCollectorState) errors.Error { + return db.CreateOrUpdate(state) +} + +func (collector *GraphqlCollector) checkpoint(inputHash string) *GraphqlCollectorState { + return &GraphqlCollectorState{ + ScopeHash: collector.scopeHash, InputHash: inputHash, TaskID: collector.taskID, + RawDataTable: collector.table, RawDataParams: collector.params, + Contract: collector.args.checkpointContract, InputDigest: collector.args.inputDigest, + RawCount: collector.expectedRawCount, + } +} + +// Shared by every collector writing the same raw scope, including nested ones. +func (collector *GraphqlCollector) rawCheckpoint() *GraphqlCollectorState { + state := collector.checkpoint("__raw__") + identity, _ := json.Marshal([]string{collector.table, collector.params}) + state.ScopeHash = graphqlCollectorInputHash(string(identity)) + return state +} + +// prepareCheckpoint atomically starts a new run (including a full-sync flush) +// or restores the same persisted task. Task-less callers keep legacy behavior. +func (collector *GraphqlCollector) prepareCheckpoint() (bool, errors.Error) { + db := collector.args.Ctx.GetDal() + if collector.taskID == 0 { + if !collector.args.Incremental { + return false, db.Delete(&RawData{}, dal.From(collector.table), dal.Where("params = ?", collector.params)) + } + return false, nil + } + identity, err := json.Marshal([]interface{}{collector.table, collector.params, collector.args.Ctx.GetName(), collector.args.checkpointIndex}) + if err != nil { + return false, errors.Convert(err) + } + collector.scopeHash = graphqlCollectorInputHash(string(identity)) + if err := db.AutoMigrate(&GraphqlCollectorState{}); err != nil { + return false, err + } + state, stateErr := loadGraphqlCollectorState(db, collector.scopeHash, "") + if stateErr != nil { + return false, stateErr + } + var rawState *GraphqlCollectorState + if collector.args.checkpointContract != "" { + key := collector.rawCheckpoint() + var err errors.Error + rawState, err = loadGraphqlCollectorState(db, key.ScopeHash, key.InputHash) + if err != nil { + return false, err + } + if rawState != nil && rawState.TaskID > collector.taskID { + return false, errors.BadInput.New("GraphQL task was superseded by a newer task") + } + } + if state != nil && state.TaskID == collector.taskID { + if collector.args.checkpointContract != "" { + if state.Contract != collector.args.checkpointContract { + return false, errors.BadInput.New("GraphQL collector version/config changed; start a new task") + } + if !state.Completed && state.InputDigest != collector.args.inputDigest { + return false, errors.BadInput.New("GraphQL input changed since interruption; start a new task") + } + count, err := db.Count(dal.From(collector.table), dal.Where("params = ?", collector.params)) + if err != nil { + return false, err + } + if rawState == nil || rawState.TaskID != collector.taskID || rawState.RawCount == nil || count != *rawState.RawCount { + return false, errors.BadInput.New("GraphQL raw data no longer matches checkpoint; start a new task") + } + collector.expectedRawCount = rawState.RawCount + } + return state.Completed, nil + } + tx := db.Begin() + defer tx.Rollback() + if !collector.args.Incremental { + if err := tx.Delete(&RawData{}, dal.From(collector.table), dal.Where("params = ?", collector.params)); err != nil { + return false, err + } + } + if err := tx.Delete(&GraphqlCollectorState{}, dal.Where("scope_hash = ?", collector.scopeHash)); err != nil { + return false, err + } + if collector.args.checkpointContract != "" { + count, err := tx.Count(dal.From(collector.table), dal.Where("params = ?", collector.params)) + if err != nil { + return false, err + } + collector.expectedRawCount = &count + } + if err := saveGraphqlCollectorState(tx, collector.checkpoint("")); err != nil { + return false, err + } + if collector.args.checkpointContract != "" { + if err := saveGraphqlCollectorState(tx, collector.rawCheckpoint()); err != nil { + return false, err + } + } + return false, tx.Commit() +} + +func (collector *GraphqlCollector) completeCheckpoint() errors.Error { + if collector.taskID == 0 { + return nil + } + tx := collector.args.Ctx.GetDal().Begin() + defer tx.Rollback() + state := collector.checkpoint("") + state.Completed = true + if err := saveGraphqlCollectorState(tx, state); err != nil { + return err + } + if err := tx.Delete(&GraphqlCollectorState{}, dal.Where("scope_hash = ? AND input_hash <> ?", collector.scopeHash, "")); err != nil { + return err + } + return tx.Commit() +} diff --git a/backend/helpers/pluginhelper/api/graphql_collector_test.go b/backend/helpers/pluginhelper/api/graphql_collector_test.go index fcab4dce098..d05d844be9b 100644 --- a/backend/helpers/pluginhelper/api/graphql_collector_test.go +++ b/backend/helpers/pluginhelper/api/graphql_collector_test.go @@ -18,10 +18,25 @@ limitations under the License. package api import ( + "context" + "encoding/json" "errors" + "net/http" + "net/http/httptest" + "reflect" + "sync" "testing" + "time" + "github.com/apache/devlake/core/dal" + deverrors "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + "github.com/merico-ai/graphql" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) func TestIsIgnorableGraphqlQueryError(t *testing.T) { @@ -29,3 +44,428 @@ func TestIsIgnorableGraphqlQueryError(t *testing.T) { assert.False(t, isIgnorableGraphqlQueryError(errors.New("some other graphql error"))) assert.False(t, isIgnorableGraphqlQueryError(nil)) } + +type resumeTestQuery struct { + Items struct { + Nodes []struct{ ID int } + PageInfo GraphqlQueryPageInfo + } `graphql:"items(after: $after)"` +} + +type checkpointTestTx struct { + dal.Dal + committed bool + rolledBack bool + commitErr deverrors.Error +} + +func (tx *checkpointTestTx) Commit() deverrors.Error { + tx.committed = tx.commitErr == nil + return tx.commitErr +} +func (tx *checkpointTestTx) Rollback() deverrors.Error { tx.rolledBack = !tx.committed; return nil } +func (tx *checkpointTestTx) LockTables(dal.LockTables) deverrors.Error { return nil } +func (tx *checkpointTestTx) UnlockTables() deverrors.Error { return nil } + +func newResumeTestCollector(t *testing.T, db *mockdal.Dal, handler http.HandlerFunc) *GraphqlCollector { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + ctx, cancel := context.WithCancel(plugin.WithTaskID(context.Background(), 1)) + t.Cleanup(cancel) + logger := unithelper.DummyLogger() + logger.On("Warn", mock.Anything, mock.Anything, mock.Anything).Maybe() + client := &GraphqlAsyncClient{ + ctx: ctx, cancel: cancel, client: graphql.NewClient(server.URL, server.Client()), + logger: logger, rateExhaustCond: sync.NewCond(&sync.Mutex{}), rateRemaining: 1000, maxRetry: 1, + } + field, ok := reflect.TypeOf(RawData{}).FieldByName("ID") + require.True(t, ok) + db.On("GetPrimaryKeyFields", mock.Anything).Return([]reflect.StructField{field}) + taskCtx := unithelper.DummySubTaskContext(db) + taskCtx.On("GetContext").Return(ctx) + collector, err := NewGraphqlCollector(GraphqlCollectorArgs{ + RawDataSubTaskArgs: RawDataSubTaskArgs{Ctx: taskCtx, Table: "resume_test", Params: "repo"}, + GraphqlClient: client, PageSize: 1, + BuildQuery: func(req *GraphqlRequestData) (interface{}, map[string]interface{}, error) { + return &resumeTestQuery{}, map[string]interface{}{"after": (*graphql.String)(req.Pager.SkipCursor)}, nil + }, + GetPageInfo: func(q interface{}, _ *GraphqlCollectorArgs) (*GraphqlQueryPageInfo, error) { + return &q.(*resumeTestQuery).Items.PageInfo, nil + }, + ResponseParser: func(q interface{}) ([]json.RawMessage, deverrors.Error) { + var rows []json.RawMessage + for _, node := range q.(*resumeTestQuery).Items.Nodes { + row, err := json.Marshal(node) + require.NoError(t, err) + rows = append(rows, row) + } + return rows, nil + }, + }) + require.NoError(t, err) + collector.scopeHash = "scope" + return collector +} + +func resumeTestResponse(w http.ResponseWriter, next bool) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]interface{}{"items": map[string]interface{}{ + "nodes": []map[string]int{{"id": 1}}, "pageInfo": GraphqlQueryPageInfo{EndCursor: "page-1", HasNextPage: next}, + }}}) +} + +func expectInput(db *mockdal.Dal, input string, state *GraphqlCollectorState, readErr deverrors.Error) { + db.On("First", mock.Anything, []dal.Clause{dal.Where("scope_hash = ? AND input_hash = ?", "scope", graphqlCollectorInputHash(input))}). + Run(func(args mock.Arguments) { + if state != nil { + *args.Get(0).(*GraphqlCollectorState) = *state + } + }).Return(readErr).Once() + if readErr != nil { + db.On("IsErrorNotFound", readErr).Return(false).Once() + } +} + +func TestGraphqlPageCountAccounting(t *testing.T) { + for _, mode := range []string{"append", "empty", "delete", "commit-error"} { + t.Run(mode, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(w http.ResponseWriter, _ *http.Request) { resumeTestResponse(w, false) }) + count := int64(100) + c.expectedRawCount = &count + c.args.checkpointContract = "contract" + expectInput(db, "null", &GraphqlCollectorState{TaskID: 1}, nil) + tx := &checkpointTestTx{Dal: db} + db.On("Begin").Return(tx).Once() + expected := int64(101) + if mode == "empty" { + c.args.ResponseParser = func(interface{}) ([]json.RawMessage, deverrors.Error) { return nil, nil } + expected = 100 + } else { + db.On("Create", mock.Anything, mock.Anything).Return(nil).Once() + } + if mode == "delete" { + parser := c.args.ResponseParser + c.args.ResponseParser = nil + c.args.ResponseParserWithDal = func(q interface{}, _ dal.Dal) ([]json.RawMessage, deverrors.Error) { return parser(q) } + expected = 99 + db.On("Count", mock.Anything).Return(expected, nil).Once() + } + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { return s.InputHash == graphqlCollectorInputHash("null") })).Return(nil).Once() + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { + return s.InputHash == "__raw__" && s.RawCount != nil && *s.RawCount == expected + })).Return(nil).Once() + if mode == "commit-error" { + tx.commitErr = deverrors.Default.New("commit failed") + } + c.exec(nil) + c.args.GraphqlClient.Wait() + require.Equal(t, mode == "commit-error", c.HasError()) + if mode == "commit-error" { + require.EqualValues(t, 100, *c.expectedRawCount) + } else { + require.Equal(t, expected, *c.expectedRawCount) + } + if mode != "delete" { + db.AssertNotCalled(t, "Count", mock.Anything) + } + }) + } +} + +func TestGraphqlCollectorResumeAndCompleteInputs(t *testing.T) { + for _, mode := range []string{"resume", "completed", "batching", "finish-with-records", "parser-error", "http-error", "db-read-error"} { + t.Run(mode, func(t *testing.T) { + db := mockdal.NewDal(t) + requests := 0 + collector := newResumeTestCollector(t, db, func(w http.ResponseWriter, r *http.Request) { + requests++ + var request struct{ Variables map[string]interface{} } + assert.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + if mode == "resume" { + assert.Equal(t, "saved", request.Variables["after"]) + } + if mode == "http-error" { + http.Error(w, "interrupted", 503) + return + } + resumeTestResponse(w, mode == "finish-with-records" || mode == "parser-error") + }) + state := &GraphqlCollectorState{TaskID: 1, SkipCursor: "saved", Completed: mode == "completed"} + var readErr deverrors.Error + if mode == "db-read-error" { + readErr = deverrors.Default.New("read failed") + } + expectInput(db, "null", state, readErr) + if mode == "batching" { + collector.args.GetPageInfo = nil + } + if mode == "finish-with-records" || mode == "parser-error" { + collector.args.ResponseParser = func(interface{}) ([]json.RawMessage, deverrors.Error) { + err := ErrFinishCollect + if mode == "parser-error" { + err = deverrors.Default.New("parser failed") + } + return []json.RawMessage{json.RawMessage(`{"id":1}`)}, err + } + } + success := mode == "resume" || mode == "batching" || mode == "finish-with-records" + tx := &checkpointTestTx{Dal: db} + if mode == "parser-error" { + db.On("Begin").Return(tx).Once() + } + if success { + db.On("Begin").Return(tx).Once() + raw := db.On("Create", mock.AnythingOfType("*api.RawData"), mock.Anything).Return(nil).Once() + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { + return s.Completed && s.TaskID == 1 && s.InputHash == graphqlCollectorInputHash("null") + })).Return(nil).Once().NotBefore(raw) + } + collector.exec(nil) + collector.args.GraphqlClient.Wait() + assert.Equal(t, !success && mode != "completed", collector.HasError()) + if mode == "completed" || mode == "db-read-error" { + assert.Zero(t, requests) + } else { + assert.Equal(t, 1, requests) + } + if success { + assert.True(t, tx.committed) + } + }) + } +} + +func TestGraphqlCollectorInputIsolation(t *testing.T) { + db := mockdal.NewDal(t) + var got []interface{} + collector := newResumeTestCollector(t, db, func(w http.ResponseWriter, r *http.Request) { + var request struct{ Variables map[string]interface{} } + assert.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + got = append(got, request.Variables["after"]) + resumeTestResponse(w, false) + }) + for _, input := range []string{"a", "b"} { + encoded, _ := json.Marshal(input) + expectInput(db, string(encoded), &GraphqlCollectorState{TaskID: 1, SkipCursor: "cursor-" + input}, nil) + db.On("Begin").Return(&checkpointTestTx{Dal: db}).Once() + db.On("Create", mock.Anything, mock.Anything).Return(nil).Once() + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { + return s.InputHash == graphqlCollectorInputHash(string(encoded)) && s.Completed + })).Return(nil).Once() + collector.exec(input) + collector.args.GraphqlClient.Wait() + } + assert.False(t, collector.HasError()) + assert.Equal(t, []interface{}{"cursor-a", "cursor-b"}, got) +} + +func TestGraphqlCollectorCheckpointFailures(t *testing.T) { + for _, stage := range []string{"raw", "checkpoint", "commit"} { + t.Run(stage, func(t *testing.T) { + db := mockdal.NewDal(t) + requests := 0 + collector := newResumeTestCollector(t, db, func(w http.ResponseWriter, r *http.Request) { requests++; resumeTestResponse(w, true) }) + expectInput(db, "null", &GraphqlCollectorState{TaskID: 1}, nil) + dbErr := deverrors.Default.New("injected failure") + tx := &checkpointTestTx{Dal: db} + if stage == "commit" { + tx.commitErr = dbErr + } + db.On("Begin").Return(tx).Once() + var rawErr, checkpointErr deverrors.Error + if stage == "raw" { + rawErr = dbErr + } + if stage == "checkpoint" { + checkpointErr = dbErr + } + raw := db.On("Create", mock.Anything, mock.Anything).Return(rawErr).Once() + if stage != "raw" { + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { return !s.Completed && s.SkipCursor == "page-1" })).Return(checkpointErr).Once().NotBefore(raw) + } + collector.exec(nil) + collector.args.GraphqlClient.Wait() + require.True(t, collector.HasError()) + assert.True(t, tx.rolledBack) + assert.False(t, tx.committed) + assert.Equal(t, 1, requests) + }) + } +} + +func TestGraphqlCollectorRunIdentity(t *testing.T) { + for _, mode := range []string{"resume", "completed", "new-task", "taskless", "read-error"} { + t.Run(mode, func(t *testing.T) { + db := mockdal.NewDal(t) + collector := newResumeTestCollector(t, db, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected HTTP") }) + tx := &checkpointTestTx{Dal: db} + if mode == "taskless" { + collector.taskID = 0 + db.On("Delete", mock.AnythingOfType("*api.RawData"), mock.Anything).Return(nil).Once() + } else { + db.On("AutoMigrate", mock.Anything).Return(nil).Once() + var readErr deverrors.Error + if mode == "read-error" { + readErr = deverrors.Default.New("read failed") + db.On("IsErrorNotFound", readErr).Return(false).Once() + } + db.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + s := args.Get(0).(*GraphqlCollectorState) + s.TaskID = 1 + if mode == "new-task" { + s.TaskID = 99 + } + s.Completed = mode == "completed" + }).Return(readErr).Once() + if mode == "new-task" { + db.On("Begin").Return(tx).Once() + db.On("Delete", mock.AnythingOfType("*api.RawData"), mock.Anything).Return(nil).Once() + db.On("Delete", mock.AnythingOfType("*models.GraphqlCollectorState"), mock.Anything).Return(nil).Once() + db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { return s.TaskID == 1 && s.InputHash == "" && !s.Completed })).Return(nil).Once() + } + } + completed, err := collector.prepareCheckpoint() + assert.Equal(t, mode == "read-error", err != nil) + assert.Equal(t, mode == "completed", completed) + if mode == "new-task" { + assert.True(t, tx.committed) + } + }) + } +} + +func TestGraphqlCollectorCompletionTransaction(t *testing.T) { + db := mockdal.NewDal(t) + collector := newResumeTestCollector(t, db, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected HTTP") }) + tx := &checkpointTestTx{Dal: db} + db.On("Begin").Return(tx).Once() + marker := db.On("CreateOrUpdate", mock.MatchedBy(func(s *GraphqlCollectorState) bool { return s.Completed && s.InputHash == "" && s.TaskID == 1 })).Return(nil).Once() + db.On("Delete", mock.Anything, []dal.Clause{dal.Where("scope_hash = ? AND input_hash <> ?", "scope", "")}).Return(nil).Once().NotBefore(marker) + require.NoError(t, collector.completeCheckpoint()) + assert.True(t, tx.committed) +} + +func TestGraphqlCancelledNextPage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := &GraphqlAsyncClient{ctx: ctx} + reported := make(chan error, 1) + client.NextTick(func() deverrors.Error { t.Error("cancelled page executed"); return nil }, func(err error) { reported <- err }) + done := make(chan struct{}) + go func() { client.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("cancelled request leaked its wait group") + } + require.ErrorIs(t, <-reported, context.Canceled) +} + +// Override only the subtask name while retaining the usual collector harness. +type namedCheckpointContext struct { + plugin.SubTaskContext + name string +} + +func (c namedCheckpointContext) GetName() string { return c.name } + +func TestGraphqlCollectorScopeIsolation(t *testing.T) { + scopes := map[string]bool{} + for _, dimension := range []string{"base", "table", "params", "subtask", "index"} { + t.Run(dimension, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("unexpected HTTP") }) + switch dimension { + case "table": + c.table += "_other" + case "params": + c.params = "other-repository" + case "subtask": + c.args.Ctx = namedCheckpointContext{c.args.Ctx, "other-subtask"} + case "index": + c.args.checkpointIndex++ + } + db.On("AutoMigrate", mock.Anything).Return(nil).Once() + db.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + *args.Get(0).(*GraphqlCollectorState) = GraphqlCollectorState{TaskID: 1, Completed: true} + }).Return(nil).Once() + completed, err := c.prepareCheckpoint() + require.NoError(t, err) + require.True(t, completed) + assert.False(t, scopes[c.scopeHash], "independent collectors must not share checkpoints") + scopes[c.scopeHash] = true + }) + } +} + +func TestGraphqlCollectorIncrementalInitialization(t *testing.T) { + for _, taskID := range []uint64{0, 1} { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("unexpected HTTP") }) + c.taskID, c.args.Incremental = taskID, true + if taskID != 0 { + db.On("AutoMigrate", mock.Anything).Return(nil).Once() + db.On("First", mock.Anything, mock.Anything).Return(nil).Once() + db.On("Begin").Return(&checkpointTestTx{Dal: db}).Once() + db.On("Delete", mock.AnythingOfType("*models.GraphqlCollectorState"), mock.Anything).Return(nil).Once() + db.On("CreateOrUpdate", mock.Anything).Return(nil).Once() + } + completed, err := c.prepareCheckpoint() + require.NoError(t, err) + require.False(t, completed) + // No raw-data Delete expectation: attempting to flush fails this test. + } +} + +func TestGraphqlCollectorLifecycleFailures(t *testing.T) { + for _, operation := range []string{"prepare", "complete"} { + for _, stage := range []string{"raw-delete", "state-delete", "state-save", "commit"} { + if operation == "complete" && stage == "raw-delete" { + continue + } + t.Run(operation+"/"+stage, func(t *testing.T) { + db := mockdal.NewDal(t) + c := newResumeTestCollector(t, db, func(http.ResponseWriter, *http.Request) { t.Error("unexpected HTTP") }) + failure := deverrors.Default.New("injected lifecycle failure") + tx := &checkpointTestTx{Dal: db} + if stage == "commit" { + tx.commitErr = failure + } + db.On("Begin").Return(tx).Once() + var rawErr, deleteErr, saveErr deverrors.Error + if stage == "raw-delete" { + rawErr = failure + } + if stage == "state-delete" { + deleteErr = failure + } + if stage == "state-save" { + saveErr = failure + } + if operation == "prepare" { + db.On("AutoMigrate", mock.Anything).Return(nil).Once() + db.On("First", mock.Anything, mock.Anything).Return(nil).Once() + db.On("Delete", mock.AnythingOfType("*api.RawData"), mock.Anything).Return(rawErr).Once() + if rawErr == nil { + db.On("Delete", mock.AnythingOfType("*models.GraphqlCollectorState"), mock.Anything).Return(deleteErr).Once() + } + if rawErr == nil && deleteErr == nil { + db.On("CreateOrUpdate", mock.Anything).Return(saveErr).Once() + } + _, err := c.prepareCheckpoint() + require.Error(t, err) + } else { + db.On("CreateOrUpdate", mock.Anything).Return(saveErr).Once() + if saveErr == nil { + db.On("Delete", mock.Anything, mock.Anything).Return(deleteErr).Once() + } + require.Error(t, c.completeCheckpoint()) + } + assert.False(t, tx.committed) + assert.True(t, tx.rolledBack) + }) + } + } +} diff --git a/backend/helpers/pluginhelper/api/graphql_database_guard_test.go b/backend/helpers/pluginhelper/api/graphql_database_guard_test.go new file mode 100644 index 00000000000..b84043739d3 --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_database_guard_test.go @@ -0,0 +1,329 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + "github.com/apache/devlake/impls/dalgorm" + "github.com/merico-ai/graphql" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func checkpointRealDB(t *testing.T) *gorm.DB { + t.Helper() + dsn := os.Getenv("DEVLAKE_CRASH_DSN") + if dsn == "" { + t.Skip("dedicated database not configured") + } + db, err := openGraphqlCrashDB(dsn) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + require.NoError(t, db.AutoMigrate(&models.CollectorLatestState{})) + return db +} + +func realCheckpointArgs(t *testing.T, db *gorm.DB, table string, task uint64) GraphqlCollectorArgs { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"items":{"nodes":[{"id":1},{"id":2}],"pageInfo":{"endCursor":"","hasNextPage":false}}}}`)) + })) + t.Cleanup(upstream.Close) + ctx, cancel := context.WithCancel(plugin.WithTaskID(context.Background(), task)) + t.Cleanup(cancel) + taskCtx := unithelper.DummySubTaskContext(dalgorm.NewDalgorm(db)) + taskCtx.On("GetContext").Return(ctx) + client := &GraphqlAsyncClient{ctx: ctx, cancel: cancel, client: graphql.NewClient(upstream.URL, upstream.Client()), logger: unithelper.DummyLogger(), rateExhaustCond: sync.NewCond(&sync.Mutex{}), rateRemaining: 1000, maxRetry: 1} + return GraphqlCollectorArgs{ + RawDataSubTaskArgs: RawDataSubTaskArgs{Ctx: taskCtx, Table: table, Params: "repo"}, + GraphqlClient: client, PageSize: 2, + BuildQuery: func(req *GraphqlRequestData) (interface{}, map[string]interface{}, error) { + return &resumeTestQuery{}, map[string]interface{}{"after": (*graphql.String)(req.Pager.SkipCursor)}, nil + }, + ResponseParser: func(q interface{}) ([]json.RawMessage, errors.Error) { + var rows []json.RawMessage + for _, n := range q.(*resumeTestQuery).Items.Nodes { + b, err := json.Marshal(n) + if err != nil { + return nil, errors.Convert(err) + } + rows = append(rows, b) + } + return rows, nil + }, + } +} + +func rawCheckpointIDs(t *testing.T, db *gorm.DB, table string) []int { + t.Helper() + var rows []RawData + require.NoError(t, db.Table(table).Order("id").Find(&rows).Error) + ids := []int{} + for _, row := range rows { + var n struct{ ID int } + require.NoError(t, json.Unmarshal(row.Data, &n)) + ids = append(ids, n.ID) + } + return ids +} + +func TestGraphqlDatabaseGuard(t *testing.T) { + for _, change := range []string{"input", "order", "config", "binary", "plugin", "raw", "superseded"} { + t.Run(change, func(t *testing.T) { + db := checkpointRealDB(t) + args := realCheckpointArgs(t, db, "guard_"+change, 100) + args.Input = &crashInputs{values: []string{"a", "b"}} + // Initialization commits; the first query fails, retaining the manifest. + build := args.BuildQuery + args.BuildQuery = func(*GraphqlRequestData) (interface{}, map[string]interface{}, error) { + return nil, nil, fmt.Errorf("interrupted") + } + c, err := NewGraphqlCollector(args) + require.NoError(t, err) + require.Error(t, c.Execute()) + args.BuildQuery = build + args.Input = &crashInputs{values: []string{"a", "b"}} + switch change { + case "input": + args.Input = &crashInputs{values: []string{"a", "c"}} + case "order": + args.Input = &crashInputs{values: []string{"b", "a"}} + case "config": + args.PageSize++ + case "plugin": + plugin.RegisterPluginCode("changed-plugin", "updated-shared-object") + args.Ctx = graphqlContextOverride{args.Ctx, plugin.WithTaskCode(args.Ctx.GetContext(), "changed-plugin")} + case "binary": + require.NoError(t, db.Model(&GraphqlCollectorState{}).Where("scope_hash = ? AND input_hash = ?", c.scopeHash, "").Update("contract", "old-binary").Error) + case "raw": + require.NoError(t, db.Table(c.table).Create(&RawData{Params: c.params, Data: json.RawMessage(`{"ID":99}`)}).Error) + case "superseded": + newer := realCheckpointArgs(t, db, "guard_"+change, 101) + n, err := NewGraphqlCollector(newer) + require.NoError(t, err) + require.NoError(t, n.Execute()) + } + retry, err := NewGraphqlCollector(args) + require.NoError(t, err) + require.Error(t, retry.Execute(), "unsafe reuse must not silently mark data complete") + }) + } +} + +func TestGraphqlDatabasePageRollback(t *testing.T) { + for _, failure := range []string{"second-row", "checkpoint", "parser", "commit-ack"} { + t.Run(failure, func(t *testing.T) { + db := checkpointRealDB(t) + args := realCheckpointArgs(t, db, "rollback_"+failure, 200) + args.Table = "rollback_" + map[string]string{"second-row": "row", "checkpoint": "state", "parser": "parser", "commit-ack": "ack"}[failure] + originalParser := args.ResponseParser + args.ResponseParser = nil + require.NoError(t, db.Exec("CREATE TABLE IF NOT EXISTS graphql_parser_effects (name VARCHAR(64) PRIMARY KEY)").Error) + require.NoError(t, db.Exec("INSERT INTO graphql_parser_effects (name) VALUES (?)", failure).Error) + injecting := true + args.ResponseParserWithDal = func(q interface{}, tx dal.Dal) ([]json.RawMessage, errors.Error) { + if err := tx.Delete(nil, dal.From("graphql_parser_effects"), dal.Where("name = ?", failure)); err != nil { + return nil, err + } + if injecting && failure == "parser" { + return nil, errors.Default.New("parser interrupted") + } + return originalParser(q) + } + creates := 0 + require.NoError(t, db.Callback().Create().After("gorm:create").Register("fail_page", func(tx *gorm.DB) { + if !injecting { + return + } + if _, ok := tx.Statement.Dest.(*RawData); ok { + creates++ + if failure == "second-row" && creates == 2 { + tx.AddError(fmt.Errorf("second row failed")) + } + } + if s, ok := tx.Statement.Dest.(*GraphqlCollectorState); ok && s.InputHash != "" && s.InputHash != "__raw__" && failure == "checkpoint" { + tx.AddError(fmt.Errorf("checkpoint failed")) + } + })) + if failure == "commit-ack" { + // Commit succeeds but its acknowledgement is lost. Resume must trust the DB, + // not retry a page based on the caller's error alone. + args.Ctx = graphqlDalOverride{args.Ctx, &uncertainCommitDal{Dal: args.Ctx.GetDal()}} + } + c, err := NewGraphqlCollector(args) + require.NoError(t, err) + require.Error(t, c.Execute()) + if failure != "commit-ack" { + require.Empty(t, rawCheckpointIDs(t, db, c.table)) + var n int64 + require.NoError(t, db.Table("graphql_parser_effects").Where("name = ?", failure).Count(&n).Error) + require.EqualValues(t, 1, n) + } + injecting = false + // Use a clean DAL and new collector as after a process restart. + args.Ctx = realCheckpointArgs(t, db, args.Table, 200).Ctx + retry, err := NewGraphqlCollector(args) + require.NoError(t, err) + require.NoError(t, retry.Execute()) + require.Equal(t, []int{1, 2}, rawCheckpointIDs(t, db, c.table)) + var n int64 + require.NoError(t, db.Table("graphql_parser_effects").Where("name = ?", failure).Count(&n).Error) + require.Zero(t, n) + }) + } +} + +type graphqlDalOverride struct { + plugin.SubTaskContext + db dal.Dal +} + +func TestGraphqlDatabaseTasklessAndEmpty(t *testing.T) { + for _, mode := range []string{"taskless-full", "taskless-incremental", "empty", "finish-empty"} { + t.Run(mode, func(t *testing.T) { + db := checkpointRealDB(t) + args := realCheckpointArgs(t, db, "compat_"+map[string]string{"taskless-full": "full", "taskless-incremental": "inc", "empty": "empty", "finish-empty": "finish"}[mode], 0) + if mode == "taskless-incremental" { + args.Incremental = true + } + if mode == "empty" { + args.Input = &crashInputs{} + } + if mode == "finish-empty" { + args.ResponseParser = func(interface{}) ([]json.RawMessage, errors.Error) { return nil, ErrFinishCollect } + } + for i := 0; i < 2; i++ { + c, err := NewGraphqlCollector(args) + require.NoError(t, err) + require.NoError(t, c.Execute()) + } + raw, err := NewRawDataSubTask(args.RawDataSubTaskArgs) + require.NoError(t, err) + expected := []int{1, 2} + if mode == "taskless-incremental" { + expected = []int{1, 2, 1, 2} + } + if mode == "empty" || mode == "finish-empty" { + expected = []int{} + } + require.Equal(t, expected, rawCheckpointIDs(t, db, raw.table)) + var n int64 + require.NoError(t, db.Model(&GraphqlCollectorState{}).Where("raw_data_table = ?", raw.table).Count(&n).Error) + require.Zero(t, n, "legacy task-less calls must not leave checkpoint state") + }) + } +} +func (c graphqlDalOverride) GetDal() dal.Dal { return c.db } + +type uncertainCommitDal struct { + dal.Dal + commits int +} + +func (d *uncertainCommitDal) Begin() dal.Transaction { + return &uncertainCommitTx{Transaction: d.Dal.Begin(), owner: d} +} + +type uncertainCommitTx struct { + dal.Transaction + owner *uncertainCommitDal +} + +func (tx *uncertainCommitTx) Commit() errors.Error { + err := tx.Transaction.Commit() + if err != nil { + return err + } + tx.owner.commits++ + if tx.owner.commits == 2 { + return errors.Default.New("commit acknowledgement lost") + } + return nil +} + +func TestGraphqlDatabaseNestedResume(t *testing.T) { + t.Run("state-manager-save", func(t *testing.T) { + db := checkpointRealDB(t) + args := realCheckpointArgs(t, db, "manager_save", 301) + injected := func(tx *gorm.DB) { + if _, ok := tx.Statement.Dest.(*models.CollectorLatestState); ok { + tx.AddError(fmt.Errorf("watermark write failed")) + } + } + require.NoError(t, db.Callback().Create().Before("gorm:create").Register("fail_watermark", injected)) + require.NoError(t, db.Callback().Update().Before("gorm:update").Register("fail_watermark", injected)) + m, err := NewStatefulApiCollector(args.RawDataSubTaskArgs) + require.NoError(t, err) + require.NoError(t, m.InitGraphQLCollector(args)) + require.Error(t, m.Execute()) + c := m.nestedCollectors[0].(*GraphqlCollector) + require.Equal(t, []int{1, 2}, rawCheckpointIDs(t, db, c.table)) + require.NoError(t, db.Callback().Create().Remove("fail_watermark")) + require.NoError(t, db.Callback().Update().Remove("fail_watermark")) + args.BuildQuery = func(*GraphqlRequestData) (interface{}, map[string]interface{}, error) { + return nil, nil, fmt.Errorf("completed collector must not query again") + } + m, err = NewStatefulApiCollector(args.RawDataSubTaskArgs) + require.NoError(t, err) + require.NoError(t, m.InitGraphQLCollector(args)) + require.NoError(t, m.Execute()) + require.Equal(t, []int{1, 2}, rawCheckpointIDs(t, db, c.table)) + }) + db := checkpointRealDB(t) + args := realCheckpointArgs(t, db, "nested_resume", 300) + fail := true + makeManager := func() *StatefulApiCollector { + m, err := NewStatefulApiCollector(args.RawDataSubTaskArgs) + require.NoError(t, err) + require.NoError(t, m.InitGraphQLCollector(args)) + second := args + second.ResponseParser = func(interface{}) ([]json.RawMessage, errors.Error) { + if fail { + return nil, errors.Default.New("second collector interrupted") + } + return []json.RawMessage{json.RawMessage(`{"ID":3}`)}, nil + } + require.NoError(t, m.InitGraphQLCollector(second)) + return m + } + first := makeManager() + require.Error(t, first.Execute()) + table := first.nestedCollectors[0].(*GraphqlCollector).table + require.Equal(t, []int{1, 2}, rawCheckpointIDs(t, db, table)) + fail = false + require.NoError(t, makeManager().Execute()) + require.Equal(t, []int{1, 2, 3}, rawCheckpointIDs(t, db, table)) + require.NoError(t, makeManager().Execute()) + require.Equal(t, []int{1, 2, 3}, rawCheckpointIDs(t, db, table)) +} diff --git a/backend/helpers/pluginhelper/api/graphql_scope_cleanup_test.go b/backend/helpers/pluginhelper/api/graphql_scope_cleanup_test.go new file mode 100644 index 00000000000..072b6f8c291 --- /dev/null +++ b/backend/helpers/pluginhelper/api/graphql_scope_cleanup_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "errors" + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "reflect" + "testing" +) + +type checkpointScopePlugin struct{} + +func (checkpointScopePlugin) Name() string { return "checkpoint_scope" } +func (checkpointScopePlugin) Description() string { return "checkpoint scope test" } +func (checkpointScopePlugin) RootPkgPath() string { return "checkpoint_scope" } +func (checkpointScopePlugin) GetTablesInfo() []dal.Tabler { return nil } + +func TestGraphqlCheckpointScopeDeletion(t *testing.T) { + require.NoError(t, plugin.RegisterPlugin("checkpoint_scope", checkpointScopePlugin{})) + for _, exists := range []bool{false, true} { + db := mockdal.NewDal(t) + db.On("AllTables").Return([]string{}, nil) + db.On("HasTable", mock.AnythingOfType("*models.GraphqlCollectorState")).Return(exists) + helper := &GenericScopeApiHelper[any, plugin.ToolLayerScope, any]{db: db, log: unithelper.DummyLogger(), plugin: "checkpoint_scope"} + tables, err := helper.getAffectedTables("checkpoint_scope") + require.NoError(t, err) + if !exists { + require.NotContains(t, tables, models.GraphqlCollectorState{}.TableName()) + continue + } + require.Contains(t, tables, models.GraphqlCollectorState{}.TableName()) + tx := mockdal.NewTransaction(t) + db.On("Begin").Return(tx) + tx.On("Exec", "DELETE FROM _devlake_graphql_collector_states WHERE raw_data_table LIKE ? AND raw_data_params = ?", []interface{}{"_raw_checkpoint_scope%", "repo"}).Return(nil).Once() + tx.On("Commit").Return(nil).Once() + db.On("Count", mock.Anything).Return(int64(0), nil).Once() + require.NoError(t, helper.transactionalDelete([]string{models.GraphqlCollectorState{}.TableName()}, "repo")) + } +} + +func TestGraphqlSQLCursorReadFailure(t *testing.T) { + rows := mockdal.NewRows(t) + failure := errors.New("connection lost while reading cursor") + rows.On("Next").Return(false).Once() + rows.On("Err").Return(failure).Once() + rows.On("Close").Return(nil).Once() + iterator, err := NewDalCursorIterator(nil, rows, reflect.TypeOf(struct{ ID int }{})) + require.NoError(t, err) + staged, _, err := stageGraphqlInput(t.Context(), iterator) + require.ErrorIs(t, err, failure) + require.Nil(t, staged) +} diff --git a/backend/helpers/pluginhelper/api/iterator.go b/backend/helpers/pluginhelper/api/iterator.go index ae40f287a4b..1eb5d473319 100644 --- a/backend/helpers/pluginhelper/api/iterator.go +++ b/backend/helpers/pluginhelper/api/iterator.go @@ -60,6 +60,10 @@ func (c *DalCursorIterator) HasNext() bool { return c.cursor.Next() } +// Err distinguishes an exhausted cursor from a failed read. Optional on +// Iterator for compatibility with existing in-memory/plugin iterators. +func (c *DalCursorIterator) Err() error { return c.cursor.Err() } + // Fetch if batching is disabled, it'll read a single row, otherwise it'll read as many rows up to the batch size, and the // runtime return type will be []interface{}. Note, HasNext needs to have been called before invoking this. func (c *DalCursorIterator) Fetch() (interface{}, errors.Error) { diff --git a/backend/helpers/pluginhelper/api/queue_test.go b/backend/helpers/pluginhelper/api/queue_test.go index 3e0baf16725..214d932911d 100644 --- a/backend/helpers/pluginhelper/api/queue_test.go +++ b/backend/helpers/pluginhelper/api/queue_test.go @@ -58,28 +58,34 @@ func TestQueue(t *testing.T) { require.Equal(t, "b", data) require.Equal(t, q.GetCount(), int64(0)) - empty := false - waited := false + empty := make(chan struct{}) + waited := make(chan struct{}) go func() { + defer close(empty) require.Equal(t, q.GetCountWithWorkingBlock(), int64(1)) data, ok := q.PullWithWorkingBlock().Data().(string) require.True(t, ok) require.Equal(t, data, "c") dataNode := q.PullWithWorkingBlock() require.Equal(t, dataNode, nil) - empty = true }() go func() { time.Sleep(100 * time.Millisecond) q.Push(NewQueueIteratorNode("c")) - waited = true + close(waited) q.Finish(3) }() - for !empty { - time.Sleep(time.Millisecond) + select { + case <-empty: + case <-time.After(5 * time.Second): + t.Fatal("queue consumer did not finish") } - require.True(t, waited) + select { + case <-waited: + default: + t.Fatal("queue producer did not finish") + } } diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go b/backend/helpers/pluginhelper/api/scope_generic_helper.go index ee4d86b0844..f42b76d76be 100644 --- a/backend/helpers/pluginhelper/api/scope_generic_helper.go +++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go @@ -551,7 +551,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) transactionalDelete(t params = []interface{}{rawDataParams} } else { // framework tables: should check plugin, connection and scope - if table == (models.CollectorLatestState{}.TableName()) { + if table == (models.CollectorLatestState{}.TableName()) || table == (models.GraphqlCollectorState{}.TableName()) { // diff sync state where = "raw_data_table LIKE ? AND raw_data_params = ?" } else { @@ -661,6 +661,9 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) getAffectedTables(plu } // additional tables tables = append(tables, models.CollectorLatestState{}.TableName()) + if gs.db.HasTable(&models.GraphqlCollectorState{}) { + tables = append(tables, models.GraphqlCollectorState{}.TableName()) + } } gs.log.Debug("Discovered %d tables used by plugin \"%s\": %v", len(tables), pluginName, tables) return tables, nil diff --git a/backend/helpers/srvhelper/graphql_scope_cleanup_test.go b/backend/helpers/srvhelper/graphql_scope_cleanup_test.go new file mode 100644 index 00000000000..a93f0f25d73 --- /dev/null +++ b/backend/helpers/srvhelper/graphql_scope_cleanup_test.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package srvhelper + +import ( + "github.com/apache/devlake/core/dal" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/helpers/unithelper" + mockdal "github.com/apache/devlake/mocks/core/dal" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "testing" +) + +type checkpointScopePlugin struct{} + +func (checkpointScopePlugin) Name() string { return "checkpoint_scope" } +func (checkpointScopePlugin) Description() string { return "checkpoint scope test" } +func (checkpointScopePlugin) RootPkgPath() string { return "checkpoint_scope" } +func (checkpointScopePlugin) GetTablesInfo() []dal.Tabler { return nil } + +type checkpointScope struct{ plugin.ToolLayerScope } + +func (checkpointScope) ScopeParams() interface{} { return "repo" } +func TestGraphqlCheckpointScopeDeletion(t *testing.T) { + require.NoError(t, plugin.RegisterPlugin("checkpoint_scope", checkpointScopePlugin{})) + db := mockdal.NewDal(t) + db.On("AllTables").Return([]string{}, nil) + db.On("HasTable", mock.AnythingOfType("*models.GraphqlCollectorState")).Return(true) + helper := &ScopeSrvHelper[plugin.ToolLayerConnection, plugin.ToolLayerScope, plugin.ToolLayerScopeConfig]{ + ModelSrvHelper: &ModelSrvHelper[plugin.ToolLayerScope]{db: db, log: unithelper.DummyLogger()}, pluginName: "checkpoint_scope", + } + tx := mockdal.NewTransaction(t) + tx.On("Exec", "DELETE FROM _devlake_graphql_collector_states WHERE raw_data_table LIKE ? AND raw_data_params = ?", []interface{}{"_raw_checkpoint_scope%", `"repo"`}).Return(nil).Once() + tx.On("Exec", mock.MatchedBy(func(sql string) bool { + return sql != "DELETE FROM _devlake_graphql_collector_states WHERE raw_data_table LIKE ? AND raw_data_params = ?" + }), mock.Anything).Return(nil) + helper.deleteScopeData(checkpointScope{}, tx) +} diff --git a/backend/helpers/srvhelper/scope_service_helper.go b/backend/helpers/srvhelper/scope_service_helper.go index 082a3275511..344cbdb7668 100644 --- a/backend/helpers/srvhelper/scope_service_helper.go +++ b/backend/helpers/srvhelper/scope_service_helper.go @@ -241,7 +241,7 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) deleteScopeData(scope plugin.ToolLayer params = []interface{}{rawDataParams} } else { // framework tables: should check plugin, connection and scope - if table == (models.CollectorLatestState{}.TableName()) { + if table == (models.CollectorLatestState{}.TableName()) || table == (models.GraphqlCollectorState{}.TableName()) { // diff sync state where = "raw_data_table LIKE ? AND raw_data_params = ?" } else { @@ -304,6 +304,9 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAffectedTables() ([]string, errors. } // additional tables tables = append(tables, models.CollectorLatestState{}.TableName()) + if scopeSrv.db.HasTable(&models.GraphqlCollectorState{}) { + tables = append(tables, models.GraphqlCollectorState{}.TableName()) + } scopeSrv.log.Debug("Discovered %d tables used by plugin \"%s\": %v", len(tables), scopeSrv.pluginName, tables) return tables, nil } diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go index 20a54741538..4ce446498c1 100644 --- a/backend/plugins/github_graphql/tasks/account_collector.go +++ b/backend/plugins/github_graphql/tasks/account_collector.go @@ -81,6 +81,7 @@ func CollectAccount(taskCtx plugin.SubTaskContext) errors.Error { // and only enriches it from _tool_github_accounts when a row is there. cursor, err := db.Cursor( dal.Select("login"), + dal.Orderby("login ASC"), dal.From(models.GithubRepoAccount{}.TableName()), dal.Where( "repo_github_id = ? and connection_id = ? and login not like ?", diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go index d895c23ecd7..7d99c9c9a8b 100644 --- a/backend/plugins/github_graphql/tasks/issue_collector.go +++ b/backend/plugins/github_graphql/tasks/issue_collector.go @@ -173,6 +173,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { cursor, err := db.Cursor( dal.From(models.GithubIssue{}.TableName()), dal.Where("state = ? AND repo_id = ? AND connection_id=?", "OPEN", data.Options.GithubId, data.Options.ConnectionId), + dal.Orderby("github_id ASC"), ) if err != nil { return err @@ -219,7 +220,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { } return query, variables, nil }, - ResponseParser: func(queryWrapper any) (messages []json.RawMessage, err errors.Error) { + ResponseParserWithDal: func(queryWrapper any, tx dal.Dal) (messages []json.RawMessage, err errors.Error) { query := queryWrapper.(*GraphqlQueryIssueDetailWrapper) v, ok := requestedIssuesByQuery.LoadAndDelete(query) var requestedIssues map[int]missingGithubIssueRef @@ -237,7 +238,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { } missingIssues := findMissingGithubIssues(requestedIssues, issues) if len(missingIssues) > 0 { - err = cleanupMissingGithubIssues(db, taskCtx.GetLogger(), missingIssues) + err = cleanupMissingGithubIssues(tx, taskCtx.GetLogger(), missingIssues) } return }, diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index 76908380607..ba0670c8c7a 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -291,7 +291,8 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { dal.Select("check_suite_node_id"), dal.From(models.GithubRun{}.TableName()), dal.Where("repo_id = ? and connection_id=?", data.Options.GithubId, data.Options.ConnectionId), - dal.Orderby("github_updated_at DESC"), + // Stable tie-breaker keeps batches identical when the task resumes. + dal.Orderby("github_updated_at DESC, check_suite_node_id ASC"), } if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { clauses = append(clauses, dal.Where("github_updated_at > ?", *apiCollector.GetSince())) diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go index 760f9c2786e..482f2b9a7d6 100644 --- a/backend/plugins/github_graphql/tasks/pr_collector.go +++ b/backend/plugins/github_graphql/tasks/pr_collector.go @@ -228,6 +228,7 @@ func CollectPrs(taskCtx plugin.SubTaskContext) errors.Error { cursor, err := db.Cursor( dal.From(models.GithubPullRequest{}.TableName()), dal.Where("state = ? AND repo_id = ? AND connection_id=?", "OPEN", data.Options.GithubId, data.Options.ConnectionId), + dal.Orderby("github_id ASC"), ) if err != nil { return err diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go index cc8ee22264a..6fe2be12992 100644 --- a/backend/plugins/linear/tasks/shared.go +++ b/backend/plugins/linear/tasks/shared.go @@ -35,6 +35,7 @@ func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *t dal.Select("id"), dal.From(&models.LinearIssue{}), dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + dal.Orderby("id ASC"), } if since != nil { clauses = append(clauses, dal.Where("updated_at > ?", *since)) diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go index ad2fc7bc6d0..b1c78827f08 100644 --- a/backend/plugins/linear/tasks/shared_clauses_test.go +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/apache/devlake/core/dal" "github.com/stretchr/testify/assert" ) @@ -29,12 +30,14 @@ import ( // issue, while an incremental run adds an updated_at filter so unchanged issues // are skipped instead of triggering a request each run. func TestIssuesToCollectChildrenClauses(t *testing.T) { - // full sync: no `since` -> select/from/where(connection,team) only + // Full sync includes a stable input ordering for durable checkpoints. full := issuesToCollectChildrenClauses(1, "team-1", nil) - assert.Len(t, full, 3) + assert.Len(t, full, 4) + assert.Contains(t, full, dal.Orderby("id ASC")) // incremental: a `since` adds the updated_at filter clause since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) incremental := issuesToCollectChildrenClauses(1, "team-1", &since) - assert.Len(t, incremental, 4) + assert.Len(t, incremental, 5) + assert.Contains(t, incremental, dal.Orderby("id ASC")) } diff --git a/backend/server/api/github_jobs_resume_test.go b/backend/server/api/github_jobs_resume_test.go new file mode 100644 index 00000000000..62af170384f --- /dev/null +++ b/backend/server/api/github_jobs_resume_test.go @@ -0,0 +1,273 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/apache/devlake/core/config" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/core/runner" + collector "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/impls/logruslog" + githubmodels "github.com/apache/devlake/plugins/github/models" + githubtasks "github.com/apache/devlake/plugins/github/tasks" + jobs "github.com/apache/devlake/plugins/github_graphql/tasks" + server "github.com/apache/devlake/server/api" + "github.com/apache/devlake/server/services" + "github.com/merico-ai/graphql" + "github.com/stretchr/testify/require" +) + +type githubJobsResumePlugin struct{ endpoint, phase, scenario string } + +func (p githubJobsResumePlugin) Name() string { return "github_jobs_resume" } +func (p githubJobsResumePlugin) Description() string { return "Real Github GraphQL Jobs crash fixture" } +func (p githubJobsResumePlugin) RootPkgPath() string { + return "github.com/apache/devlake/plugins/github_graphql" +} +func (p githubJobsResumePlugin) PrepareTaskData(ctx plugin.TaskContext, _ map[string]interface{}) (interface{}, errors.Error) { + client, err := collector.CreateAsyncGraphqlClient(ctx, graphql.NewClient(p.endpoint, http.DefaultClient), ctx.GetLogger(), nil) + if err != nil { + return nil, err + } + return &githubtasks.GithubTaskData{ + Options: &githubtasks.GithubOptions{ConnectionId: 1, GithubId: 2, Name: "fixture/repo"}, + GraphqlClient: client, RegexEnricher: collector.NewRegexEnricher(), + }, nil +} +func (p githubJobsResumePlugin) SubTaskMetas() []plugin.SubTaskMeta { + extract := jobs.ExtractJobsMeta + extract.EntryPoint = func(ctx plugin.SubTaskContext) errors.Error { + if p.phase == "crash" && p.scenario == "extract" { + fmt.Println("CRASH_READY: collection finished, extractor not yet run") + select {} + } + return jobs.ExtractJobs(ctx) + } + return []plugin.SubTaskMeta{jobs.CollectJobsMeta, extract} +} + +// Runs the actual CollectJobs -> ExtractJobs tasks with a real SQL cursor. +// Only the upstream GitHub endpoint and connection preparation are fixtures. +func TestGithubJobsServerResumeCrash(t *testing.T) { + dbURL := os.Getenv("DEVLAKE_SERVER_CRASH_DB_URL") + if dbURL == "" { + t.Skip("dedicated database not configured") + } + phase, scenario := os.Getenv("DEVLAKE_CRASH_PHASE"), os.Getenv("DEVLAKE_JOBS_SCENARIO") + require.Contains(t, []string{"crash", "crash-again", "resume"}, phase) + require.Contains(t, []string{"batch", "page", "extract"}, scenario) + cfg := config.GetConfig() + cfg.Set("DB_URL", dbURL) + cfg.Set("RESUME_PIPELINES", true) + cfg.Set("CONSUME_PIPELINES", true) + cfg.Set("FORCE_MIGRATION", true) + cfg.Set("PIPELINE_MAX_PARALLEL", 1) + cfg.Set("DISABLED_REMOTE_PLUGINS", true) + cfg.Set("PLUGIN_DIR", t.TempDir()) + cfg.Set("LOGGING_DIR", t.TempDir()) + cfg.Set("AUTH_ENABLED", false) + cfg.Set("GITHUB_GRAPHQL_JOB_COLLECTION_MODE", "BATCHING") + cfg.Set("GITHUB_GRAPHQL_JOB_BATCHING_INPUT_STEP", 2) + cfg.Set("GITHUB_GRAPHQL_JOB_BATCHING_PAGE_SIZE", 3) + if scenario == "page" { + cfg.Set("GITHUB_GRAPHQL_JOB_COLLECTION_MODE", "PAGINATING") + cfg.Set("GITHUB_GRAPHQL_JOB_PAGINATING_PAGE_SIZE", 1) + } + cfg.Set(plugin.EncodeKeyEnvStr, "DFLFZLMBBFDDCYWRECDCIYUROPPAKQDFQMMJEFPIKVFVHZBRGAZIHKRJIJZMOHWEVRSCETAGGONPSULGOXITVXISVCQGPSFAOGRDLUANEYDQFBDKVMYYHUZFHYVYGPPT") + db, err := runner.NewGormDb(cfg, logruslog.Global) + require.NoError(t, err) + require.NoError(t, db.Exec("CREATE TABLE IF NOT EXISTS jobs_requests (phase VARCHAR(16), run_id INTEGER, cursor_value VARCHAR(64))").Error) + var original struct { + ID, PipelineId uint64 + Status string + } + committedJobs := map[int]bool{} + if phase != "crash" { + require.NoError(t, db.Table(models.Task{}.TableName()).Take(&original).Error) + require.Equal(t, models.TASK_RUNNING, original.Status) + var committed []collector.RawData + require.NoError(t, db.Table("_raw_github_graphql_jobs").Find(&committed).Error) + for _, raw := range committed { + var job jobs.DbCheckRun + require.NoError(t, json.Unmarshal(raw.Data, &job)) + committedJobs[job.DatabaseId] = true + } + } + nodePattern := regexp.MustCompile(`(?:(\w+)\s*:\s*)?node\s*\(\s*id\s*:\s*\$(\w+)`) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Query string + Variables map[string]interface{} + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + matches := nodePattern.FindAllStringSubmatch(req.Query, -1) + if len(matches) == 0 { + t.Errorf("no node fields in query: %s / %v", req.Query, req.Variables) + http.Error(w, "bad fixture query", 400) + return + } + cursor, _ := req.Variables["skipCursor"].(string) + data := map[string]interface{}{"rateLimit": map[string]int{"cost": 1}} + for _, match := range matches { + field := match[1] + if field == "" { + field = "node" + } + idString, _ := req.Variables[match[2]].(string) + runID, err := strconv.Atoi(strings.TrimPrefix(idString, "suite-")) + if err != nil { + t.Error(err) + return + } + if err := db.Exec("INSERT INTO jobs_requests (phase,run_id,cursor_value) VALUES (?,?,?)", phase, runID, cursor).Error; err != nil { + t.Error(err) + return + } + if (phase == "crash" && ((scenario == "batch" && runID == 3) || (scenario == "page" && runID == 1 && cursor == "p1"))) || (phase == "crash-again" && runID == 1 && cursor == "p2") { + fmt.Println("CRASH_READY: real CollectJobs checkpoint committed") + <-r.Context().Done() + return + } + first, last := 1, 3 + next := false + endCursor := "" + if scenario == "page" { + if cursor != "" { + first, _ = strconv.Atoi(strings.TrimPrefix(cursor, "p")) + first++ + } + last = first + next = first < 3 + endCursor = fmt.Sprintf("p%d", first) + } + nodes := []map[string]interface{}{} + for n := first; n <= last; n++ { + nodes = append(nodes, map[string]interface{}{ + "id": fmt.Sprintf("job-%d", runID*10+n), "databaseId": runID*10 + n, "name": fmt.Sprintf("Job %d", n), + "status": "COMPLETED", "conclusion": "SUCCESS", "startedAt": "2026-01-01T00:00:00Z", "completedAt": "2026-01-01T00:01:00Z", + "steps": map[string]interface{}{"totalCount": 0, "nodes": []interface{}{}}, + }) + } + data[field] = map[string]interface{}{"id": idString, "__typename": "CheckSuite", + "workflowRun": map[string]int{"databaseId": runID}, + "checkRuns": map[string]interface{}{"totalCount": 3, "nodes": nodes, "pageInfo": map[string]interface{}{"endCursor": endCursor, "hasNextPage": next}}, + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": data}) + })) + defer upstream.Close() + p := githubJobsResumePlugin{upstream.URL, phase, scenario} + require.NoError(t, plugin.RegisterPlugin(p.Name(), p)) + server.Init() + services.InitExecuteMigration() + require.Equal(t, services.SERVICE_STATUS_READY, services.CurrentStatus()) + if phase == "crash-again" { + select {} + } + if phase == "crash" { + require.NoError(t, db.AutoMigrate(&githubmodels.GithubRun{}, &githubmodels.GithubJob{})) + sameTime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // Deliberately reverse insertion order with identical updated_at values. + for _, id := range []int{3, 2, 1} { + require.NoError(t, db.Create(&githubmodels.GithubRun{ConnectionId: 1, RepoId: 2, ID: id, CheckSuiteNodeID: fmt.Sprintf("suite-%d", id), GithubUpdatedAt: &sameTime}).Error) + } + _, err := services.CreatePipeline(&models.NewPipeline{Name: "real GitHub Jobs resume", Plan: models.PipelinePlan{{&models.PipelineTask{Plugin: p.Name(), Options: map[string]interface{}{}}}}}, false) + require.NoError(t, err) + select {} + } + require.Eventually(t, func() bool { + var pipeline models.Pipeline + return db.First(&pipeline, original.PipelineId).Error == nil && pipeline.Status == models.TASK_COMPLETED + }, 30*time.Second, 100*time.Millisecond) + var completedPipeline models.Pipeline + require.NoError(t, db.First(&completedPipeline, original.PipelineId).Error) + require.Equal(t, 1, completedPipeline.TotalTasks) + require.Equal(t, 1, completedPipeline.FinishedTasks, "restarts must not inflate task progress") + var tasks []models.Task + require.NoError(t, db.Find(&tasks).Error) + require.Len(t, tasks, 1) + require.Equal(t, original.ID, tasks[0].ID) + var subtasks []models.Subtask + require.NoError(t, db.Where("task_id = ?", original.ID).Find(&subtasks).Error) + require.Len(t, subtasks, 2, "restart must not duplicate subtask records") + for _, subtask := range subtasks { + require.NotNil(t, subtask.FinishedAt) + require.False(t, subtask.IsFailed) + } + var actual []githubmodels.GithubJob + require.NoError(t, db.Order("id").Find(&actual).Error) + require.Len(t, actual, 9) + for i, job := range actual { + require.Equal(t, (i/3+1)*10+i%3+1, job.ID) + require.Equal(t, i/3+1, job.RunID) + require.Equal(t, 2, job.RepoId) + require.EqualValues(t, 1, job.ConnectionId) + require.Equal(t, "SUCCESS", job.Conclusion) + } + var requests []struct { + RunID int + CursorValue string + } + require.NoError(t, db.Table("jobs_requests").Where("phase = ?", phase).Find(&requests).Error) + switch scenario { + case "extract": + require.Empty(t, requests, "completed collector must be skipped") + case "batch": + require.Len(t, requests, 1) + require.Equal(t, 3, requests[0].RunID) + case "page": + require.Len(t, requests, 9-len(committedJobs)) + seen := map[int]bool{} + for _, r := range requests { + page := 1 + if r.CursorValue != "" { + previous, err := strconv.Atoi(strings.TrimPrefix(r.CursorValue, "p")) + require.NoError(t, err) + page = previous + 1 + } + id := r.RunID*10 + page + require.False(t, committedJobs[id], "committed page must not be refetched: %d", id) + require.False(t, seen[id], "page requested twice: %d", id) + seen[id] = true + } + } + var rawCount int64 + require.NoError(t, db.Table("_raw_github_graphql_jobs").Count(&rawCount).Error) + require.EqualValues(t, 9, rawCount) + var state models.CollectorLatestState + require.NoError(t, db.Where("raw_data_table = ?", "_raw_github_graphql_jobs").First(&state).Error) + require.NotNil(t, state.LatestSuccessStart) + require.WithinDuration(t, *tasks[0].BeganAt, *state.LatestSuccessStart, time.Millisecond) +} diff --git a/backend/server/api/graphql_other_collectors_test.go b/backend/server/api/graphql_other_collectors_test.go new file mode 100644 index 00000000000..f90a345b0bd --- /dev/null +++ b/backend/server/api/graphql_other_collectors_test.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/apache/devlake/core/config" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/core/runner" + collector "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/helpers/unithelper" + "github.com/apache/devlake/impls/dalgorm" + "github.com/apache/devlake/impls/logruslog" + mockplugin "github.com/apache/devlake/mocks/core/plugin" + githubmodels "github.com/apache/devlake/plugins/github/models" + githubtasks "github.com/apache/devlake/plugins/github/tasks" + githubgraphql "github.com/apache/devlake/plugins/github_graphql/tasks" + linear "github.com/apache/devlake/plugins/linear/tasks" + "github.com/merico-ai/graphql" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestGraphqlOtherCollectorsResume(t *testing.T) { + dbURL := os.Getenv("DEVLAKE_SERVER_CRASH_DB_URL") + if dbURL == "" { + t.Skip("dedicated database not configured") + } + cfg := config.GetConfig() + cfg.Set("DB_URL", dbURL) + db, err := runner.NewGormDb(cfg, logruslog.Global) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&models.CollectorLatestState{}, &githubmodels.GithubPullRequest{}, &githubmodels.GithubIssue{})) + old := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + require.NoError(t, db.Create(&githubmodels.GithubPullRequest{ConnectionId: 1, GithubId: 2, RepoId: 2, Number: 2, State: "OPEN", GithubCreatedAt: old, GithubUpdatedAt: old}).Error) + require.NoError(t, db.Create(&githubmodels.GithubIssue{ConnectionId: 1, GithubId: 2, RepoId: 2, Number: 2, State: "OPEN", GithubCreatedAt: old, GithubUpdatedAt: old}).Error) + for i, kind := range []string{"pr", "issue", "linear"} { + t.Run(kind, func(t *testing.T) { + var mu sync.Mutex + fail := true + var trace []string + detailPattern := regexp.MustCompile(`(\w+)\s*:\s*(?:pullRequest|issue)\s*\(`) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Query string + Variables map[string]interface{} + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + cursor, _ := req.Variables["skipCursor"].(string) + detail := detailPattern.FindStringSubmatch(req.Query) + stage := "list" + if len(detail) > 0 || cursor != "" { + stage = "detail" + } + mu.Lock() + trace = append(trace, stage) + shouldFail := fail && stage == "detail" + mu.Unlock() + if shouldFail { + http.Error(w, "injected interruption", 503) + return + } + node := map[string]interface{}{"databaseId": 1, "number": 1, "title": "first", "updatedAt": "2026-01-01T00:00:00Z", "createdAt": "2026-01-01T00:00:00Z"} + data := map[string]interface{}{"rateLimit": map[string]int{"cost": 1}} + if kind == "linear" { + node = map[string]interface{}{"id": "one", "number": 1, "title": "first", "updatedAt": "2026-01-01T00:00:00Z"} + if cursor != "" { + node["id"] = "two" + node["number"] = 2 + } + data = map[string]interface{}{"team": map[string]interface{}{"issues": map[string]interface{}{"nodes": []interface{}{node}, "pageInfo": map[string]interface{}{"endCursor": "next", "hasNextPage": cursor == ""}}}} + } else if len(detail) > 0 { + node["databaseId"] = 2 + node["number"] = 2 + node["title"] = "second" + data["repository"] = map[string]interface{}{detail[1]: node} + } else { + field := "issues" + if kind == "pr" { + field = "pullRequests" + } + data["repository"] = map[string]interface{}{field: map[string]interface{}{"nodes": []interface{}{node}, "pageInfo": map[string]interface{}{"endCursor": "", "hasNextPage": false}, "totalCount": 1}} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": data}) + })) + defer upstream.Close() + execute := func() error { + ctx, cancel := context.WithCancel(plugin.WithTaskID(context.Background(), uint64(900+i))) + defer cancel() + taskCtx := unithelper.DummySubTaskContext(dalgorm.NewDalgorm(db)) + taskCtx.On("GetContext").Return(ctx) + parent := taskCtx.TaskContext().(*mockplugin.TaskContext) + parent.On("GetContext").Return(ctx) + parent.On("GetConfig", mock.Anything).Return("") + client, err := collector.CreateAsyncGraphqlClient(parent, graphql.NewClient(upstream.URL, upstream.Client()), logruslog.Global, nil) + require.NoError(t, err) + defer client.Release() + client.SetMaxRetry(1, 0) + if kind == "linear" { + taskCtx.On("GetData").Return(&linear.LinearTaskData{Options: &linear.LinearOptions{ConnectionId: 1, TeamId: "team"}, GraphqlClient: client}) + return linear.CollectIssues(taskCtx) + } + taskCtx.On("GetData").Return(&githubtasks.GithubTaskData{Options: &githubtasks.GithubOptions{ConnectionId: 1, GithubId: 2, Name: "fixture/repo"}, GraphqlClient: client}) + if kind == "pr" { + return githubgraphql.CollectPrs(taskCtx) + } + return githubgraphql.CollectIssues(taskCtx) + } + require.Error(t, execute()) + mu.Lock() + fail = false + mu.Unlock() + require.NoError(t, execute()) + mu.Lock() + require.Equal(t, []string{"list", "detail", "detail"}, trace) + mu.Unlock() + table := "_raw_github_graphql_issues" + if kind == "pr" { + table = "_raw_github_graphql_prs" + } + if kind == "linear" { + table = "_raw_linear_issues" + } + var rows []collector.RawData + require.NoError(t, db.Table(table).Order("id").Find(&rows).Error) + require.Len(t, rows, 2) + for j, row := range rows { + var value map[string]interface{} + require.NoError(t, json.Unmarshal(row.Data, &value)) + numbers := map[string]interface{}{} + for key, v := range value { + numbers[strings.ToLower(key)] = v + } + require.EqualValues(t, j+1, numbers["number"]) + } + }) + } +} diff --git a/backend/server/api/graphql_resume_crash_test.go b/backend/server/api/graphql_resume_crash_test.go new file mode 100644 index 00000000000..881bcc3328f --- /dev/null +++ b/backend/server/api/graphql_resume_crash_test.go @@ -0,0 +1,234 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api_test + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/apache/devlake/core/config" + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/core/models" + "github.com/apache/devlake/core/plugin" + "github.com/apache/devlake/core/runner" + collector "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/apache/devlake/impls/logruslog" + server "github.com/apache/devlake/server/api" + "github.com/apache/devlake/server/services" + "github.com/merico-ai/graphql" + "github.com/stretchr/testify/require" +) + +type resumeFixturePlugin struct{ endpoint string } + +type serverCrashRequest struct { + ID uint64 `gorm:"primaryKey;autoIncrement"` + Phase string + CursorValue string +} + +func (serverCrashRequest) TableName() string { return "server_crash_requests" } + +func (p resumeFixturePlugin) Name() string { return "graphql_resume_fixture" } +func (p resumeFixturePlugin) Description() string { return "Crash/restart fixture" } +func (p resumeFixturePlugin) RootPkgPath() string { return "github.com/apache/devlake/server/api" } +func (p resumeFixturePlugin) PrepareTaskData(ctx plugin.TaskContext, _ map[string]interface{}) (interface{}, errors.Error) { + return collector.CreateAsyncGraphqlClient(ctx, graphql.NewClient(p.endpoint, http.DefaultClient), ctx.GetLogger(), nil) +} + +type resumeFixtureQuery struct { + Items struct { + Nodes []struct{ ID int } + PageInfo collector.GraphqlQueryPageInfo + } `graphql:"items(after: $after)"` +} + +func (p resumeFixturePlugin) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{{Name: "Collect fixture", EnabledByDefault: true, EntryPoint: func(ctx plugin.SubTaskContext) errors.Error { + c, err := collector.NewGraphqlCollector(collector.GraphqlCollectorArgs{ + RawDataSubTaskArgs: collector.RawDataSubTaskArgs{Ctx: ctx, Table: "server_resume_fixture", Params: "repo"}, + GraphqlClient: ctx.GetData().(*collector.GraphqlAsyncClient), PageSize: 1, + BuildQuery: func(req *collector.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + return &resumeFixtureQuery{}, map[string]interface{}{"after": (*graphql.String)(req.Pager.SkipCursor)}, nil + }, + GetPageInfo: func(q interface{}, _ *collector.GraphqlCollectorArgs) (*collector.GraphqlQueryPageInfo, error) { + return &q.(*resumeFixtureQuery).Items.PageInfo, nil + }, + ResponseParser: func(q interface{}) ([]json.RawMessage, errors.Error) { + var rows []json.RawMessage + for _, node := range q.(*resumeFixtureQuery).Items.Nodes { + row, err := json.Marshal(node) + if err != nil { + return nil, errors.Convert(err) + } + rows = append(rows, row) + } + return rows, nil + }, + }) + if err != nil { + return err + } + return c.Execute() + }}} +} + +// Opt-in: start a real API server and its production queue/runner. Kill the +// crash-phase container after CRASH_READY, then run resume against the SAME +// disposable database. No task IDs or statuses are injected by this test. +func TestGraphqlServerResumeCrash(t *testing.T) { + dbURL := os.Getenv("DEVLAKE_SERVER_CRASH_DB_URL") + if dbURL == "" { + t.Skip("dedicated server crash database not configured") + } + phase := os.Getenv("DEVLAKE_CRASH_PHASE") + rerun := os.Getenv("DEVLAKE_SERVER_RERUN") == "true" + require.Contains(t, []string{"crash", "resume"}, phase) + cfg := config.GetConfig() + cfg.Set("DB_URL", dbURL) + cfg.Set("RESUME_PIPELINES", !rerun) + cfg.Set("CONSUME_PIPELINES", true) + cfg.Set("FORCE_MIGRATION", true) + cfg.Set("PIPELINE_MAX_PARALLEL", 1) + cfg.Set("DISABLED_REMOTE_PLUGINS", true) + cfg.Set("PLUGIN_DIR", t.TempDir()) + cfg.Set("LOGGING_DIR", t.TempDir()) + cfg.Set("AUTH_ENABLED", false) + cfg.Set(plugin.EncodeKeyEnvStr, "DFLFZLMBBFDDCYWRECDCIYUROPPAKQDFQMMJEFPIKVFVHZBRGAZIHKRJIJZMOHWEVRSCETAGGONPSULGOXITVXISVCQGPSFAOGRDLUANEYDQFBDKVMYYHUZFHYVYGPPT") + db, err := runner.NewGormDb(cfg, logruslog.Global) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&serverCrashRequest{})) + var original struct { + ID uint64 + PipelineId uint64 + Status string + } + if phase == "resume" { + require.NoError(t, db.Table(models.Task{}.TableName()).Take(&original).Error) + require.Equal(t, models.TASK_RUNNING, original.Status) + var pipeline struct{ Status string } + require.NoError(t, db.Table(models.Pipeline{}.TableName()).Where("id = ?", original.PipelineId).Take(&pipeline).Error) + require.Equal(t, models.TASK_RUNNING, pipeline.Status) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct{ Variables struct{ After *string } } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + cursor := "" + if req.Variables.After != nil { + cursor = *req.Variables.After + } + if err := db.Exec("INSERT INTO server_crash_requests (phase, cursor_value) VALUES (?, ?)", phase, cursor).Error; err != nil { + t.Error(err) + return + } + if phase == "crash" && cursor == "page-1" { + fmt.Println("CRASH_READY: real server pipeline running, first page committed") + <-r.Context().Done() + return + } + id, next := 1, true + if cursor != "" { + id, next = 2, false + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]interface{}{"items": map[string]interface{}{ + "nodes": []map[string]int{{"id": id}}, "pageInfo": collector.GraphqlQueryPageInfo{EndCursor: "page-1", HasNextPage: next}, + }}}) + })) + defer upstream.Close() + p := resumeFixturePlugin{endpoint: upstream.URL} + require.NoError(t, plugin.RegisterPlugin(p.Name(), p)) + server.Init() + services.InitExecuteMigration() + require.Equal(t, services.SERVICE_STATUS_READY, services.CurrentStatus()) + router := server.CreateApiServer() + server.SetupApiServer(router) + apiServer := httptest.NewServer(router) + defer apiServer.Close() + if phase == "crash" { + payload := models.NewPipeline{Name: "graphql crash resume", Plan: models.PipelinePlan{{&models.PipelineTask{Plugin: p.Name(), Options: map[string]interface{}{}}}}} + body, err := json.Marshal(payload) + require.NoError(t, err) + response, err := http.Post(apiServer.URL+"/pipelines", "application/json", bytes.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + require.Less(t, response.StatusCode, 300) + // The orchestrator must SIGKILL this process before the timeout. + select {} + } + if rerun { + var interrupted models.Task + require.NoError(t, db.First(&interrupted, original.ID).Error) + require.Equal(t, models.TASK_FAILED, interrupted.Status) + var count int64 + require.NoError(t, db.Table("server_crash_requests").Where("phase = ?", phase).Count(&count).Error) + require.Zero(t, count, "RESUME_PIPELINES=false must not restart collection") + response, err := http.Post(fmt.Sprintf("%s/pipelines/%d/rerun", apiServer.URL, original.PipelineId), "application/json", nil) + require.NoError(t, err) + require.Less(t, response.StatusCode, 300) + response.Body.Close() + } + require.Eventually(t, func() bool { + var pipeline models.Pipeline + return db.First(&pipeline, original.PipelineId).Error == nil && pipeline.Status == models.TASK_COMPLETED + }, 30*time.Second, 100*time.Millisecond) + var tasks []models.Task + require.NoError(t, db.Order("id").Find(&tasks).Error) + completedID := original.ID + if rerun { + require.Len(t, tasks, 2) + require.Equal(t, models.TASK_FAILED, tasks[0].Status) + require.Greater(t, tasks[1].ID, original.ID) + completedID = tasks[1].ID + } else { + require.Len(t, tasks, 1) + require.Equal(t, original.ID, tasks[0].ID) + } + require.Equal(t, models.TASK_COMPLETED, tasks[len(tasks)-1].Status) + var subtask models.Subtask + require.NoError(t, db.Where("task_id = ?", completedID).First(&subtask).Error) + require.NotNil(t, subtask.FinishedAt) + var requests []string + require.NoError(t, db.Table("server_crash_requests").Where("phase = ?", phase).Order("id").Pluck("cursor_value", &requests).Error) + if rerun { + require.Equal(t, []string{"", "page-1"}, requests) + } else { + require.Equal(t, []string{"page-1"}, requests) + } + var rows []collector.RawData + require.NoError(t, db.Table("_raw_server_resume_fixture").Order("id").Find(&rows).Error) + require.Len(t, rows, 2) + for i, row := range rows { + var node struct{ ID int } + require.NoError(t, json.Unmarshal(row.Data, &node)) + require.Equal(t, i+1, node.ID) + } + var state collector.GraphqlCollectorState + require.NoError(t, db.Where("input_hash = ? AND raw_data_table = ?", "", "_raw_server_resume_fixture").First(&state).Error) + require.Equal(t, completedID, state.TaskID) + require.True(t, state.Completed) +} diff --git a/backend/test/graphql-resume-coverage.md b/backend/test/graphql-resume-coverage.md new file mode 100644 index 00000000000..db3d1ed3670 --- /dev/null +++ b/backend/test/graphql-resume-coverage.md @@ -0,0 +1,90 @@ + + +# GraphQL crash-resume coverage (#9142) + +## Reproduce + +Run `bash backend/test/graphql-resume-crash.sh` from the repository root. +For PostgreSQL, run `DEVLAKE_CRASH_DRIVER=postgres bash backend/test/graphql-resume-crash.sh`. + +Only Docker is required. The script uses Go 1.26, MySQL 8.4.11 or PostgreSQL +18.4, and generates missing mocks with mockery v3.7.4. Test binaries enable +the race detector. Databases are disposable, use no published ports, and +only containers created by the script are removed. Logs remain in the +temporary directory printed on exit. Upstream HTTP responses are fixtures; +no production database or live GitHub/Linear endpoint is modified. + +The local validation on September 18, rebased onto upstream main at +`a0cc75183badb3d7b370942c96ea563fba26bb7b`, passed 36 checks per database and +14 SIGKILLs per database (exit 137, not OOM). Eight related packages passed +`go test -race -count=1`. These are local results, not a claim that the +GitHub Actions jobs have passed. + +## Audit matrix + +| Area | Implementation and evidence | +| --- | --- | +| Actual Jobs path | Production queue/runner executes CollectJobs then ExtractJobs in batch, page, and extractor-restart scenarios. | +| Batch members and tail | Three inputs with batch size two; verify every Job ID, RunID, RepoId, ConnectionId and Conclusion, including the final singleton batch. | +| Multiple rows/pages/crashes | Fail the second SQL row of a page; collect three pages per input; SIGKILL the same Jobs task twice. | +| Collector registration | Test automatic checkpoint indices; actual PR/Issue collectors exercise two nested collectors. | +| Nested collectors | A completes, B fails, then resume; preserve A's raw rows and verify another rerun does not duplicate them. | +| Incremental watermark | LatestSuccessStart remains anchored to the original Task.BeganAt. Incremental runs retain existing raw records. | +| Crash windows | SIGKILL during initialization, an intermediate/final page transaction, completion transaction, and after collection/before extraction. Inject state-manager and runner-status write failures separately. | +| Recovery from errors | Recreate collectors after SQL/parser failures and lost commit acknowledgements; compare exact raw IDs. | +| Iterators | Real SQL cursors; Rows.Err, Fetch/Close/marshal failures, empty input, duplicate input, and cancellation. Validate/stage the input before modifying raw data. | +| Page boundaries | Reject nil/error pageInfo and empty/unchanged next cursors. Cover ErrFinishCollect with and without records. | +| Partial GraphQL errors | Unknown errors cannot complete durable collection, even with IgnoreQueryErrors=true. | +| Async cancellation | Cover HTTP/retry/rate-limit/NextTick cancellation; multi-input Jobs crash tests run with the race detector. | +| API and runner | RESUME_PIPELINES=true preserves task identity; false marks interrupted work FAILED and the rerun API creates a new task. Skip completed collection when resuming extraction. | +| Task-less compatibility | Full/incremental Execute, empty input, and ErrFinishCollect on real databases; no durable checkpoints for callers without task IDs. | +| Identity and interference | Isolate table/params/subtask/index; reject overlapping raw scopes, changed input/order/page or batch settings, changed executable/plugin code, and tasks superseded by newer work. | +| Database failures | Initialization/completion delete/save failures; partial raw writes; checkpoint/commit failures; successful commit with lost acknowledgement. Raw and checkpoint updates share a transaction. | +| Database/migration compatibility | Run the same suite on MySQL/PostgreSQL, including production migrations and restart with existing data. Compare migration-snapshot and runtime-model field types/tags. | +| Other callers | Interrupt and resume real GitHub CollectPrs/CollectIssues and Linear CollectIssues. Completed list work is not repeated; unfinished detail/page work resumes. | +| Assertions | Compare raw IDs, all batch members, requests against committed pages, extracted Job fields, task/subtask uniqueness, and pipeline progress rather than counts alone. | +| Continuous validation | Add a MySQL/PostgreSQL matrix to the unit-test workflow and preserve logs as artifacts. | +| Hot-path cost | Pure parsers add the inserted row count without scanning all accumulated raw rows per page. Recount only for transactional parsers that may delete rows. Test append, empty, deletion, and failed-commit accounting. | + +Additional fixes covered by these tests: + +- Distinguish SQL cursor failure from EOF. +- Do not record success FinishedAt on subtask error/panic; propagate status-write failures. +- Do not create duplicate subtask rows after restart. +- Run Issue parser cleanup inside the raw/checkpoint transaction. +- Delete checkpoints through both scope-deletion helpers; test table discovery and deletion predicates. +- Reject raw-count mismatch instead of silently trusting stale checkpoints. +- Stabilize Jobs ordering for identical update timestamps and order PR/Issue/Account/Linear child inputs. +- Synchronize the existing Queue test's completion flags with channels so race validation is meaningful. + +## Resume contract and limitations + +- Resume requires the same persisted task, code, input and page/batch settings, + with its database intact. Detected changes fail safely; use a new task (rerun) + instead of reusing an incompatible checkpoint. +- Input payloads are staged to a temporary file. Fixed-size duplicate-detection + hashes are retained in memory for each input. +- Raw integrity checking compares counts. It is not a checksum capable of + detecting manual same-count rewrites. +- This does not promise a remote GitHub/Linear snapshot: cursors may expire and + upstream result sets may change. Upstream errors leave checkpoints intact + rather than reporting success. +- Arbitrary environment/connection settings are not all fingerprinted. Changing + endpoints or collection conditions between attempts requires a new task. +- Local fixtures do not replace live-service load testing or testing every + combination of every plugin. The per-attempt subtask progress display is not + restored to a cumulative pre-crash page count; persisted cursors/raw data, + not that UI counter, determine what is refetched. diff --git a/backend/test/graphql-resume-crash.sh b/backend/test/graphql-resume-crash.sh new file mode 100644 index 00000000000..757d39b6ae3 --- /dev/null +++ b/backend/test/graphql-resume-crash.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run from any directory: bash backend/test/graphql-resume-crash.sh +# Requires Docker; generates missing mocks with the pinned tool version. +# Uses only newly created containers; no existing DB or published port. +set -euo pipefail +repo=$(cd "$(dirname "$0")/../.." && pwd) +artifacts=$(mktemp -d "${TMPDIR:-/tmp}/devlake-graphql-crash.XXXXXX") +prefix="devlake-graphql-crash-$$" +driver=${DEVLAKE_CRASH_DRIVER:-mysql} +mysql_name="$prefix-$driver" +containers=() +cleanup() { + for name in "${containers[@]}"; do docker rm -f "$name" >/dev/null 2>&1 || true; done + echo "Logs and binaries: $artifacts" +} +trap cleanup EXIT +if [[ ! -f "$repo/backend/mocks/core/dal/Dal.go" ]]; then + for config in .mockery.core.yml .mockery.helpers.yml; do + docker run --rm -v "$repo:/src" -v devlake-9142-go-cache:/go \ + -v devlake-9142-build-cache:/root/.cache/go-build -w /src/backend golang:1.26 \ + go run github.com/vektra/mockery/v3@v3.7.4 --config "$config" + done +fi +build() { + docker run --rm -v "$repo:/src:ro" -v "$artifacts:/out" \ + -v devlake-9142-go-cache:/go -v devlake-9142-build-cache:/root/.cache/go-build \ + -w /src/backend golang:1.26 go test -race -c -o "/out/$1.test" "$2" +} +build collector ./helpers/pluginhelper/api +build server ./server/api +containers+=("$mysql_name") +if [[ "$driver" == postgres ]]; then + docker run -d --name "$mysql_name" --tmpfs /var/lib/postgresql \ + -e POSTGRES_PASSWORD=crash-test-only -e POSTGRES_DB=crash_test postgres:18.4 >/dev/null +elif [[ "$driver" == mysql ]]; then + docker run -d --name "$mysql_name" --tmpfs /var/lib/mysql \ + -e MYSQL_ROOT_PASSWORD=crash-test-only -e MYSQL_DATABASE=crash_test mysql:8.4.11 >/dev/null +else + echo "Unsupported driver: $driver"; exit 1 +fi +sql() { + if [[ "$driver" == postgres ]]; then docker exec "$mysql_name" psql -U postgres -d crash_test -c "$1"; + else docker exec -e MYSQL_PWD=crash-test-only "$mysql_name" mysql -uroot -e "$1"; fi +} +ready=0 +for ((attempt=0; attempt<120; attempt++)); do + if sql 'SELECT 1' >/dev/null 2>&1; then ready=1; break; fi + sleep 1 +done +if [[ "$ready" != 1 ]]; then docker logs "$mysql_name"; exit 1; fi +set_database() { + local database="$1" + common=(--network "container:$mysql_name" -v "$artifacts:/out:ro" -e "DEVLAKE_CRASH_DRIVER=$driver") + if [[ "$driver" == postgres ]]; then + common+=(-e "DEVLAKE_CRASH_DSN=host=127.0.0.1 user=postgres password=crash-test-only dbname=$database port=5432 sslmode=disable" + -e "DEVLAKE_SERVER_CRASH_DB_URL=postgres://postgres:crash-test-only@127.0.0.1:5432/$database?sslmode=disable") + else + common+=(-e "DEVLAKE_CRASH_DSN=root:crash-test-only@tcp(127.0.0.1:3306)/$database?parseTime=true" + -e "DEVLAKE_SERVER_CRASH_DB_URL=mysql://root:crash-test-only@127.0.0.1:3306/$database?parseTime=true") + fi +} +set_database crash_test +crash() { + local name="$1" binary="$2" test_name="$3" scenario="$4" phase="${5:-crash}" + containers+=("$name") + docker run -d --name "$name" "${common[@]}" -e "DEVLAKE_CRASH_PHASE=$phase" \ + -e "DEVLAKE_CRASH_SCENARIO=$scenario" golang:1.26 "/out/$binary.test" \ + -test.run "^$test_name$" -test.v -test.timeout 3m >/dev/null + local ready=0 + for ((attempt=0; attempt<600; attempt++)); do + docker logs "$name" > "$artifacts/$scenario-crash.log" 2>&1 + if grep -q CRASH_READY "$artifacts/$scenario-crash.log"; then ready=1; break; fi + if [[ $(docker inspect -f '{{.State.Running}}' "$name") != true ]]; then break; fi + sleep 0.2 + done + if [[ "$ready" != 1 ]]; then cat "$artifacts/$scenario-crash.log"; return 1; fi + docker kill --signal KILL "$name" >/dev/null + docker logs "$name" > "$artifacts/$scenario-crash.log" 2>&1 + if grep -q 'WARNING: DATA RACE' "$artifacts/$scenario-crash.log"; then + cat "$artifacts/$scenario-crash.log"; return 1 + fi + docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' "$name" > "$artifacts/$scenario-exit.log" + grep -qx '137 false' "$artifacts/$scenario-exit.log" +} +for scenario in single multi transaction batch incremental initialize final-page complete; do + crash "$prefix-$scenario" collector TestGraphqlCollectorMySQLCrash "$scenario" + for phase in resume repeat fresh; do + docker run --rm "${common[@]}" -e "DEVLAKE_CRASH_PHASE=$phase" \ + -e "DEVLAKE_CRASH_SCENARIO=$scenario" golang:1.26 /out/collector.test \ + -test.run '^TestGraphqlCollectorMySQLCrash$' -test.v -test.timeout 1m \ + > "$artifacts/$scenario-$phase.log" 2>&1 || { cat "$artifacts/$scenario-$phase.log"; exit 1; } + echo "PASS $scenario/$phase" + done +done +for test_name in TestGraphqlCollectorMySQLLifecycleRecovery TestGraphqlCollectorMySQLScopeIsolation TestGraphqlDatabaseGuard TestGraphqlDatabasePageRollback TestGraphqlDatabaseNestedResume TestGraphqlDatabaseTasklessAndEmpty; do + docker run --rm "${common[@]}" golang:1.26 /out/collector.test \ + -test.run "^$test_name$" -test.v -test.timeout 1m \ + > "$artifacts/$test_name.log" 2>&1 || { cat "$artifacts/$test_name.log"; exit 1; } + echo "PASS $test_name" +done +crash "$prefix-server" server TestGraphqlServerResumeCrash server +docker run --rm "${common[@]}" -e DEVLAKE_CRASH_PHASE=resume golang:1.26 \ + /out/server.test -test.run '^TestGraphqlServerResumeCrash$' -test.v -test.timeout 2m \ + > "$artifacts/server-resume.log" 2>&1 || { cat "$artifacts/server-resume.log"; exit 1; } +echo "PASS real server RESUME_PIPELINES" +sql "CREATE DATABASE server_rerun" >/dev/null +set_database server_rerun +common+=(-e DEVLAKE_SERVER_RERUN=true) +crash "$prefix-rerun" server TestGraphqlServerResumeCrash server-rerun +docker run --rm "${common[@]}" -e DEVLAKE_CRASH_PHASE=resume golang:1.26 \ + /out/server.test -test.run '^TestGraphqlServerResumeCrash$' -test.v -test.timeout 2m \ + > "$artifacts/server-rerun.log" 2>&1 || { cat "$artifacts/server-rerun.log"; exit 1; } +echo "PASS RESUME_PIPELINES=false and rerun API" +sql "CREATE DATABASE other_collectors" >/dev/null +set_database other_collectors +docker run --rm "${common[@]}" golang:1.26 /out/server.test \ + -test.run '^TestGraphqlOtherCollectorsResume$' -test.v -test.timeout 2m \ + > "$artifacts/other-collectors.log" 2>&1 || { cat "$artifacts/other-collectors.log"; exit 1; } +echo "PASS real GitHub PR/Issue and Linear collectors" +for scenario in batch page extract; do + sql "CREATE DATABASE jobs_$scenario" >/dev/null + set_database "jobs_$scenario" + common+=(-e "DEVLAKE_JOBS_SCENARIO=$scenario") + crash "$prefix-jobs-$scenario" server TestGithubJobsServerResumeCrash "jobs-$scenario" + if [[ "$scenario" == page ]]; then + crash "$prefix-jobs-page-again" server TestGithubJobsServerResumeCrash jobs-page-again crash-again + fi + docker run --rm "${common[@]}" -e DEVLAKE_CRASH_PHASE=resume golang:1.26 \ + /out/server.test -test.run '^TestGithubJobsServerResumeCrash$' -test.v -test.timeout 2m \ + > "$artifacts/jobs-$scenario-resume.log" 2>&1 || { cat "$artifacts/jobs-$scenario-resume.log"; exit 1; } + echo "PASS real CollectJobs/ExtractJobs $scenario" +done