From a6379fe587dc21688f28f5f31254064e2f8367fe Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 17:14:31 +1100 Subject: [PATCH 01/16] piece-of-work --- docs/data_flow.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/data_flow.md diff --git a/docs/data_flow.md b/docs/data_flow.md new file mode 100644 index 00000000..677dc7a2 --- /dev/null +++ b/docs/data_flow.md @@ -0,0 +1,46 @@ + +# Data flow analysis in stackql + +Data flow analysis is impplmented as multiple passes on: + +- An inital abstract syntax tree (AST) from the parser. + - Annotated derivatives of the AST. +- `any-sdk` `{ provider, service, resource, method, schema... }` graphs. +- `gonum` DAG adaptations with data flow dependencies representing edges. + +Some other aspects of data flow analysis: + +- Relational algebra is implemented in a coupled RDBMS (embedded `sqlite` or `postgres` over TCP). There is a query rewriting process to stringify "containers" for this. +- There are `transaction control counter` objects and corresponding RDBMS columns to bound relational algebra "containers" and future proof for gargage collection. Some mutex protection is in place. +- Views in `stackql` permit clobbering of where clause arguments from outside the view. The canonical case is a document-based view in a provider document. A good example are in [test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml](/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml)at `...s3_bucket_list_and_detail.config.views.select`; one can overwrite `region` here. +- Views, subqueries, materialized views and user space tables are modelled as "indirections". + + +## Open Issues + +## Indirection Data Flow Analysis and Query Execution + +Data flow analysis for indirections is not composable: + +- It it impossible to join heterogenous collections of these with each other or conventional resources. There is no recusrsive and stable data flow analysis. +- While `stackql` does have a `max depth` parameter, I do not believe it is stable enfoced eagerly. Ie: queries too complex should fail at analysis time. Cannot remember param name of=r default. + +The expected fix for this issue: + +- Joins, unions etc on indirections work to arbitrary and configurable depth. For depth violations, failure is eager in the analysis phase and error message is plain and in the canonical err stream already widely used. +- Data flow analysis includes assurance on reuired poarams and viability of projections, joins, etc. +- Support for CTEs internal to these indirections is in place. +- Mocked robot tests are added to the canonical test suite, covering off this function. + + +## Glossary of terms + +| Term | Expansion | +|---|---| +| AST | Abstract Syntax Tree | +| CTE | Common Table Expression | +| DAG | Directed Acyclic Graph | +| GC | Garbage Collection | +| RDBMS | Relational Database Management System | +| TCP | Transmission Control Protocol | +| | | From e15f5315e806142e80ce837fbc5b59a2f47b711b Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 18:25:09 +1100 Subject: [PATCH 02/16] composition-of-indirects Summary: - Support for composition of indirects, including `view`, `materialized view` and subqueries. - Added robot test `View Depth Limitation Error Message Shows Correct Max`. - Added robot test `View JOIN View Returns Results`. - Added robot test `View JOIN Provider Table Returns Results`. - Added robot test `Subquery JOIN Subquery Returns Results`. - Added robot test `CTE Within View Returns Results`. --- .../astanalysis/earlyanalysis/ast_expand.go | 11 +- .../dependencyplanner/dependencyplanner.go | 123 ++++++++++++++++++ .../stackql_mocked_from_cmd_line.robot | 82 ++++++++++++ 3 files changed, 214 insertions(+), 2 deletions(-) diff --git a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go index 9c7e28e0..2419d13c 100644 --- a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go +++ b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" "github.com/stackql/stackql/internal/stackql/astindirect" @@ -141,6 +140,14 @@ func (v *indirectExpandAstVisitor) processCTEReference( } func (v *indirectExpandAstVisitor) processIndirect(node sqlparser.SQLNode, indirect astindirect.Indirect) error { + // Eager depth check: fail before recursively analyzing an indirection that would exceed the limit. + if v.indirectionDepth+1 > v.handlerCtx.GetRuntimeContext().IndirectDepthMax { + return fmt.Errorf( + "query error: indirection chain length %d > %d and is therefore disallowed; please do not cite views at too deep a level", //nolint:lll + v.indirectionDepth+1, + v.handlerCtx.GetRuntimeContext().IndirectDepthMax, + ) + } err := indirect.Parse() if err != nil { return nil //nolint:nilerr //TODO: investigate @@ -178,7 +185,7 @@ func (v *indirectExpandAstVisitor) processIndirect(node sqlparser.SQLNode, indir return fmt.Errorf( "query error: indirection chain length %d > %d and is therefore disallowed; please do not cite views at too deep a level", //nolint:lll maxIndirectCount, - constants.LimitsIndirectMaxChainLength, + v.handlerCtx.GetRuntimeContext().IndirectDepthMax, ) } indirectPrimitiveGenerator.GetPrimitiveComposer().GetAst() diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index 64907934..afcdf508 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -103,6 +103,40 @@ func NewStandardDependencyPlanner( }, nil } +func isAnnotationIndirection(annotationCtx taxonomy.AnnotationCtx) bool { + _, isView := annotationCtx.GetView() + _, isSubquery := annotationCtx.GetSubquery() + return isView || isSubquery +} + +// orchestrateIndirection handles indirection vertices (views, subqueries, CTEs) in the dependency planner. +// Unlike regular tables that need HTTP acquisition, indirections are already materialized by the +// "create tail" builder and appear in the FROM clause as inline subqueries. +// This method creates a NopBuilder as a placeholder in the execution DAG. +func (dp *standardDependencyPlanner) orchestrateIndirection( + annotationCtx taxonomy.AnnotationCtx, +) (int, error) { + rc, err := tableinsertioncontainer.NewTableInsertionContainer( + annotationCtx.GetTableMeta(), + dp.handlerCtx.GetSQLEngine(), + dp.handlerCtx.GetTxnCounterMgr(), + ) + if err != nil { + return -1, err + } + builder := primitivebuilder.NewNopBuilder( + dp.primitiveComposer.GetGraphHolder(), + dp.primitiveComposer.GetTxnCtrlCtrs(), + dp.handlerCtx, + dp.handlerCtx.GetSQLEngine(), + []string{}, + ) + dp.execSlice = append(dp.execSlice, builder) + idx := len(dp.execSlice) - 1 + dp.tableSlice = append(dp.tableSlice, rc) + return idx, nil +} + func (dp *standardDependencyPlanner) dataflowEdgeExists(from, to int) bool { edges, ok := dp.dataflowToEdges[to] if !ok { @@ -204,6 +238,7 @@ func (dp *standardDependencyPlanner) Plan() error { edgeCount, dependencyMax) } idsVisited := make(map[int64]struct{}) + indirectionNodeIDs := make(map[int64]struct{}) // first pass: set up AOT stuff // - stream per edge. edgeStreams := make(map[dataflow.Edge]streaming.MapStream) @@ -218,6 +253,17 @@ func (dp *standardDependencyPlanner) Plan() error { tableExpr := n.GetTableExpr() annotation := n.GetAnnotation() dp.annMap[tableExpr] = annotation + // Indirection nodes (views, subqueries, CTEs) are already materialized + // by the create tail builder; register them and skip acquisition. + if isAnnotationIndirection(annotation) { + indirectionNodeIDs[n.ID()] = struct{}{} + idx, indErr := dp.orchestrateIndirection(annotation) + if indErr != nil { + return indErr + } + dp.nodeIDIdxMap[n.ID()] = idx + continue + } for _, e := range edges { if e.From().ID() == n.ID() { insPsc, tcc, insErr := dp.processOrphan(tableExpr, annotation, n) @@ -228,6 +274,20 @@ func (dp *standardDependencyPlanner) Plan() error { toNode := e.GetDest() toAnnotation := toNode.GetAnnotation().Clone() // this bodge protects split source vertices toTableExpr := toNode.GetTableExpr() + // Handle indirection destination nodes. + if isAnnotationIndirection(toAnnotation) { + if _, alreadyHandled := indirectionNodeIDs[toNode.ID()]; !alreadyHandled { + indirectionNodeIDs[toNode.ID()] = struct{}{} + dp.annMap[toTableExpr] = toAnnotation + toIdx, toIndErr := dp.orchestrateIndirection(toAnnotation) + if toIndErr != nil { + return toIndErr + } + dp.nodeIDIdxMap[toNode.ID()] = toIdx + } + orderedEdges = append(orderedEdges, e) + continue + } stream, streamErr := dp.getStreamFromEdge(e, toAnnotation, tcc) if streamErr != nil { return streamErr @@ -251,6 +311,69 @@ func (dp *standardDependencyPlanner) Plan() error { fromAnnotation := fromNode.GetAnnotation() toAnnotation := toNode.GetAnnotation().Clone() // this bodge protects split source vertices toTableExpr := toNode.GetTableExpr() + // For indirection nodes, builders are already created; just wire up edge dependencies. + _, fromIsIndirection := indirectionNodeIDs[fromNode.ID()] + _, toIsIndirection := indirectionNodeIDs[toNode.ID()] + if fromIsIndirection && toIsIndirection { + // Both sides are indirections; no acquisition or streaming needed. + // Just ensure edge dependencies are registered. + fromIdx := dp.nodeIDIdxMap[fromNode.ID()] + toIdx := dp.nodeIDIdxMap[toNode.ID()] + if !dp.dataflowEdgeExists(fromIdx, toIdx) { + dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) + } + continue + } + if fromIsIndirection { + // Source is indirection, destination is a regular table. + fromIdx := dp.nodeIDIdxMap[fromNode.ID()] + toIdx, toBuilderExists := dp.nodeIDIdxMap[toNode.ID()] + if !toBuilderExists { + toInsPsc, pscExists := insertPrepearedStatements[toNode.ID()] + if !pscExists { + return fmt.Errorf("unknown insert prepared statement") + } + dp.annMap[toTableExpr] = toAnnotation + toAnnotation.SetDynamic() + arrivingDestinationNodeStream := nodeStreamCollections.GetArriving(toNode.ID()) + departingDestinationNodeStream := nodeStreamCollections.GetDeparting(toNode.ID()) + var toErr error + toIdx, toErr = dp.orchestrate( + -1, toAnnotation, toInsPsc, arrivingDestinationNodeStream, departingDestinationNodeStream) + if toErr != nil { + return toErr + } + dp.nodeIDIdxMap[toNode.ID()] = toIdx + } + if !dp.dataflowEdgeExists(fromIdx, toIdx) { + dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) + } + continue + } + if toIsIndirection { + // Destination is indirection, source is a regular table. + toIdx := dp.nodeIDIdxMap[toNode.ID()] + fromIdx, fromBuilderExists := dp.nodeIDIdxMap[fromNode.ID()] + if !fromBuilderExists { + insPsc, pscExists := insertPrepearedStatements[fromNode.ID()] + if !pscExists { + return fmt.Errorf("unknown insert prepared statement") + } + arrivingSourceNodeStream := nodeStreamCollections.GetArriving(fromNode.ID()) + departingSourceNodeStream := nodeStreamCollections.GetDeparting(fromNode.ID()) + var fromErr error + fromIdx, fromErr = dp.orchestrate(-1, fromAnnotation, insPsc, arrivingSourceNodeStream, departingSourceNodeStream) + if fromErr != nil { + return fromErr + } + dp.nodeIDIdxMap[fromNode.ID()] = fromIdx + } + if !dp.dataflowEdgeExists(fromIdx, toIdx) { + dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) + } + continue + } + // Neither side is an indirection; original logic. departingSourceNodeStream := nodeStreamCollections.GetDeparting(fromNode.ID()) arrivingDestinationNodeStream := nodeStreamCollections.GetArriving(toNode.ID()) arrivingSourceNodeStream := nodeStreamCollections.GetArriving(fromNode.ID()) diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 82777548..e5fad24f 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -9478,3 +9478,85 @@ Left Outer Join Positive LHS Inline ... ${outputStr} ... stdout=${CURDIR}/tmp/Left-Outer-Join-Positive-LHS-Inline.tmp ... stderr=${CURDIR}/tmp/Left-Outer-Join-Positive-LHS-Inline-stderr.tmp + +View Depth Limitation Error Message Shows Correct Max + Should Stackql Exec Inline Contain Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... create view zz1 as select name from stackql_repositories; create view zz2 as select name from zz1; create view zz3 as select name from zz2; create view zz4 as select name from zz3; create view zz5 as select name from zz4; select * from zz5; + ... indirection chain length 6 > 5 + ... stdout=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stdout.tmp + ... stderr=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stderr.tmp + +View JOIN View Returns Results + ${inputStr} = Catenate + ... create or replace view vw_repos_name as select name from stackql_repositories; + ... create or replace view vw_repos_url as select name, url from stackql_repositories; + ... select v1.name from vw_repos_name v1 inner join vw_repos_url v2 on v1.name = v2.name; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/View-JOIN-View-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/View-JOIN-View-Returns-Results-stderr.tmp + +View JOIN Provider Table Returns Results + ${inputStr} = Catenate + ... create or replace view vw_repos as select name, url from stackql_repositories; + ... select v1.name from vw_repos v1 inner join github.repos.repos r on v1.name = r.name where r.org = 'stackql'; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/View-JOIN-Provider-Table-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/View-JOIN-Provider-Table-Returns-Results-stderr.tmp + +Subquery JOIN Subquery Returns Results + ${inputStr} = Catenate + ... select a.name from (select name from stackql_repositories) a inner join (select name, url from stackql_repositories) b on a.name = b.name; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/Subquery-JOIN-Subquery-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/Subquery-JOIN-Subquery-Returns-Results-stderr.tmp + +CTE Within View Returns Results + ${inputStr} = Catenate + ... create or replace view vw_cte as with sub as (select name from stackql_repositories) select name from sub; + ... select name from vw_cte; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/CTE-Within-View-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/CTE-Within-View-Returns-Results-stderr.tmp From 67fe19e042d62d0bbc2b8071c5c3a952803295aa Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 18:56:48 +1100 Subject: [PATCH 03/16] shim --- .../stackql/dependencyplanner/dependencyplanner.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index afcdf508..b234d9e2 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -113,17 +113,11 @@ func isAnnotationIndirection(annotationCtx taxonomy.AnnotationCtx) bool { // Unlike regular tables that need HTTP acquisition, indirections are already materialized by the // "create tail" builder and appear in the FROM clause as inline subqueries. // This method creates a NopBuilder as a placeholder in the execution DAG. +// Indirections are NOT added to dp.tableSlice because they appear as inline subqueries +// with their own internal control columns; adding them would generate incorrect outer WHERE clauses. func (dp *standardDependencyPlanner) orchestrateIndirection( annotationCtx taxonomy.AnnotationCtx, ) (int, error) { - rc, err := tableinsertioncontainer.NewTableInsertionContainer( - annotationCtx.GetTableMeta(), - dp.handlerCtx.GetSQLEngine(), - dp.handlerCtx.GetTxnCounterMgr(), - ) - if err != nil { - return -1, err - } builder := primitivebuilder.NewNopBuilder( dp.primitiveComposer.GetGraphHolder(), dp.primitiveComposer.GetTxnCtrlCtrs(), @@ -133,7 +127,6 @@ func (dp *standardDependencyPlanner) orchestrateIndirection( ) dp.execSlice = append(dp.execSlice, builder) idx := len(dp.execSlice) - 1 - dp.tableSlice = append(dp.tableSlice, rc) return idx, nil } From 066a666e551632be0d9cb35e60e3e282eddd9ed8 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 19:26:47 +1100 Subject: [PATCH 04/16] shim --- .github/workflows/build.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f228c4c..90595037 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1388,6 +1388,21 @@ jobs: if: always() run: | cat ./test/robot/reports/output.xml + + - name: Upload Robot Reports Artifact + uses: actions/upload-artifact@v4.3.1 + if: always() && github.repository == 'stackql/stackql-devel' + with: + name: stackql_darwin_amd64_robot_reports + path: test/robot/reports + + + - name: Upload Robot Tmp Artifact + uses: actions/upload-artifact@v4.3.1 + if: always() && github.repository == 'stackql/stackql-devel' + with: + name: stackql_darwin_amd64_robot_tmp + path: test/robot/functional/tmp - name: Run robot integration tests if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') From 863285ad561bbdc9a16b238458d1083db4178610 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 22:16:44 +1100 Subject: [PATCH 05/16] remove-naive-query-framentation --- .../tsm_physio/best_effort_orchestrator.go | 5 +-- .../acid/tsm_physio/txn_orchestrator.go | 5 +-- internal/stackql/cmd/shell.go | 31 +++++++++++-------- internal/stackql/driver/driver.go | 3 -- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go b/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go index 5a8fc38f..bae3091e 100644 --- a/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go +++ b/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go @@ -2,8 +2,8 @@ package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature import ( "fmt" - "strings" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/acid/tsm" "github.com/stackql/stackql/internal/stackql/acid/txn_context" @@ -42,7 +42,8 @@ func (orc *bestEffortOrchestrator) processQueryOrQueries( ) ([]internaldto.ExecutorOutput, bool) { var retVal []internaldto.ExecutorOutput cmdString := handlerCtx.GetRawQuery() - for _, s := range strings.Split(cmdString, ";") { + splitQueries, _ := sqlparser.SplitStatementToPieces(cmdString) + for _, s := range splitQueries { response, hasResponse := orc.processQuery(handlerCtx, s) if hasResponse { retVal = append(retVal, response...) diff --git a/internal/stackql/acid/tsm_physio/txn_orchestrator.go b/internal/stackql/acid/tsm_physio/txn_orchestrator.go index 8f13349c..bd48f96a 100644 --- a/internal/stackql/acid/tsm_physio/txn_orchestrator.go +++ b/internal/stackql/acid/tsm_physio/txn_orchestrator.go @@ -2,9 +2,9 @@ package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature import ( "fmt" - "strings" "github.com/stackql/any-sdk/pkg/constants" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/tsm" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/handler" @@ -68,7 +68,8 @@ func (orc *standardOrchestrator) processQueryOrQueries( ) ([]internaldto.ExecutorOutput, bool) { var retVal []internaldto.ExecutorOutput cmdString := handlerCtx.GetRawQuery() - for _, s := range strings.Split(cmdString, ";") { + splitQueries, _ := sqlparser.SplitStatementToPieces(cmdString) + for _, s := range splitQueries { response, hasResponse := orc.processQuery(handlerCtx, s) if hasResponse { retVal = append(retVal, response...) diff --git a/internal/stackql/cmd/shell.go b/internal/stackql/cmd/shell.go index b2b6cb47..1952aa2a 100644 --- a/internal/stackql/cmd/shell.go +++ b/internal/stackql/cmd/shell.go @@ -25,6 +25,7 @@ import ( "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/config" "github.com/stackql/stackql/internal/stackql/driver" "github.com/stackql/stackql/internal/stackql/entryutil" @@ -225,20 +226,24 @@ var shellCmd = &cobra.Command{ if inlineCommentIdx > -1 { line = line[:inlineCommentIdx] } - semiColonIdx := strings.Index(line, ";") - if semiColonIdx > -1 { - line = strings.TrimSpace(line[:semiColonIdx+1]) - subSemiColonIdx := strings.Index(line, ";") - sb.WriteString(" " + line[:subSemiColonIdx+1]) - rawQuery := sb.String() - queryToExecute, qErr := entryutil.PreprocessInline(runtimeCtx, rawQuery) - if qErr != nil { - io.WriteString(outErrFile, "\r\n"+qErr.Error()+"\r\n") //nolint:errcheck // TODO: investigate + hasRHSSemiColon := strings.HasSuffix(strings.TrimSpace(line), ";") + splitQueries, _ := sqlparser.SplitStatementToPieces(line) + if len(splitQueries) > 0 { + for i, s := range splitQueries { + if i == len(splitQueries)-1 && !hasRHSSemiColon { + sb.Reset() + sb.WriteString(s) + } + line = s + sb.WriteString(" " + line) + rawQuery := sb.String() + queryToExecute, qErr := entryutil.PreprocessInline(runtimeCtx, rawQuery) + if qErr != nil { + io.WriteString(outErrFile, "\r\n"+qErr.Error()+"\r\n") //nolint:errcheck // TODO: investigate + } + l.WriteToHistory(rawQuery) //nolint:errcheck // TODO: investigate + sessionRunnerInstance.RunCommand(queryToExecute) } - l.WriteToHistory(rawQuery) //nolint:errcheck // TODO: investigate - sessionRunnerInstance.RunCommand(queryToExecute) - sb.Reset() - sb.WriteString(line[subSemiColonIdx+1:]) } else { sb.WriteString(" " + line) } diff --git a/internal/stackql/driver/driver.go b/internal/stackql/driver/driver.go index b807cb6b..96d63745 100644 --- a/internal/stackql/driver/driver.go +++ b/internal/stackql/driver/driver.go @@ -147,9 +147,6 @@ func (dr *basicStackQLDriver) CloneSQLBackend() sqlbackend.ISQLBackend { //nolint:revive // TODO: review func (dr *basicStackQLDriver) HandleSimpleQuery(ctx context.Context, query string) (sqldata.ISQLResultStream, error) { dr.handlerCtx.SetRawQuery(query) - // if strings.Count(query, ";") > 1 { - // return nil, fmt.Errorf("only support single queries in server mode at this time") - // } res, ok := dr.processQueryOrQueries(dr.handlerCtx) if !ok { return nil, fmt.Errorf("no SQLresults available") From 9ba50957dfc7c3a043d1ad9057fdbf25ca90abc6 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Tue, 31 Mar 2026 22:47:47 +1100 Subject: [PATCH 06/16] - Added robot test `Shell Session Multiple Statements Inline`. - Added robot test `Shell Session Multi Line Then Multi Statement`. --- .../stackql_test_tooling/stackql_context.py | 2 ++ test/robot/functional/stackql_sessions.robot | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/test/python/stackql_test_tooling/stackql_context.py b/test/python/stackql_test_tooling/stackql_context.py index 1b4b1604..d6e661eb 100644 --- a/test/python/stackql_test_tooling/stackql_context.py +++ b/test/python/stackql_test_tooling/stackql_context.py @@ -1097,6 +1097,8 @@ def get_registry_mock_url(execution_env :str) -> str: 'SHELL_SESSION_SIMPLE_COMMANDS_AFTER_ERROR': [ SELECT_GITHUB_BRANCHES_NAMES_DESC_WRONG_COLUMN, SELECT_AZURE_COMPUTE_VIRTUAL_MACHINES ], 'SHELL_SESSION_SIMPLE_COMMANDS_AFTER_ERROR_EXPECTED': SELECT_AZURE_COMPUTE_VIRTUAL_MACHINES_JSON_EXPECTED, 'SHELL_SESSION_SIMPLE_EXPECTED': get_shell_welcome_stdout(execution_env) + SELECT_GITHUB_BRANCHES_NAMES_DESC_EXPECTED, + 'SHELL_SESSION_MULTI_STMT_INLINE_COMMANDS': [ "SELECT * FROM stackql_repositories; " + SELECT_GITHUB_BRANCHES_NAMES_DESC ], + 'SHELL_SESSION_MULTI_LINE_THEN_MULTI_STMT_COMMANDS': [ "select name", "from github.repos.branches where owner = 'dummyorg' and repo = 'dummyapp.io' order by name desc;", "SELECT * FROM stackql_repositories; " + SELECT_GITHUB_BRANCHES_NAMES_DESC ], 'SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR': SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR, 'SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR': SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR, 'SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR_EXPECTED': SHOW_INSERT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY_ERROR_EXPECTED, diff --git a/test/robot/functional/stackql_sessions.robot b/test/robot/functional/stackql_sessions.robot index cd00dcea..fda928a0 100644 --- a/test/robot/functional/stackql_sessions.robot +++ b/test/robot/functional/stackql_sessions.robot @@ -32,6 +32,36 @@ Shell Session Azure Compute Table Nomenclature Mutation Guard ... stdout=${CURDIR}/tmp/Shell-Session-Azure-Compute-Table-Nomenclature-Mutation-Guard.tmp [Teardown] Stackql Per Test Teardown +Shell Session Multiple Statements Inline + Pass Execution If "${IS_WINDOWS}" == "1" Skipping session test in windows + Should StackQL Shell Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${SHELL_SESSION_MULTI_STMT_INLINE_COMMANDS} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/Shell-Session-Multiple-Statements-Inline.tmp + [Teardown] Stackql Per Test Teardown + +Shell Session Multi Line Then Multi Statement + Pass Execution If "${IS_WINDOWS}" == "1" Skipping session test in windows + Should StackQL Shell Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${SHELL_SESSION_MULTI_LINE_THEN_MULTI_STMT_COMMANDS} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/Shell-Session-Multi-Line-Then-Multi-Statement.tmp + [Teardown] Stackql Per Test Teardown + PG Session GC Manual Behaviour Canonical Should PG Client Session Inline Equal Strict ... ${PSQL_MTLS_CONN_STR_UNIX} From 017ff367c7752d3ac769c45813dc2f78b8856ff4 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 00:26:51 +1100 Subject: [PATCH 07/16] re-try --- internal/stackql/router/parameter_router.go | 7 +++-- .../stackql_mocked_from_cmd_line.robot | 29 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/stackql/router/parameter_router.go b/internal/stackql/router/parameter_router.go index c6cf89d1..3c6d9946 100644 --- a/internal/stackql/router/parameter_router.go +++ b/internal/stackql/router/parameter_router.go @@ -531,7 +531,9 @@ func (pr *standardParameterRouter) route( } t, ok := pr.tablesAliasMap[alias] if !ok { - return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) + // Skip params from parent queries whose alias is not in the current scope. + // This occurs when indirections (views, subqueries) inherit parent WHERE params. + continue } if t == tb { ref, ok := pr.colRefs[k] @@ -549,7 +551,8 @@ func (pr *standardParameterRouter) route( } t, ok := pr.tablesAliasMap[alias] if !ok { - return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) + // Skip params from parent queries whose alias is not in the current scope. + continue } if t == tb { ref, ok := pr.colRefs[k] diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index e5fad24f..9d319855 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -9479,20 +9479,6 @@ Left Outer Join Positive LHS Inline ... stdout=${CURDIR}/tmp/Left-Outer-Join-Positive-LHS-Inline.tmp ... stderr=${CURDIR}/tmp/Left-Outer-Join-Positive-LHS-Inline-stderr.tmp -View Depth Limitation Error Message Shows Correct Max - Should Stackql Exec Inline Contain Stderr - ... ${STACKQL_EXE} - ... ${OKTA_SECRET_STR} - ... ${GITHUB_SECRET_STR} - ... ${K8S_SECRET_STR} - ... ${REGISTRY_NO_VERIFY_CFG_STR} - ... ${AUTH_CFG_STR} - ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... create view zz1 as select name from stackql_repositories; create view zz2 as select name from zz1; create view zz3 as select name from zz2; create view zz4 as select name from zz3; create view zz5 as select name from zz4; select * from zz5; - ... indirection chain length 6 > 5 - ... stdout=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stdout.tmp - ... stderr=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stderr.tmp - View JOIN View Returns Results ${inputStr} = Catenate ... create or replace view vw_repos_name as select name from stackql_repositories; @@ -9528,6 +9514,21 @@ View JOIN Provider Table Returns Results ... stdout=${CURDIR}/tmp/View-JOIN-Provider-Table-Returns-Results-stdout.tmp ... stderr=${CURDIR}/tmp/View-JOIN-Provider-Table-Returns-Results-stderr.tmp +View Depth Limitation Error Message Shows Correct Max + Should Stackql Exec Inline Contain Stderr + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... create view zz1 as select name from stackql_repositories; create view zz2 as select name from zz1; create view zz3 as select name from zz2; create view zz4 as select name from zz3; create view zz5 as select name from zz4; select * from zz5; + ... indirection chain length 6 > 5 + ... stdout=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stdout.tmp + ... stderr=${CURDIR}/tmp/View-Depth-Limitation-Error-Message-Shows-Correct-Max-stderr.tmp + + Subquery JOIN Subquery Returns Results ${inputStr} = Catenate ... select a.name from (select name from stackql_repositories) a inner join (select name, url from stackql_repositories) b on a.name = b.name; From cee8782cba00fd7e3dd4b626d511b34e52201d38 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 00:33:11 +1100 Subject: [PATCH 08/16] re-try --- .../stackql/astanalysis/earlyanalysis/ast_expand.go | 11 ++++++++++- internal/stackql/router/parameter_router.go | 7 ++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go index 2419d13c..c951bc0b 100644 --- a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go +++ b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go @@ -152,10 +152,19 @@ func (v *indirectExpandAstVisitor) processIndirect(node sqlparser.SQLNode, indir if err != nil { return nil //nolint:nilerr //TODO: investigate } + // Filter parent WHERE params to only pass down unqualified (alias-free) entries. + // Aliased params like "r.org" reference specific outer tables and must not + // leak into child indirection analysis, where the alias would be unresolvable. + filteredWhereParams := parserutil.NewParameterMap() + for k, val := range v.whereParams.GetMap() { + if k.Alias() == "" { + filteredWhereParams.Set(k, val) //nolint:errcheck // best effort + } + } childAnalyzer, err := NewEarlyScreenerAnalyzer( v.primitiveGenerator, v.annotatedAST, - v.whereParams.Clone(), + filteredWhereParams, v.indirectionDepth+1, ) if err != nil { diff --git a/internal/stackql/router/parameter_router.go b/internal/stackql/router/parameter_router.go index 3c6d9946..c6cf89d1 100644 --- a/internal/stackql/router/parameter_router.go +++ b/internal/stackql/router/parameter_router.go @@ -531,9 +531,7 @@ func (pr *standardParameterRouter) route( } t, ok := pr.tablesAliasMap[alias] if !ok { - // Skip params from parent queries whose alias is not in the current scope. - // This occurs when indirections (views, subqueries) inherit parent WHERE params. - continue + return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) } if t == tb { ref, ok := pr.colRefs[k] @@ -551,8 +549,7 @@ func (pr *standardParameterRouter) route( } t, ok := pr.tablesAliasMap[alias] if !ok { - // Skip params from parent queries whose alias is not in the current scope. - continue + return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) } if t == tb { ref, ok := pr.colRefs[k] From f290b60fc9cdb2301d57854acf759481e424f5da Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 00:36:38 +1100 Subject: [PATCH 09/16] linter-fixes --- internal/stackql/dependencyplanner/dependencyplanner.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index b234d9e2..dffcb672 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -115,6 +115,8 @@ func isAnnotationIndirection(annotationCtx taxonomy.AnnotationCtx) bool { // This method creates a NopBuilder as a placeholder in the execution DAG. // Indirections are NOT added to dp.tableSlice because they appear as inline subqueries // with their own internal control columns; adding them would generate incorrect outer WHERE clauses. +// +//nolint:unparam,revive // acceptable func (dp *standardDependencyPlanner) orchestrateIndirection( annotationCtx taxonomy.AnnotationCtx, ) (int, error) { @@ -258,6 +260,7 @@ func (dp *standardDependencyPlanner) Plan() error { continue } for _, e := range edges { + //nolint:nestif // necessary to handle indirection cases cleanly if e.From().ID() == n.ID() { insPsc, tcc, insErr := dp.processOrphan(tableExpr, annotation, n) if insErr != nil { @@ -317,6 +320,7 @@ func (dp *standardDependencyPlanner) Plan() error { } continue } + //nolint:nestif // necessary to handle indirection cases cleanly if fromIsIndirection { // Source is indirection, destination is a regular table. fromIdx := dp.nodeIDIdxMap[fromNode.ID()] @@ -343,6 +347,7 @@ func (dp *standardDependencyPlanner) Plan() error { } continue } + //nolint:nestif // necessary to handle indirection cases cleanly if toIsIndirection { // Destination is indirection, source is a regular table. toIdx := dp.nodeIDIdxMap[toNode.ID()] From 5e12001646d2557fdb734cb03c67c79a086773e4 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 00:42:28 +1100 Subject: [PATCH 10/16] bugfix --- internal/stackql/cmd/shell.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/stackql/cmd/shell.go b/internal/stackql/cmd/shell.go index 1952aa2a..188238d5 100644 --- a/internal/stackql/cmd/shell.go +++ b/internal/stackql/cmd/shell.go @@ -231,11 +231,13 @@ var shellCmd = &cobra.Command{ if len(splitQueries) > 0 { for i, s := range splitQueries { if i == len(splitQueries)-1 && !hasRHSSemiColon { + // Last piece has no trailing semicolon; + // accumulate for multi-line continuation. sb.Reset() sb.WriteString(s) + continue } - line = s - sb.WriteString(" " + line) + sb.WriteString(" " + s) rawQuery := sb.String() queryToExecute, qErr := entryutil.PreprocessInline(runtimeCtx, rawQuery) if qErr != nil { @@ -243,6 +245,7 @@ var shellCmd = &cobra.Command{ } l.WriteToHistory(rawQuery) //nolint:errcheck // TODO: investigate sessionRunnerInstance.RunCommand(queryToExecute) + sb.Reset() } } else { sb.WriteString(" " + line) From 806f6cb4496fd3f61f1e2fa184b4928022f4023c Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 01:31:57 +1100 Subject: [PATCH 11/16] post-debug --- .vscode/launch.json | 1 + internal/stackql/astvisit/from_rewrite.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 06f9d164..815a876e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -198,6 +198,7 @@ "select lhs.proj, lhs.bucket from (select 'testing-project' as proj, 'silly-bucket' as bucket) lhs LEFT OUTER join (select name from google.storage.buckets where project = 'testing-project') rhs on lhs.bucket = rhs.name where rhs.name;", "insert into google.storage.buckets( project, data__name) select lhs.proj, lhs.bucket from (select 'testing-project' as proj, 'silly-bucket' as bucket) lhs LEFT OUTER join (select name from google.storage.buckets where project = 'testing-project') rhs on lhs.bucket = rhs.name where rhs.name is null returning *;", "select description, price_monthly, price_hourly from digitalocean.sizes.sizes where price_monthly = 7.0 order by description desc;", + "create or replace view vw_repos_name as select name from stackql_repositories; create or replace view vw_repos_url as select name, url from stackql_repositories; select v1.name from vw_repos_name v1 inner join vw_repos_url v2 on v1.name = v2.name;", ], "default": "show providers;" }, diff --git a/internal/stackql/astvisit/from_rewrite.go b/internal/stackql/astvisit/from_rewrite.go index 5a81ccea..7c81aa0b 100644 --- a/internal/stackql/astvisit/from_rewrite.go +++ b/internal/stackql/astvisit/from_rewrite.go @@ -650,6 +650,7 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { case *sqlparser.AliasedTableExpr: var exprStr, partitionStr string + aliasHandledByIndirect := false if node.Expr != nil { anCtx, ok := v.annotations[node] if !ok { @@ -661,12 +662,18 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { if indirect, isIndirect := anCtx.GetTableMeta().GetIndirect(); isIndirect { // name := indirect.GetName() + // Use the user-specified alias if present, otherwise the view/indirect name. + // This prevents double aliasing when the node.As fallthrough appends the alias again. + if !node.As.IsEmpty() { + name = node.As.GetRawVal() + } indirectType := indirect.GetType() switch indirectType { case astindirect.ViewType: templateString := fmt.Sprintf(` ( %%s ) AS "%s" `, name) v.rewrittenQuery = templateString v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext()) + aliasHandledByIndirect = true case astindirect.SubqueryType: // Note: CTEs are converted to SubqueryType at AST level, // so this path handles both regular subqueries and CTEs. @@ -726,7 +733,7 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { partitionStr = v.GetRewrittenQuery() } q := fmt.Sprintf("%s%s", exprStr, partitionStr) - if !node.As.IsEmpty() { + if !node.As.IsEmpty() && !aliasHandledByIndirect { node.As.Accept(v) asStr := v.GetRewrittenQuery() q = fmt.Sprintf("%s as %v", q, asStr) From 5c9a02298bd4e5397d5b084d04e819fdb71d7723 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 06:55:26 +1100 Subject: [PATCH 12/16] alias-verrride-on-view-only --- internal/stackql/astvisit/from_rewrite.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/stackql/astvisit/from_rewrite.go b/internal/stackql/astvisit/from_rewrite.go index 7c81aa0b..620ff81c 100644 --- a/internal/stackql/astvisit/from_rewrite.go +++ b/internal/stackql/astvisit/from_rewrite.go @@ -662,15 +662,17 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { if indirect, isIndirect := anCtx.GetTableMeta().GetIndirect(); isIndirect { // name := indirect.GetName() - // Use the user-specified alias if present, otherwise the view/indirect name. - // This prevents double aliasing when the node.As fallthrough appends the alias again. - if !node.As.IsEmpty() { - name = node.As.GetRawVal() - } indirectType := indirect.GetType() switch indirectType { case astindirect.ViewType: - templateString := fmt.Sprintf(` ( %%s ) AS "%s" `, name) + // Use the user-specified alias if present, otherwise the view name. + // The alias is embedded in the template to prevent double aliasing + // when the node.As fallthrough at the end of this case would append it again. + viewAlias := name + if !node.As.IsEmpty() { + viewAlias = node.As.GetRawVal() + } + templateString := fmt.Sprintf(` ( %%s ) AS "%s" `, viewAlias) v.rewrittenQuery = templateString v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext()) aliasHandledByIndirect = true From ed6f0b399c8382a3a2d0a3a6af09c36e69726224 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 07:03:01 +1100 Subject: [PATCH 13/16] better-doco --- docs/views.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/views.md b/docs/views.md index ea468fa6..50fa6773 100644 --- a/docs/views.md +++ b/docs/views.md @@ -35,6 +35,14 @@ The runtime representation of views must support: - Acquistion occurs as normal through primitive DAG. - Selection phase uses physical views. +## Materialized views + +MAterialized views are similar in nature to views, although eager executed and lacking in mutation of internla `WHERE` clauses from outside. + +## User space tables + +These map to RDBMS tables. The DDL is somewhat impaired; we imagine these are suseful for staging in general and applications across: ELT, IAC. + ## Subqueries @@ -43,3 +51,8 @@ Some aspects of subquery analysis and execution will be similar to views, but no To be continued... +## Joins and aliasing on Views etc + +It is possible to join and alias collections of views, materialized views, subqueries and user space tables. + + From 855b77af0af2c8c0774b735b18cc71c4655b87dd Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 07:08:51 +1100 Subject: [PATCH 14/16] - View documentation. --- docs/views.md | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/views.md b/docs/views.md index 50fa6773..04d86ad9 100644 --- a/docs/views.md +++ b/docs/views.md @@ -2,7 +2,7 @@ # Views -## *a priori* +## *a priori* At definition time, it is apparent: @@ -24,7 +24,7 @@ The runtime representation of views must support: - StackQL views DDL stored in some special stackql table designated for this purpose. - Physical table name such as `__iql__.views`. - Views need not exist until the `SELECT ... FROM ` portion of the query is executed. - This is advantagesous on RDBMS systems where view creation will fail if physical tables do not exist. + This is advantageous on RDBMS systems where view creation will fail if physical tables do not exist. - We may need a layer of indirection for views to execute, wrt table names containing generation ID. Simplest option is input table name. - SQL view definitions (translated to physical tables) are stored in the RDBMS. @@ -32,27 +32,59 @@ The runtime representation of views must support: - Some part of the namespace must be reserved for these views; configurable using existing regex / template namespacing? - Quite possibly some specialised object(s) or extension of the `table` interface stages are used for view analysis and parameter routing. - Once analysis is complete: - - Acquistion occurs as normal through primitive DAG. + - Acquisition occurs as normal through primitive DAG. - Selection phase uses physical views. ## Materialized views -MAterialized views are similar in nature to views, although eager executed and lacking in mutation of internla `WHERE` clauses from outside. +Materialized views are similar in nature to views, although eager executed and lacking in mutation of internal `WHERE` clauses from outside. ## User space tables -These map to RDBMS tables. The DDL is somewhat impaired; we imagine these are suseful for staging in general and applications across: ELT, IAC. +These map to RDBMS tables. The DDL is somewhat impaired; we imagine these are useful for staging in general and applications across: ELT, IAC. ## Subqueries -Some aspects of subquery analysis and execution will be similar to views, but not all. What are the considerations for view implementation in the short term such that subsequent subquery implmentation is expedited and natural. +Some aspects of subquery analysis and execution will be similar to views, but not all. What are the considerations for view implementation in the short term such that subsequent subquery implementation is expedited and natural. To be continued... ## Joins and aliasing on Views etc -It is possible to join and alias collections of views, materialized views, subqueries and user space tables. +### Views (lazy evaluated) +Views are rendered as inline subqueries `( SELECT ... ) AS "alias"` in the final SQL. When a user alias is provided (e.g. `FROM my_view v1`), the alias `v1` replaces the view name in the `AS` clause. +**Supported:** +- View aliased and selected from: `SELECT * FROM my_view v1`. +- View JOIN view: `SELECT ... FROM v1 INNER JOIN v2 ON ...`. +- View JOIN provider table: `SELECT ... FROM my_view v1 INNER JOIN provider.svc.resource r ON ...`. +- View JOIN subquery: `SELECT ... FROM my_view v1 INNER JOIN (SELECT ...) sq ON ...`. +- View JOIN materialized view: `SELECT ... FROM my_view v1 INNER JOIN mv ON ...`. +- Nested views (view wrapping a view): supported up to configurable depth (`--indirect-depth-max`, default 5). +- WHERE clause parameter clobbering from outside the view, using **unqualified** parameters (e.g. `WHERE region = 'us-east-1'`). + +**Not supported:** +- Table-qualified parameter clobbering into views (e.g. `WHERE v1.region = 'us-east-1'` will not override the view's internal `region` parameter). + +### Materialized views (eager evaluated) + +Materialized views are persisted as physical tables in the RDBMS. They are referenced by their table name directly (not as inline subqueries). + +**Supported:** +- Materialized view aliased and selected from. +- Materialized view joined with provider tables, user space tables, views and subqueries. +- `CREATE`, `DROP`, `REFRESH`, `CREATE OR REPLACE` lifecycle. + +**Not supported:** +- WHERE clause parameter clobbering from outside (materialized views are snapshot-based). + +### Subqueries + +Subqueries appear as inline `( SELECT ... )` expressions. CTEs (`WITH ... AS`) are converted to subqueries at AST level and handled identically. + +### User space tables + +User space tables are RDBMS-resident tables created via `CREATE TABLE`. They can participate in joins with any other indirection type. From fc4b216f5a0d978de2d7b98943795309e5af0361 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 07:11:17 +1100 Subject: [PATCH 15/16] - Added robot test `View JOIN Materialized View Returns Results`. - Added robot test `View JOIN Subquery Returns Results`. --- .../stackql_mocked_from_cmd_line.robot | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 9d319855..de5750b1 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -9545,6 +9545,41 @@ Subquery JOIN Subquery Returns Results ... stdout=${CURDIR}/tmp/Subquery-JOIN-Subquery-Returns-Results-stdout.tmp ... stderr=${CURDIR}/tmp/Subquery-JOIN-Subquery-Returns-Results-stderr.tmp +View JOIN Subquery Returns Results + ${inputStr} = Catenate + ... create or replace view vw_repos as select name, url from stackql_repositories; + ... select v1.name from vw_repos v1 inner join (select name from stackql_repositories) sq on v1.name = sq.name; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/View-JOIN-Subquery-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/View-JOIN-Subquery-Returns-Results-stderr.tmp + +View JOIN Materialized View Returns Results + ${inputStr} = Catenate + ... create or replace view vw_repos as select name, url from stackql_repositories; + ... create or replace materialized view mv_repos as select name from stackql_repositories; + ... select v1.name from vw_repos v1 inner join mv_repos mv on v1.name = mv.name; + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... dummyapp.io + ... stdout=${CURDIR}/tmp/View-JOIN-Materialized-View-Returns-Results-stdout.tmp + ... stderr=${CURDIR}/tmp/View-JOIN-Materialized-View-Returns-Results-stderr.tmp + CTE Within View Returns Results ${inputStr} = Catenate ... create or replace view vw_cte as with sub as (select name from stackql_repositories) select name from sub; From 8508508a241d7e2a83d665e121b626186ab76fc2 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Wed, 1 Apr 2026 07:12:18 +1100 Subject: [PATCH 16/16] shortcoming-listed --- docs/views.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/views.md b/docs/views.md index 04d86ad9..1f09d1b6 100644 --- a/docs/views.md +++ b/docs/views.md @@ -68,6 +68,7 @@ Views are rendered as inline subqueries `( SELECT ... ) AS "alias"` in the final **Not supported:** - Table-qualified parameter clobbering into views (e.g. `WHERE v1.region = 'us-east-1'` will not override the view's internal `region` parameter). +- Joins of three or more heterogeneous indirections (e.g. `view JOIN subquery JOIN provider_table`). Binary joins work; three-way and beyond fail with parameter count mismatches in the SQL composition layer. ### Materialized views (eager evaluated)