Sync Repos: Release 170.179.0

.github/instructions/testing.guidelines.instructions.md

@@ -321,6 +321,25 @@ Expected: 'SELECT JSON_ARRAY('value1', 'value2');'
 
 **Solution**: Copy the "Actual" output to your baseline file (note spacing differences).
 
+**CRITICAL CHECK**: After copying baseline, compare it against your test script:
+```sql
+-- Input script (TestScripts/MyTest.sql)
+SELECT * FROM FUNC() WITH (HINT);
+
+-- Generated baseline (Baselines170/MyTest.sql)  
+SELECT *
+FROM FUNC();
+-- ⚠️ WHERE IS 'WITH (HINT)'?
+```
+
+**If baseline is missing syntax from input:**
+1. **This is likely a BUG** - not just formatting difference
+2. Check if AST has member to store the missing syntax
+3. Verify grammar stores value: `vResult.PropertyName = vValue;`
+4. Check script generator outputs the value: `GenerateFragmentIfNotNull(node.Property)`
+5. If syntax should be preserved, add AST storage and script generation
+6. Document in spec if intentional omission (e.g., query optimizer hints)
+
 #### 2. Error Count Mismatch  
 ```
 TestYourFeature.sql: number of errors after parsing is different from expected.
@@ -800,6 +819,38 @@ new ParserTest160("RegressionBugFix12345Tests160.sql", nErrors150: 1),  // Bug e
 new ParserTest170("RegressionBugFix12345Tests170.sql", nErrors160: 1),  // Bug existed in SQL 2022
 ```
 
+## Round-Trip Fidelity Validation Checklist
+
+**CRITICAL: After generating baseline files, ALWAYS verify:**
+
+✅ **Input Preservation Check**:
+1. Open test script side-by-side with baseline file
+2. For each SQL statement, verify baseline preserves all syntax from input
+3. Check optional clauses: `WITH`, `WHERE`, `HAVING`, `ORDER BY`, etc.
+4. Check hints: Table hints, query hints, join hints
+5. Check keywords: All keywords from input should appear in baseline (unless documented normalization)
+
+✅ **Missing Syntax Investigation**:
+If baseline omits syntax from input:
+- [ ] Is this intentional keyword normalization? (e.g., APPROX → APPROXIMATE)
+- [ ] Is this a query optimizer hint that doesn't need preservation?
+- [ ] Is this a BUG where AST doesn't store the value?
+
+✅ **Bug Indicators**:
+- ❌ Input: `FUNCTION() WITH (HINT)` → Baseline: `FUNCTION()` = **LIKELY BUG**
+- ❌ Input: `SELECT ... ORDER BY col` → Baseline: `SELECT ...` = **BUG**
+- ✅ Input: `FETCH APPROX` → Baseline: `FETCH APPROXIMATE` = Acceptable normalization
+- ✅ Input: `SELECT /*+ HINT */` → Baseline: `SELECT` = Query hint (document in spec)
+
+✅ **Resolution Steps**:
+1. Check AST definition in `Ast.xml` for member to store value
+2. Verify grammar assigns value: `vResult.Property = vValue;`
+3. Check script generator outputs value: `if (node.Property != null) { ... }`
+4. If missing: Add AST member, update grammar, update script generator, rebuild
+5. Document decision in spec if intentional omission
+
+---
+
 ## Summary
 
 The SqlScriptDOM testing framework provides comprehensive validation of parser functionality through:
@@ -808,7 +859,11 @@ The SqlScriptDOM testing framework provides comprehensive validation of parser f
 - **Cross-version validation** (Test syntax across SQL Server versions)
 - **Error condition testing** (Invalid syntax produces expected errors)
 - **Exact syntax verification** (Exact T-SQL from user requests is tested precisely)
+- **Round-trip fidelity validation** (Baseline preserves all input syntax unless documented)
 
 Following these guidelines ensures robust test coverage for parser functionality and prevents regressions when adding new features or fixing bugs.
 
-**Key Principle**: Always test the exact T-SQL syntax provided in user prompts or requests to verify that the specific syntax works as expected, rather than testing generalized or simplified versions of the syntax.
+**Key Principles**: 
+1. Always test the exact T-SQL syntax provided in user prompts or requests
+2. Always verify baseline output preserves input syntax (missing syntax may indicate bugs)
+3. Document any intentional omissions (normalization, query hints) in spec

.gitignore

@@ -359,4 +359,15 @@ out/
 .packages/
 
 # Temporary build artifacts
-tmp/
+tmp/
+
+# Speckit files
+speckit.files
+.github/.specify/
+.github/agents/speckit.*.agent.md
+.github/prompts/speckit.*
+
+# Specs directory - ignore all files except spec.md
+specs/**/*
+!specs/**/
+!specs/**/spec.md

SqlScriptDom/Parser/TSql/Ast.xml

@@ -4808,5 +4808,6 @@
     <Member Name="SimilarTo" Type="ScalarExpression" Summary="The vector used for search." />
     <Member Name="Metric" Type="StringLiteral" Summary="The distance metric to use for the search." />
     <Member Name="TopN" Type="ScalarExpression" Summary="The maximum number of similar vectors that must be returned." />
+    <Member Name="ForceAnnOnly" Type="bool" Summary="Whether the WITH (FORCE_ANN_ONLY) hint is specified." />
   </Class>
 </Types>

SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs

@@ -75,6 +75,7 @@ internal static class CodeGenerationSupporter
         internal const string AnsiWarnings = "ANSI_WARNINGS";
         internal const string ForcePlan = "FORCEPLAN";
         internal const string ForAppend = "FOR_APPEND";
+        internal const string ForceAnnOnly = "FORCE_ANN_ONLY";
         internal const string ShowPlanAll = "SHOWPLAN_ALL";
         internal const string ShowPlanText = "SHOWPLAN_TEXT";
         internal const string IO = "IO";

SqlScriptDom/Parser/TSql/TSql170.g

@@ -19321,19 +19321,35 @@ vectorSearchTableReference returns [VectorSearchTableReference vResult = Fragmen
             MatchString(vMetric, CodeGenerationSupporter.Cosine, CodeGenerationSupporter.Dot, CodeGenerationSupporter.Euclidean);
             vResult.Metric = vMetric;
         }
-        Comma tTopN:Identifier EqualsSign vTopN = signedIntegerOrVariableOrColumnReference
-        {
-            Match(tTopN, CodeGenerationSupporter.TopN);
-
-            // Validate that TOP_N is not a negative number
-            if (vTopN is UnaryExpression unaryExpr && unaryExpr.UnaryExpressionType == UnaryExpressionType.Negative)
+        // TOP_N is optional per SQL Server 2025 (commit 12d3e8fc)
+        (
+            Comma tTopN:Identifier EqualsSign vTopN = signedIntegerOrVariableOrColumnReference
             {
-                ThrowParseErrorException("SQL46010", unaryExpr, TSqlParserResource.SQL46010Message, "-");
+                Match(tTopN, CodeGenerationSupporter.TopN);
+
+                // Validate that TOP_N is not a negative number
+                if (vTopN is UnaryExpression unaryExpr && unaryExpr.UnaryExpressionType == UnaryExpressionType.Negative)
+                {
+                    ThrowParseErrorException("SQL46010", unaryExpr, TSqlParserResource.SQL46010Message, "-");
+                }
+
+                vResult.TopN = vTopN;
             }
-
-            vResult.TopN = vTopN;
+        )?
+        tRParen:RightParenthesis
+        {
+            UpdateTokenInfo(vResult, tRParen);
         }
-        RightParenthesis simpleTableReferenceAliasOpt[vResult]
+        // WITH clause per SQL Server 2025 (commit 12d3e8fc)
+        (
+            With LeftParenthesis tForceAnnOnly:Identifier tRParen2:RightParenthesis
+            {
+                Match(tForceAnnOnly, CodeGenerationSupporter.ForceAnnOnly);
+                UpdateTokenInfo(vResult, tRParen2);
+                vResult.ForceAnnOnly = true;
+            }
+        )?
+        simpleTableReferenceAliasOpt[vResult]
     ;
 
 predictTableReference[SubDmlFlags subDmlFlags] returns [PredictTableReference vResult]
@@ -33167,6 +33183,7 @@ jsonObjectBuiltInFunctionCall [FunctionCall vParent]
 
 jsonObjectAggBuiltInFunctionCall [FunctionCall vParent]
 {
+    OverClause vOverClause;
 }
     :   (
             jsonObjectAggExpressionList[vParent]
@@ -33177,6 +33194,12 @@ jsonObjectAggBuiltInFunctionCall [FunctionCall vParent]
         {
             UpdateTokenInfo(vParent, tRParen);
         }
+        (
+            vOverClause=overClauseNoOrderBy
+            {
+                vParent.OverClause = vOverClause;
+            }
+        )?
     ;
 
 jsonQueryBuiltInFunctionCall [FunctionCall vParent]

SqlScriptDom/Parser/TSql/TSqlFabricDW.g

@@ -23761,8 +23761,6 @@ dropSecurityPolicyStatement returns [DropSecurityPolicyStatement vResult = Fragm
 createExternalDataSourceStatement returns [CreateExternalDataSourceStatement vResult = FragmentFactory.CreateFragment<CreateExternalDataSourceStatement>()]
 {
     Identifier vName;
-    ExternalDataSourceOption vExternalDataSourceOption;
-    long encounteredOptions = 0;
     vResult.DataSourceType = ExternalDataSourceType.EXTERNAL_GENERICS;
 }
     :   tData:Identifier tSource:Identifier vName = identifier
@@ -23774,88 +23772,15 @@ createExternalDataSourceStatement returns [CreateExternalDataSourceStatement vRe
         }
 
         tWith:With LeftParenthesis
-        (
-            {NextTokenMatches(CodeGenerationSupporter.Type)}?
-            externalDataSourceType[vResult]
-        |
-            {NextTokenMatches(CodeGenerationSupporter.Location)}?
-            externalDataSourceLocation[vResult]
-        |
-            {NextTokenMatches(CodeGenerationSupporter.PushdownOption)}?
-            externalDataSourcePushdownOption[vResult]
-        |
-            vExternalDataSourceOption = externalDataSourceLiteralOrIdentifierOption
-            {
-                CheckOptionDuplication(ref encounteredOptions, (int)vExternalDataSourceOption.OptionKind, vExternalDataSourceOption);
-                AddAndUpdateTokenInfo(vResult, vResult.ExternalDataSourceOptions, vExternalDataSourceOption);
-            }
-        )
 
-        (
-            tComma:Comma
-            (
-                {NextTokenMatches(CodeGenerationSupporter.Type)}?
-                externalDataSourceType[vResult]
-            |
-                {NextTokenMatches(CodeGenerationSupporter.Location)}?
-                externalDataSourceLocation[vResult]
-            |
-                {NextTokenMatches(CodeGenerationSupporter.PushdownOption)}?
-                externalDataSourcePushdownOption[vResult]
-            |
-                vExternalDataSourceOption = externalDataSourceLiteralOrIdentifierOption
-                {
-                    CheckOptionDuplication(ref encounteredOptions, (int)vExternalDataSourceOption.OptionKind, vExternalDataSourceOption);
-                    AddAndUpdateTokenInfo(vResult, vResult.ExternalDataSourceOptions, vExternalDataSourceOption);
-                }
-            )
-        )*
+        externalDataSourceLocation[vResult]
 
         tRParen:RightParenthesis
         {
             UpdateTokenInfo(vResult,tRParen);
         }
     ;
 
-externalDataSourceType[CreateExternalDataSourceStatement vParent]
-    :
-        tDataSourceType:Identifier EqualsSign
-        {
-            Match(tDataSourceType, CodeGenerationSupporter.Type);
-            UpdateTokenInfo(vParent, tDataSourceType);
-        }
-        (
-            tExternalDataSourceType:Identifier
-            {
-                if (TryMatch(tExternalDataSourceType, CodeGenerationSupporter.Hadoop))
-                {
-                    UpdateTokenInfo(vParent, tExternalDataSourceType);
-                    vParent.DataSourceType = ExternalDataSourceType.HADOOP;
-                }
-                else if (TryMatch(tExternalDataSourceType, CodeGenerationSupporter.Rdbms))
-                {
-                    UpdateTokenInfo(vParent, tExternalDataSourceType);
-                    vParent.DataSourceType = ExternalDataSourceType.RDBMS;
-                }
-                else if (TryMatch(tExternalDataSourceType, CodeGenerationSupporter.ShardMapManager))
-                {
-                    UpdateTokenInfo(vParent, tExternalDataSourceType);
-                    vParent.DataSourceType = ExternalDataSourceType.SHARD_MAP_MANAGER;
-                }
-                else if (TryMatch(tExternalDataSourceType, CodeGenerationSupporter.BlobStorage))
-                {
-                    UpdateTokenInfo(vParent, tExternalDataSourceType);
-                    vParent.DataSourceType = ExternalDataSourceType.BLOB_STORAGE;
-                }
-				else
-				{
-					UpdateTokenInfo(vParent, tExternalDataSourceType);
-                    vParent.DataSourceType = ExternalDataSourceType.EXTERNAL_GENERICS;
-				}
-            }
-        )
-    ;
-
 externalDataSourceLocation[ExternalDataSourceStatement vParent]
 {
     Literal vLocation;
@@ -23868,68 +23793,9 @@ externalDataSourceLocation[ExternalDataSourceStatement vParent]
         }
     ;
 
-externalDataSourcePushdownOption[ExternalDataSourceStatement vParent]
-    :
-        tPushdownOption:Identifier
-        {
-            Match(tPushdownOption, CodeGenerationSupporter.PushdownOption);
-            UpdateTokenInfo(vParent, tPushdownOption);
-        }
-        EqualsSign
-        (
-           tOn:On
-                {
-                    vParent.PushdownOption = ExternalDataSourcePushdownOption.ON;
-                    UpdateTokenInfo(vParent, tOn);
-                }
-          | tOff:Off
-                {
-                    vParent.PushdownOption = ExternalDataSourcePushdownOption.OFF;
-                    UpdateTokenInfo(vParent, tOff);
-                }
-        )
-    ;
-
-externalDataSourceLiteralOrIdentifierOption returns [ExternalDataSourceLiteralOrIdentifierOption vResult = this.FragmentFactory.CreateFragment<ExternalDataSourceLiteralOrIdentifierOption>()]
-{
-    Literal vLiteral;
-    Identifier vIdentifier;
-}
-    :
-        tOption:Identifier
-        {
-            vResult.OptionKind = ExternalDataSourceOptionHelper.Instance.ParseOption(tOption);
-        }
-        EqualsSign
-        (
-            vIdentifier = identifier
-            {
-                if (vResult.OptionKind != ExternalDataSourceOptionKind.Credential)
-                {
-                    throw GetUnexpectedTokenErrorException(tOption);
-                }
-                vResult.Value = IdentifierOrValueExpression(vIdentifier);
-            }
-        |
-            vLiteral = stringLiteral
-            {
-                if (vResult.OptionKind != ExternalDataSourceOptionKind.ResourceManagerLocation &&
-                    vResult.OptionKind != ExternalDataSourceOptionKind.DatabaseName &&
-                    vResult.OptionKind != ExternalDataSourceOptionKind.ShardMapName &&
-                    vResult.OptionKind != ExternalDataSourceOptionKind.ConnectionOptions)
-                {
-                    throw GetUnexpectedTokenErrorException(tOption);
-                }
-                vResult.Value = IdentifierOrValueExpression(vLiteral);
-            }
-        )
-    ;
-
 alterExternalDataSourceStatement returns [AlterExternalDataSourceStatement vResult = FragmentFactory.CreateFragment<AlterExternalDataSourceStatement>()]
 {
     Identifier vName;
-    ExternalDataSourceOption vExternalDataSourceOption;
-    long encounteredOptions = 0;
 }
     :   tData:Identifier tSource:Identifier vName = identifier
         {
@@ -23940,36 +23806,8 @@ alterExternalDataSourceStatement returns [AlterExternalDataSourceStatement vResu
         }
 
         tSet:Set
-        (
-            {NextTokenMatches(CodeGenerationSupporter.Location)}?
-            externalDataSourceLocation[vResult]
-        |
-            {NextTokenMatches(CodeGenerationSupporter.PushdownOption)}?
-            externalDataSourcePushdownOption[vResult]
-        |
-            vExternalDataSourceOption = externalDataSourceLiteralOrIdentifierOption
-            {
-                CheckOptionDuplication(ref encounteredOptions, (int)vExternalDataSourceOption.OptionKind, vExternalDataSourceOption);
-                AddAndUpdateTokenInfo(vResult, vResult.ExternalDataSourceOptions, vExternalDataSourceOption);
-            }
-        )
 
-        (
-            tComma:Comma
-            (
-                {NextTokenMatches(CodeGenerationSupporter.Location)}?
-                externalDataSourceLocation[vResult]
-            |
-                {NextTokenMatches(CodeGenerationSupporter.PushdownOption)}?
-                externalDataSourcePushdownOption[vResult]
-            |
-                vExternalDataSourceOption = externalDataSourceLiteralOrIdentifierOption
-                {
-                    CheckOptionDuplication(ref encounteredOptions, (int)vExternalDataSourceOption.OptionKind, vExternalDataSourceOption);
-                    AddAndUpdateTokenInfo(vResult, vResult.ExternalDataSourceOptions, vExternalDataSourceOption);
-                }
-            )
-        )*
+        externalDataSourceLocation[vResult]
     ;
 
 dropExternalDataSourceStatement returns [DropExternalDataSourceStatement vResult = FragmentFactory.CreateFragment<DropExternalDataSourceStatement>()]

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs

@@ -70,6 +70,8 @@ public override void ExplicitVisit(FunctionCall node)
                     GenerateSpace();
                 GenerateReturnType(node?.ReturnType);
                 GenerateSymbol(TSqlTokenType.RightParenthesis);
+                // Generate OVER clause for windowed json_objectagg
+                GenerateSpaceAndFragmentIfNotNull(node.OverClause);
             }
             else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArray)
             {