ci: add Internal Python tests to InternalCompat job

Expand the InternalCompat job to also run Internal Python tests
against the OSS build. This catches Python SDK compatibility
breaks (API changes, return type changes) that the Scala-only
checks would miss.

New steps added:
- sbt packagePython + publishM2 for Internal
- Run AIFuncPandas and AIFuncPySpark Python test suites
- Separate test result publishing for Scala and Python

Skipping multimodal suites and ExcludeAIFunc for now to keep
CI time reasonable — can expand incrementally.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: smamindl

pipeline.yaml

@@ -903,8 +903,9 @@ jobs:
       inputs:
         azureSubscription: 'SynapseML Build'
         keyVaultName: mmlspark-keys
+    - template: templates/fabric_kv.yml
     - task: AzureCLI@2
-      displayName: 'Run Internal Scala tests (non-Fabric)'
+      displayName: 'Run Internal Scala tests'
       timeoutInMinutes: 60
       inputs:
         azureSubscription: 'SynapseML Build'
@@ -916,12 +917,9 @@ jobs:
           source activate synapseml-internal
           export CREATE_SEMPY_WRITER=false
           export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
-          echo "Running non-Fabric Internal tests against OSS $(OSS_VERSION)..."
-          # Only spark.aifunc is runnable without Fabric creds.
-          # powerbi/ebm/predict extend HasSparkSession which eagerly inits
-          # FabricTestConstants (requires INTEGRATION_ACCOUNT from fabrictest-cert-admin-kv).
+          echo "Running Internal tests against OSS $(OSS_VERSION)..."
           FAILURES=0
-          for pkg in spark.aifunc; do
+          for pkg in spark.aifunc powerbi ebm predict nbtest; do
             echo "=== Testing $pkg ==="
             if ! sbt "testOnly com.microsoft.azure.synapse.ml.$pkg.**"; then
               echo "##vso[task.logissue type=warning]$pkg tests failed"
@@ -932,9 +930,82 @@ jobs:
             echo "##vso[task.logissue type=warning]$FAILURES test package(s) failed"
             exit 1
           fi
+      env:
+        INTEGRATION_ENV: $(sempy-integration-region)
+        INTEGRATION_ACCOUNT: $(sempy-integration-account)
+        INTEGRATION_CERTIFICATE: $(sempy-integration-certificate)
+    - task: AzureCLI@2
+      displayName: 'Package Internal Python against OSS'
+      timeoutInMinutes: 15
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML-Internal
+          source activate synapseml-internal
+          export CREATE_SEMPY_WRITER=false
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          echo "Packaging Internal Python against OSS $(OSS_VERSION)..."
+          sbt packagePython
+          sbt publishM2
+      condition: succeededOrFailed()
+    - task: AzureCLI@2
+      displayName: 'Run Internal Python tests (AIFunc)'
+      timeoutInMinutes: 30
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML-Internal
+          source activate synapseml-internal
+          export CREATE_SEMPY_WRITER=false
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          echo "Running Internal Python tests against OSS $(OSS_VERSION)..."
+          FAILURES=0
+          for suite in AIFuncPandas AIFuncPandasMultimodal AIFuncPySpark AIFuncPySparkMultimodal; do
+            echo "=== Testing Python $suite ==="
+            if ! sbt "testPython${suite}"; then
+              echo "##vso[task.logissue type=warning]Python $suite tests failed"
+              FAILURES=$((FAILURES + 1))
+            fi
+          done
+          if [ $FAILURES -gt 0 ]; then
+            echo "##vso[task.logissue type=warning]$FAILURES Python test suite(s) failed"
+            exit 1
+          fi
+      condition: succeededOrFailed()
+    - task: AzureCLI@2
+      displayName: 'Run Internal Python tests (ExcludeAIFunc)'
+      timeoutInMinutes: 30
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML-Internal
+          source activate synapseml-internal
+          export CREATE_SEMPY_WRITER=false
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          echo "Creating MLflow model fixtures..."
+          python ./src/test/python/make_mlflow_models.py
+          echo "Running ExcludeAIFunc Python tests against OSS $(OSS_VERSION)..."
+          sbt testPythonExcludeAIFunc
+      condition: succeededOrFailed()
     - task: PublishTestResults@2
-      displayName: 'Publish Internal Test Results'
+      displayName: 'Publish Internal Scala Test Results'
       inputs:
         testResultsFiles: '**/test-reports/TEST-*.xml'
         failTaskOnFailedTests: false
       condition: succeededOrFailed()
+    - task: PublishTestResults@2
+      displayName: 'Publish Internal Python Test Results'
+      inputs:
+        testResultsFiles: '**/python-test-*.xml'
+        searchFolder: '$(Build.SourcesDirectory)/SynapseML-Internal'
+        failTaskOnFailedTests: false
+      condition: succeededOrFailed()