AISearch Knowledge BAse retrieval call cannot reach vectorization endpoint

We have the following setup. We have an openai instance in sweden central (no private endpoint, so a public IP) and we have an AI search instance in west europe.

We also have an ADLSg2 instance in west europe (private endpoint). Both the storage account and the openai instance can be authenticated to using a user managed identity. We have absolutely unequivocally confirmed that all necessary role assignments are there, for both storage and openai.

In the AI search instance we have added some knowledge sources and a knowledge base. The indexers created in this process can successfully reach the storage account, and also index the documents. We have set both an vectorizer (embedding) model and an chat completion model on the knowledge source. This works, both text documents and documents with images successfully get indexed. This also means Azure OpenAI is reachable from the AI search instance, and the role assignments are correct.

The problem occurs when we try to do a retrieval using a knowledge base. This knowledge base uses the same openai instance for its embedding model, and the same user assigned identity. Yet, when we make the retrieval call we get the following response:

Status: 502 (Bad Gateway)
Content:
{"error":{"code":"","message":"Could not complete vectorization action. The service failed to authenticate to the vectorization endpoint."}}
Headers:
Cache-Control: no-cache,no-store
Pragma: no-cache
client-request-id: xxx
x-ms-client-request-id: xxx
request-id: 7f702b5e-59cb-11f1-9f38-9940d3f436b4
elapsed-time: 22
Strict-Transport-Security: REDACTED
Date: Wed, 27 May 2026 12:56:40 GMT
Content-Length: 140
Content-Type: application/json; charset=utf-8
Content-Language: REDACTED
Expires: -1
Knowledge source '<some_ks>': Could not complete vectorization action. The service failed to authenticate to the vectorization endpoint.
Status: 502 (Bad Gateway)

Let me know if you need any more info. This seems impossible to debug.

Anyone know what's going on here? Here's some extra info that might be handy:

Knowledge source JSON:

{
  "@odata.context": "https://<SRCH>.search.windows.net/$metadata#knowledgesources/$entity",
  "@odata.etag": "\"0x8DEBC8B36E9FB00\"",
  "name": "ks",
  "kind": "azureBlob",
  "description": "<DESCR>",
  "encryptionKey": null,
  "searchIndexParameters": null,
  "remoteSearchIndexParameters": null,
  "azureBlobParameters": {
    "connectionString": "<redacted>",
    "containerName": "<CONT>",
    "folderPath": "<SOME_DIR>",
    "isADLSGen2": true,
    "ingestionParameters": {
      "disableImageVerbalization": false,
      "ingestionPermissionOptions": [],
      "contentExtractionMode": "minimal",
      "identity": {
        "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
        "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>",
        "federatedIdentityClientId": null
      },
      "embeddingModel": {
        "kind": "azureOpenAI",
        "azureOpenAIParameters": {
          "resourceUri": "https://<OPENAI>.openai.azure.com",
          "deploymentId": "text-embedding-3-large",
          "apiKey": null,
          "modelName": "text-embedding-3-large",
          "authIdentity": {
            "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
            "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>",
            "federatedIdentityClientId": null
          }
        }
      },
      "chatCompletionModel": {
        "kind": "azureOpenAI",
        "azureOpenAIParameters": {
          "resourceUri": "https://<OPENAI>.openai.azure.com",
          "deploymentId": "gpt-5-mini",
          "apiKey": null,
          "modelName": "gpt-5-mini",
          "authIdentity": {
            "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
            "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>",
            "federatedIdentityClientId": null
          }
        }
      },
      "ingestionSchedule": {
        "interval": "PT30M",
        "startTime": "2026-01-01T00:00:00Z"
      },
      "assetStore": null,
      "aiServices": null,
      "freshnessPolicy": null
    },
    "createdResources": {
      "datasource": "ks-datasource",
      "indexer": "ks-indexer",
      "skillset": "ks-skillset",
      "index": "ks-index"
    }
  },
  "mcpToolParameters": null,
  "mcpServerParameters": null,
  "fabricOntologyParameters": null,
  "fabricDataAgentParameters": null,
  "webParameters": null,
  "remoteSharePointParameters": null,
  "indexedSharePointParameters": null,
  "indexedOneLakeParameters": null,
  "fileParameters": null,
  "indexedSqlParameters": null,
  "workIQParameters": null
}

Knowledge base:

{
  "@odata.context": "https://<SRCH>.search.windows.net/$metadata#knowledgebases/$entity",
  "@odata.etag": "\"0x8DEBC8B36D16DD1\"",
  "name": "test",
  "description": "",
  "retrievalInstructions": "Just do it",
  "answerInstructions": null,
  "outputMode": "answerSynthesis",
  "knowledgeSources": [
    {       "name": "ks",
      "enableImageServing": null,
      "enableFreshness": null
    }
  ],
  "models": [
    {
      "kind": "azureOpenAI",
      "azureOpenAIParameters": {
        "resourceUri": "https://<OPENAI>.openai.azure.com",         
"deploymentId": "gpt-5-mini",
        "apiKey": null,
        "modelName": "gpt-5-mini",
        "authIdentity": {
          "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
          "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>",           "federatedIdentityClientId": null
        }
      }
    }
  ],
  "encryptionKey": null,
  "retrievalReasoningEffort": {
    "kind": "low"
  },
  "corsOptions": null
}

Skillset:

{
  "@odata.etag": "\"0x8DEBC8B369DE2BE\"",
  "name": "strategy-ks-skillset",
  "description": "Skillset for knowledge source 'strategy-ks'",
  "skills": [
    {
      "@odata.type": "#Microsoft.Skills.Text.SplitSkill",
      "name": "SplitSkill",
      "description": "Split document content into chunks",
      "context": "/document",
      "defaultLanguageCode": "en",
      "textSplitMode": "pages",
      "maximumPageLength": 2000,
      "pageOverlapLength": 200,
      "maximumPagesToTake": 0,
      "unit": "characters",
      "inputs": [
        {
          "name": "text",
          "source": "/document/content",
          "inputs": []
        }
      ],
      "outputs": [
        {
          "name": "textItems",
          "targetName": "pages"
        }
      ]
    },
    {
      "@odata.type": "#Microsoft.Skills.Text.AzureOpenAIEmbeddingSkill",
      "name": "AzureOpenAIEmbeddingSkill",
      "description": "Generate embeddings",
      "context": "/document/pages/*",
      "resourceUri": "https://<OPENAI>.openai.azure.com",
      "deploymentId": "text-embedding-3-large",
      "dimensions": 3072,
      "modelName": "text-embedding-3-large",
      "inputs": [
        {
          "name": "text",
          "source": "/document/pages/*",
          "inputs": []
        }
      ],
      "outputs": [
        {
          "name": "embedding",
          "targetName": "text_vector"
        }
      ],
      "authIdentity": {
        "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
        "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>"
      }
    },
    {
      "@odata.type": "#Microsoft.Skills.Custom.ChatCompletionSkill",
      "name": "GenAISkill",
      "description": "Generate chat responses for image verbalization",
      "context": "/document/normalized_images/*",
      "uri": "https://<OPENAI>.openai.azure.com/openai/deployments/gpt-5-mini/chat/completions?api-version=2024-10-21",
      "inputs": [
        {
          "name": "systemMessage",
          "source": "='You are tasked with generating concise, accurate descriptions of images, figures, diagrams, or charts in documents.'",
          "inputs": []
        },
        {
          "name": "userMessage",
          "source": "='Please describe this image.'",
          "inputs": []
        },
        {
          "name": "image",
          "source": "/document/normalized_images/*/data",
          "inputs": []
        }
      ],
      "outputs": [
        {
          "name": "response",
          "targetName": "verbalizedImage"
        }
      ],
      "authIdentity": {
        "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
        "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>"
      }
    },
    {
      "@odata.type": "#Microsoft.Skills.Text.AzureOpenAIEmbeddingSkill",
      "name": "VerbalizedImageAzureOpenAIEmbeddingSkill",
      "description": "Generate embeddings",
      "context": "/document/normalized_images/*",
      "resourceUri": "https://<OPENAI>.openai.azure.com",
      "deploymentId": "text-embedding-3-large",
      "dimensions": 3072,
      "modelName": "text-embedding-3-large",
      "inputs": [
        {
          "name": "text",
          "source": "/document/normalized_images/*/verbalizedImage",
          "inputs": []
        }
      ],
      "outputs": [
        {
          "name": "embedding",
          "targetName": "verbalizedImage_vector"
        }
      ],
      "authIdentity": {
        "@odata.type": "#Microsoft.Azure.Search.DataUserAssignedIdentity",
        "userAssignedIdentity": "/subscriptions/<SUB>/resourcegroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<ID>"
      }
    }
  ],
  "indexProjections": {
    "selectors": [
      {
        "targetIndexName": "strategy-ks-index",
        "parentKeyFieldName": "snippet_parent_id",
        "sourceContext": "/document/pages/*",
        "mappings": [
          {
            "name": "snippet_vector",
            "source": "/document/pages/*/text_vector",
            "inputs": []
          },
          {
            "name": "snippet",
            "source": "/document/pages/*",
            "inputs": []
          },
          {
            "name": "blob_url",
            "source": "/document/blob_url",
            "inputs": []
          }
        ]
      },
      {
        "targetIndexName": "strategy-ks-index",
        "parentKeyFieldName": "image_snippet_parent_id",
        "sourceContext": "/document/normalized_images/*",
        "mappings": [
          {
            "name": "snippet_vector",
            "source": "/document/normalized_images/*/verbalizedImage_vector",
            "inputs": []
          },
          {
            "name": "snippet",
            "source": "/document/normalized_images/*/verbalizedImage",
            "inputs": []
          },
          {
            "name": "blob_url",
            "source": "/document/blob_url",
            "inputs": []
          }
        ]
      }
    ],
    "parameters": {
      "projectionMode": "skipIndexingParentDocuments"
    }
  }
}

It's important to note that all openai models have model names and deployment names that match the ones we use here.

<ID> has the following relevant role assignment on the <OPENAI> instance:

Cognitive Services Contributor, Cognitive Services OpenAI User, Cognitive Services User