diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml
index 3c51e8100..b2ccf3d81 100644
--- a/.github/azure-steps.yml
+++ b/.github/azure-steps.yml
@@ -60,8 +60,8 @@ steps:
     condition: eq(variables['python_version'], '3.8')
 
   - script: |
-      python -W error -m spacy info ca_core_news_sm --url
-    displayName: 'Test info --url CLI'
+      python -W error -m spacy info ca_core_news_sm | grep -q download_url
+    displayName: 'Test download_url in info CLI'
     condition: eq(variables['python_version'], '3.8')
 
   - script: |
diff --git a/spacy/cli/info.py b/spacy/cli/info.py
index 1dc06d166..290608b99 100644
--- a/spacy/cli/info.py
+++ b/spacy/cli/info.py
@@ -138,7 +138,7 @@ def info_installed_model_url(model: str) -> Optional[str]:
     """
     try:
         dist = importlib_metadata.distribution(model)
-        data = dist.read_text("direct_url.json")
+        data = json.loads(dist.read_text("direct_url.json"))
         return data["url"]
     except importlib_metadata.PackageNotFoundError:
         # no such package