diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml
index c47e9799e..823509888 100644
--- a/.github/azure-steps.yml
+++ b/.github/azure-steps.yml
@@ -94,7 +94,8 @@ steps:
 
   - script: |
       python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
-      PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
+      # temporarily ignore W095
+      PYTHONWARNINGS="error,ignore:[W095]:UserWarning,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
     displayName: 'Test assemble CLI'
     condition: eq(variables['python_version'], '3.8')
 
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 844946845..245407189 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,10 +13,13 @@ trigger:
       - "*.md"
 pr:
   paths:
-    exclude:
-      - "website/*"
-      - "*.md"
     include:
+      - "*.cfg"
+      - "*.py"
+      - "*.toml"
+      - "*.yml"
+      - ".github/azure-steps.yml"
+      - "spacy/*"
       - "website/meta/universe.json"
 
 jobs: