diff --git a/.github/workflows/readme-linter.yml b/.github/workflows/readme-linter.yml
new file mode 100644
index 000000000..c0b5dc49c
--- /dev/null
+++ b/.github/workflows/readme-linter.yml
@@ -0,0 +1,23 @@
+name: Lint plugin readmes
+on:
+#  push:
+#    branches-ignore: master
+  pull_request:
+    branches: # Names of target branches, not source branches
+      - master
+jobs:
+  run-readme-linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v18.7
+        with:
+          base_sha: ${{ github.event.pull_request.base.sha }}
+          files: plugins/**/README.md
+      - name: Run readme linter on changed files
+        run: go run ./tools/readme_linter ${{ steps.changed-files.outputs.all_changed_files }}
diff --git a/tools/readme_linter/assert.go b/tools/readme_linter/assert.go
index a0f54a0e4..16233c508 100644
--- a/tools/readme_linter/assert.go
+++ b/tools/readme_linter/assert.go
@@ -140,3 +140,7 @@ func (t *T) assertHeadingLevel(expected int, n ast.Node) {
 
 	t.printFailedAssertf(n, "expected header level %d, have %d", expected, h.Level)
 }
+
+func (t *T) pass() bool {
+	return t.fails == 0
+}
diff --git a/tools/readme_linter/main.go b/tools/readme_linter/main.go
index f561100f5..1ef8c52f1 100644
--- a/tools/readme_linter/main.go
+++ b/tools/readme_linter/main.go
@@ -20,11 +20,17 @@ func main() {
 	flag.Parse()
 
 	var err error
+	pass := true
 	for _, filename := range flag.Args() {
-		err = checkFile(filename, guessPluginType(filename), *sourceFlag)
+		var filePass bool
+		filePass, err = checkFile(filename, guessPluginType(filename), *sourceFlag)
 		if err != nil {
 			panic(err)
 		}
+		pass = pass && filePass
+	}
+	if !pass {
+		os.Exit(1)
 	}
 }
 
@@ -57,10 +63,10 @@ func init() {
 	rules[pluginInput] = append(rules[pluginInput], inputRules...)
 }
 
-func checkFile(filename string, pluginType plugin, sourceFlag bool) error {
+func checkFile(filename string, pluginType plugin, sourceFlag bool) (bool, error) {
 	md, err := os.ReadFile(filename)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	// Goldmark returns locations as offsets. We want line
@@ -107,10 +113,10 @@ func checkFile(filename string, pluginType plugin, sourceFlag bool) error {
 	for _, rule := range rules {
 		err = rule(&tester, root)
 		if err != nil {
-			return err
+			return false, err
 		}
 	}
 	tester.printPassFail()
 
-	return nil
+	return tester.pass(), nil
 }