chore: notify #dev of nightly gauntlet failures (#16105)

Expands on https://github.com/coder/coder/pull/16102

This workflow is currently failing every night, so this will not only
raise immediate awareness but will also be easy to validate this job.

Signed-off-by: Danny Kopping <danny@coder.com>

.github/workflows/ci.yaml

@@ -1263,7 +1263,7 @@ jobs:
                 "type": "header",
                 "text": {
                   "type": "plain_text",
-                  "text": "❌ CI Failure in `main`",
+                  "text": "❌ CI Failure in main",
                   "emoji": true
                 }
               },

.github/workflows/nightly-gauntlet.yaml

@@ -72,3 +72,52 @@ jobs:
         if: always()
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  notify-slack-on-failure:
+    runs-on: ubuntu-latest
+    if: always() && failure()
+
+    steps:
+      - name: Send Slack notification
+        run: |
+          curl -X POST -H 'Content-type: application/json' \
+          --data '{
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": "❌ Nightly gauntlet failed",
+                  "emoji": true
+                }
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Workflow:*\n${{ github.workflow }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Failed Job:*\n${{ github.job }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Committer:*\n${{ github.actor }}"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Commit:*\n${{ github.sha }}"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
+                }
+              }
+            ]
+          }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}