From 837c49e4a1fc8f86979ace907b338097a9ba9079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nuno=20G=C3=B3is?= <github@nunogois.com>
Date: Mon, 4 Aug 2025 15:57:29 +0100
Subject: [PATCH] chore: AI flag cleanup should focus on relevant flag (#10462)

https://linear.app/unleash/issue/2-3749/prevent-ai-flag-cleanup-from-removing-the-wrong-flag

This slightly refines our prompt so we're strictly focusing on the
relevant flag, instead of mistakenly targeting other flags.

Also includes:
- Adding our prompt to our output so we can more easily debug it
- Only grab the last cleanup summary in case there are multiple (more
than one step from the agent)
- Add a warning to the summary in case the agent couldn't find
conditional logic related to the flag, only definitions / configurations

Did a few manual tests and it seemed to work correctly. Example:
https://github.com/Unleash/unleash/pull/10461
---
 .github/workflows/ai-flag-cleanup-pr.yml | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ai-flag-cleanup-pr.yml b/.github/workflows/ai-flag-cleanup-pr.yml
index 22eedaac28..31229d5c77 100644
--- a/.github/workflows/ai-flag-cleanup-pr.yml
+++ b/.github/workflows/ai-flag-cleanup-pr.yml
@@ -173,10 +173,17 @@ jobs:
       - name: Create prompt
         id: create_prompt
         run: |
+          FLAG="${{ steps.extract_flag.outputs.flag-name }}"
           ISSUE_BODY="${{ fromJson(steps.get_issue.outputs.result).body }}"
-          cat <<'EOF' > cleanup_prompt.txt
+
+          cat <<-EOF > cleanup_prompt.txt
+          **Flag to clean up:** \`$FLAG\`
           Based on the issue description below, refactor the codebase to permanently apply the desired outcome for this feature flag (e.g. enable, keep variant, or discard), by removing all conditional checks and dead branches, preserving only the correct code path.
 
+          **Safety constraint**: Only modify or remove code, tests, and config related to the specified feature flag ($FLAG), and do not touch any other flags or their associated code.
+
+          You may **find no conditional logic** or actual usage of \`$FLAG\`. You may only find its **definition** or **configuration**. In that case, clean up any definitions or configurations for that flag and add a short **warning** to your summary saying no logic was found here (as the actual logic for the feature flag may live in another repo).
+
           After making the changes, provide a **Markdown summary** of what was changed, written for a developer reviewing the PR. Keep it clear, focused, and readable. Use the exact following format (including start & end separator lines, headings, bullets, emojis):
 
           \`\`\`md
@@ -208,6 +215,8 @@ jobs:
           $ISSUE_BODY
           EOF
 
+          echo "Cleanup prompt created:"
+          cat cleanup_prompt.txt
           echo "prompt-file=cleanup_prompt.txt" >> $GITHUB_OUTPUT
 
       - name: Set up Python
@@ -247,11 +256,14 @@ jobs:
               exit 1
           }
 
-          SUMMARY=$(sed -n '/=== AI Flag Cleanup Summary Start ===/,/=== AI Flag Cleanup Summary End ===/{
-            /=== AI Flag Cleanup Summary Start ===/d
-            /=== AI Flag Cleanup Summary End ===/d
-            p
-          }' flag_cleanup.txt)
+          SUMMARY=$(
+            awk '
+              /^=== AI Flag Cleanup Summary Start ===/ { inside=1; buf=""; next }
+              /^=== AI Flag Cleanup Summary End ===/   { inside=0; last=buf; next }
+              inside { buf = buf $0 "\n" }
+              END { printf "%s", last }
+            ' flag_cleanup.txt
+          )
 
           echo "summary<<EOF" >> $GITHUB_OUTPUT
           echo "$SUMMARY"       >> $GITHUB_OUTPUT