Workflow to detect breaking changes (#12532)

New CI to detect breaking changes in the REST API or the GraphQL API

.github/workflows/breaking-changes.yaml

@@ -0,0 +1,777 @@
+name: GraphQL and OpenAPI Breaking Changes Detection
+
+on:
+  pull_request:
+    types: [opened, synchronize, edited]
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MAIN_SERVER_PORT: 3000
+  CURRENT_SERVER_PORT: 3002
+
+permissions:
+  contents: read
+  pull-requests: write
+  checks: write
+
+jobs:
+  changed-files-check:
+    uses: ./.github/workflows/changed-files.yaml
+    with:
+      files: |
+        package.json
+        packages/twenty-server/**
+        packages/twenty-emails/**
+        packages/twenty-shared/**
+
+  api-breaking-changes:
+    needs: changed-files-check
+    if: needs.changed-files-check.outputs.any_changed == 'true'
+    timeout-minutes: 45
+    runs-on: depot-ubuntu-24.04-8
+    env:
+      NX_REJECT_UNKNOWN_LOCAL_CACHE: 0
+    services:
+      postgres:
+        image: twentycrm/twenty-postgres-spilo
+        env:
+          PGUSER_SUPERUSER: postgres
+          PGPASSWORD_SUPERUSER: postgres
+          ALLOW_NOSSL: 'true'
+          SPILO_PROVIDER: 'local'
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+      clickhouse:
+        image: clickhouse/clickhouse-server:latest
+        env:
+          CLICKHOUSE_PASSWORD: clickhousePassword
+          CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty"
+        ports:
+          - 8123:8123
+          - 9000:9000
+        options: >-
+          --health-cmd "clickhouse-client --host=localhost --port=9000 --user=default --password=clickhousePassword --query='SELECT 1'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
+
+    steps:
+      - name: Checkout current branch
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        uses: ./.github/workflows/actions/yarn-install
+
+      - name: Build shared dependencies
+        run: |
+          npx nx build twenty-shared
+          npx nx build twenty-emails
+
+      - name: Build current branch server
+        run: npx nx build twenty-server
+
+      - name: Setup databases
+        run: |
+          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "current_branch";'
+          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "main_branch";'
+
+      - name: Run ClickHouse migrations
+        run: npx nx clickhouse:migrate twenty-server
+        env:
+          CLICKHOUSE_URL: http://default:clickhousePassword@localhost:8123/twenty
+          CLICKHOUSE_PASSWORD: clickhousePassword
+
+      - name: Setup current branch database
+        run: |
+          npx nx reset:env twenty-server
+          # Function to set or update environment variable
+          set_env_var() {
+            local var_name="$1"
+            local var_value="$2"
+            local env_file="packages/twenty-server/.env"
+            
+            if grep -q "^${var_name}=" "$env_file"; then
+              sed -i "s|^${var_name}=.*|${var_name}=${var_value}|" "$env_file"
+            else
+              echo "${var_name}=${var_value}" >> "$env_file"
+            fi
+          }
+          
+          set_env_var "PG_DATABASE_URL" "postgres://postgres:postgres@localhost:5432/current_branch"
+          set_env_var "NODE_PORT" "${{ env.CURRENT_SERVER_PORT }}"
+          set_env_var "REDIS_URL" "redis://localhost:6379"
+          set_env_var "CLICKHOUSE_URL" "http://default:clickhousePassword@localhost:8123/twenty"
+          set_env_var "CLICKHOUSE_PASSWORD" "clickhousePassword"
+          
+          npx nx run twenty-server:database:init:prod
+          npx nx run twenty-server:database:migrate:prod
+
+      - name: Seed current branch database with test data
+        run: |
+          npx nx command-no-deps twenty-server -- workspace:seed:dev
+
+      - name: Start current branch server in background
+        run: |
+          echo "=== Current branch .env file contents ==="
+          cat packages/twenty-server/.env
+          echo "=== Starting current branch server ==="
+          nohup npx nx run twenty-server:start:prod > /tmp/current-server.log 2>&1 &
+          echo $! > /tmp/current-server.pid
+          echo "Current server PID: $(cat /tmp/current-server.pid)"
+
+      - name: Wait for current branch server to be ready
+        run: |
+          echo "Waiting for current branch server to start..."
+          timeout=300
+          interval=5
+          elapsed=0
+          
+          while [ $elapsed -lt $timeout ]; do
+            if curl -s "http://localhost:${{ env.CURRENT_SERVER_PORT }}/graphql" > /dev/null 2>&1 && \
+               curl -s "http://localhost:${{ env.CURRENT_SERVER_PORT }}/rest/open-api/core" > /dev/null 2>&1; then
+              echo "Current branch server is ready!"
+              break
+            fi
+            
+            echo "Current branch server not ready yet, waiting ${interval}s..."
+            sleep $interval
+            elapsed=$((elapsed + interval))
+          done
+          
+          if [ $elapsed -ge $timeout ]; then
+            echo "Timeout waiting for current branch server to start"
+            echo "Current server log:"
+            cat /tmp/current-server.log || echo "No current server log found"
+            exit 1
+          fi
+
+      - name: Download GraphQL and REST responses from current branch
+        run: |
+          # Admin token from jest-integration.config.ts
+          ADMIN_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyMDIwMjAyMC05ZTNiLTQ2ZDQtYTU1Ni04OGI5ZGRjMmIwMzQiLCJ3b3Jrc3BhY2VJZCI6IjIwMjAyMDIwLTFjMjUtNGQwMi1iZjI1LTZhZWNjZjdlYTQxOSIsIndvcmtzcGFjZU1lbWJlcklkIjoiMjAyMDIwMjAtMDY4Ny00YzQxLWI3MDctZWQxYmZjYTk3MmE3IiwidXNlcldvcmtzcGFjZUlkIjoiMjAyMDIwMjAtOWUzYi00NmQ0LWE1NTYtODhiOWRkYzJiMDM1IiwiaWF0IjoxNzM5NTQ3NjYxLCJleHAiOjMzMjk3MTQ3NjYxfQ.fbOM9yhr3jWDicPZ1n771usUURiPGmNdeFApsgrbxOw"
+          
+          # Load introspection query from file
+          INTROSPECTION_QUERY=$(cat packages/twenty-utils/graphql-introspection-query.graphql)
+          
+          # Prepare the query payload
+          QUERY_PAYLOAD=$(echo "$INTROSPECTION_QUERY" | tr '\n' ' ' | sed 's/"/\\"/g')
+          
+          echo "Downloading GraphQL schema from current server..."
+          curl -X POST "http://localhost:${{ env.CURRENT_SERVER_PORT }}/graphql" \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -d "{\"query\":\"${QUERY_PAYLOAD}\"}" \
+            -o current-schema-introspection.json \
+            -w "HTTP Status: %{http_code}\n" \
+            -s
+          
+          echo "Downloading GraphQL metadata schema from current server..."
+          curl -X POST "http://localhost:${{ env.CURRENT_SERVER_PORT }}/metadata" \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -d "{\"query\":\"${QUERY_PAYLOAD}\"}" \
+            -o current-metadata-schema-introspection.json \
+            -w "HTTP Status: %{http_code}\n" \
+            -s
+          
+          # Download current branch OpenAPI specs
+          echo "Downloading OpenAPI specifications from current server..."
+          curl -s "http://localhost:${{ env.CURRENT_SERVER_PORT }}/rest/open-api/core" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -o current-rest-api.json \
+            -w "HTTP Status: %{http_code}\n"
+          
+          curl -s "http://localhost:${{ env.CURRENT_SERVER_PORT }}/rest/open-api/metadata" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -o current-rest-metadata-api.json \
+            -w "HTTP Status: %{http_code}\n"
+          
+          # Verify the downloads
+          echo "Current branch files downloaded:"
+          ls -la current-*
+          
+
+      - name: Preserve current branch files
+        run: |
+          # Create a temp directory to store current branch files
+          mkdir -p /tmp/current-branch-files
+          
+          # Move current branch files to temp directory
+          mv current-* /tmp/current-branch-files/ 2>/dev/null || echo "No current-* files to preserve"
+          
+          echo "Preserved current branch files for later restoration"
+
+      - name: Stop current branch server
+        run: |
+          if [ -f /tmp/current-server.pid ]; then
+            echo "Stopping current branch server..."
+            kill $(cat /tmp/current-server.pid) || true
+            # Wait a bit for graceful shutdown
+            sleep 5
+            # Force kill if still running
+            kill -9 $(cat /tmp/current-server.pid) 2>/dev/null || true
+            rm -f /tmp/current-server.pid
+          fi
+
+      - name: Checkout main branch
+        run: |
+          git stash
+          git checkout origin/main
+          git clean -fd
+
+      - name: Install dependencies for main branch
+        uses: ./.github/workflows/actions/yarn-install
+
+      - name: Build main branch dependencies
+        run: |
+          npx nx build twenty-shared
+          npx nx build twenty-emails
+
+      - name: Build main branch server
+        run: npx nx build twenty-server
+
+      - name: Setup main branch database
+        run: |
+          # Function to set or update environment variable
+          set_env_var() {
+            local var_name="$1"
+            local var_value="$2"
+            local env_file="packages/twenty-server/.env"
+            
+            if grep -q "^${var_name}=" "$env_file"; then
+              sed -i "s|^${var_name}=.*|${var_name}=${var_value}|" "$env_file"
+            else
+              echo "${var_name}=${var_value}" >> "$env_file"
+            fi
+          }
+          
+          set_env_var "PG_DATABASE_URL" "postgres://postgres:postgres@localhost:5432/main_branch"
+          set_env_var "NODE_PORT" "${{ env.MAIN_SERVER_PORT }}"
+          
+          npx nx run twenty-server:database:init:prod
+          npx nx run twenty-server:database:migrate:prod
+
+      - name: Seed main branch database with test data
+        run: |
+          npx nx command-no-deps twenty-server -- workspace:seed:dev
+
+      - name: Start main branch server in background
+        run: |
+          echo "=== Main branch .env file contents ==="
+          cat packages/twenty-server/.env
+          echo "=== Starting main branch server ==="
+          nohup npx nx run twenty-server:start:prod > /tmp/main-server.log 2>&1 &
+          echo $! > /tmp/main-server.pid
+          echo "Main server PID: $(cat /tmp/main-server.pid)"
+
+      - name: Wait for main branch server to be ready
+        run: |
+          echo "Waiting for main branch server to start..."
+          timeout=300
+          interval=5
+          elapsed=0
+          
+          while [ $elapsed -lt $timeout ]; do
+            if curl -s "http://localhost:${{ env.MAIN_SERVER_PORT }}/graphql" > /dev/null 2>&1 && \
+               curl -s "http://localhost:${{ env.MAIN_SERVER_PORT }}/rest/open-api/core" > /dev/null 2>&1; then
+              echo "Main branch server is ready!"
+              break
+            fi
+            
+            echo "Main branch server not ready yet, waiting ${interval}s..."
+            sleep $interval
+            elapsed=$((elapsed + interval))
+          done
+          
+          if [ $elapsed -ge $timeout ]; then
+            echo "Timeout waiting for main branch server to start"
+            echo "Main server log:"
+            cat /tmp/main-server.log || echo "No main server log found"
+            exit 1
+          fi
+
+      - name: Download GraphQL and REST responses from main branch
+        run: |
+          # Admin token from jest-integration.config.ts
+          ADMIN_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyMDIwMjAyMC05ZTNiLTQ2ZDQtYTU1Ni04OGI5ZGRjMmIwMzQiLCJ3b3Jrc3BhY2VJZCI6IjIwMjAyMDIwLTFjMjUtNGQwMi1iZjI1LTZhZWNjZjdlYTQxOSIsIndvcmtzcGFjZU1lbWJlcklkIjoiMjAyMDIwMjAtMDY4Ny00YzQxLWI3MDctZWQxYmZjYTk3MmE3IiwidXNlcldvcmtzcGFjZUlkIjoiMjAyMDIwMjAtOWUzYi00NmQ0LWE1NTYtODhiOWRkYzJiMDM1IiwiaWF0IjoxNzM5NTQ3NjYxLCJleHAiOjMzMjk3MTQ3NjYxfQ.fbOM9yhr3jWDicPZ1n771usUURiPGmNdeFApsgrbxOw"
+          
+          # Load introspection query from file
+          INTROSPECTION_QUERY=$(cat packages/twenty-utils/graphql-introspection-query.graphql)
+          
+          # Prepare the query payload
+          QUERY_PAYLOAD=$(echo "$INTROSPECTION_QUERY" | tr '\n' ' ' | sed 's/"/\\"/g')
+          
+          echo "Downloading GraphQL schema from main server..."
+          curl -X POST "http://localhost:${{ env.MAIN_SERVER_PORT }}/graphql" \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -d "{\"query\":\"${QUERY_PAYLOAD}\"}" \
+            -o main-schema-introspection.json \
+            -w "HTTP Status: %{http_code}\n" \
+            -s
+          
+          echo "Downloading GraphQL metadata schema from main server..."
+          curl -X POST "http://localhost:${{ env.MAIN_SERVER_PORT }}/metadata" \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -d "{\"query\":\"${QUERY_PAYLOAD}\"}" \
+            -o main-metadata-schema-introspection.json \
+            -w "HTTP Status: %{http_code}\n" \
+            -s
+          
+          # Download main branch OpenAPI specs
+          echo "Downloading OpenAPI specifications from main server..."
+          curl -s "http://localhost:${{ env.MAIN_SERVER_PORT }}/rest/open-api/core" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -o main-rest-api.json \
+            -w "HTTP Status: %{http_code}\n"
+          
+          curl -s "http://localhost:${{ env.MAIN_SERVER_PORT }}/rest/open-api/metadata" \
+            -H "Authorization: Bearer ${ADMIN_TOKEN}" \
+            -o main-rest-metadata-api.json \
+            -w "HTTP Status: %{http_code}\n"
+          
+          # Verify the downloads
+          echo "Main branch files downloaded:"
+          ls -la main-*
+          
+
+      - name: Restore current branch files
+        run: |
+          # Move current branch files back to working directory
+          mv /tmp/current-branch-files/* . 2>/dev/null || echo "No files to restore"
+          
+          # Verify all files are present
+          echo "All API files restored:"
+          ls -la current-* main-* 2>/dev/null || echo "Some files may be missing"
+          
+          # Clean up temp directory
+          rm -rf /tmp/current-branch-files
+
+      - name: Install OpenAPI Diff Tool
+        run: |
+          # Using the Java-based OpenAPITools/openapi-diff via Docker
+          echo "Using OpenAPITools/openapi-diff via Docker"
+
+      - name: Generate GraphQL Schema Diff Reports
+        run: |
+          echo "=== INSTALLING GRAPHQL INSPECTOR CLI ==="
+          npm install -g @graphql-inspector/cli
+          
+          echo "=== GENERATING GRAPHQL DIFF REPORTS ==="
+          
+          # Check if GraphQL schema has changes
+          echo "Checking GraphQL schema for changes..."
+          if graphql-inspector diff main-schema-introspection.json current-schema-introspection.json >/dev/null 2>&1; then
+            echo "✅ No changes in GraphQL schema"
+            # Don't create a diff file for no changes
+          else
+            echo "⚠️ Changes detected in GraphQL schema, generating report..."
+            echo "# GraphQL Schema Changes" > graphql-schema-diff.md
+            echo "" >> graphql-schema-diff.md
+            graphql-inspector diff main-schema-introspection.json current-schema-introspection.json >> graphql-schema-diff.md 2>&1 || {
+              echo "⚠️ **Breaking changes or errors detected in GraphQL schema**" >> graphql-schema-diff.md
+              echo "" >> graphql-schema-diff.md
+              echo "\`\`\`" >> graphql-schema-diff.md
+              graphql-inspector diff main-schema-introspection.json current-schema-introspection.json 2>&1 >> graphql-schema-diff.md || echo "Error generating diff" >> graphql-schema-diff.md
+              echo "\`\`\`" >> graphql-schema-diff.md
+            }
+          fi
+          
+          # Check if GraphQL metadata schema has changes
+          echo "Checking GraphQL metadata schema for changes..."
+          if graphql-inspector diff main-metadata-schema-introspection.json current-metadata-schema-introspection.json >/dev/null 2>&1; then
+            echo "✅ No changes in GraphQL metadata schema"
+            # Don't create a diff file for no changes
+          else
+            echo "⚠️ Changes detected in GraphQL metadata schema, generating report..."
+            echo "# GraphQL Metadata Schema Changes" > graphql-metadata-diff.md
+            echo "" >> graphql-metadata-diff.md
+            graphql-inspector diff main-metadata-schema-introspection.json current-metadata-schema-introspection.json >> graphql-metadata-diff.md 2>&1 || {
+              echo "⚠️ **Breaking changes or errors detected in GraphQL metadata schema**" >> graphql-metadata-diff.md
+              echo "" >> graphql-metadata-diff.md
+              echo "\`\`\`" >> graphql-metadata-diff.md
+              graphql-inspector diff main-metadata-schema-introspection.json current-metadata-schema-introspection.json 2>&1 >> graphql-metadata-diff.md || echo "Error generating diff" >> graphql-metadata-diff.md
+              echo "\`\`\`" >> graphql-metadata-diff.md
+            }
+          fi
+          
+          # Show summary
+          echo "Generated diff files:"
+          ls -la *-diff.md 2>/dev/null || echo "No diff files generated (no changes detected)"
+
+      - name: Check REST API Breaking Changes
+        run: |
+          echo "=== CHECKING REST API FOR BREAKING CHANGES ==="
+          
+          # Use the Java-based openapi-diff via Docker
+          docker run --rm -v "$(pwd):/specs" openapitools/openapi-diff:latest \
+            --json /specs/rest-api-diff.json \
+            /specs/main-rest-api.json /specs/current-rest-api.json || echo "OpenAPI diff completed with exit code $?"
+          
+          # Check if the output file was created and is valid JSON
+          if [ -f "rest-api-diff.json" ] && jq empty rest-api-diff.json 2>/dev/null; then
+            # Check for breaking changes using Java openapi-diff JSON structure
+            incompatible=$(jq -r '.incompatible // false' rest-api-diff.json)
+            different=$(jq -r '.different // false' rest-api-diff.json)
+            
+            # Count changes
+            new_endpoints=$(jq -r '.newEndpoints | length' rest-api-diff.json 2>/dev/null || echo "0")
+            missing_endpoints=$(jq -r '.missingEndpoints | length' rest-api-diff.json 2>/dev/null || echo "0")
+            changed_operations=$(jq -r '.changedOperations | length' rest-api-diff.json 2>/dev/null || echo "0")
+            
+            if [ "$incompatible" = "true" ]; then
+              echo "❌ Breaking changes detected in REST API"
+              
+              # Generate breaking changes report
+              echo "# REST API Breaking Changes" > rest-api-diff.md
+              echo "" >> rest-api-diff.md
+              echo "⚠️ **Breaking changes detected that may affect existing API consumers**" >> rest-api-diff.md
+              echo "" >> rest-api-diff.md
+              
+              # Parse and format the changes from Java openapi-diff
+              jq -r '
+                if (.missingEndpoints | length) > 0 then
+                  "## 🚨 Removed Endpoints (" + (.missingEndpoints | length | tostring) + ")\n" +
+                  (.missingEndpoints | map("- **" + .method + " " + .pathUrl + "**: " + (.summary // "")) | join("\n"))
+                else "" end,
+                if (.changedOperations | length) > 0 then
+                  "\n## ⚠️ Changed Operations (" + (.changedOperations | length | tostring) + ")\n" +
+                  (.changedOperations | map("- **" + .method + " " + .pathUrl + "**: " + (.summary // "Modified operation")) | join("\n"))
+                else "" end,
+                if (.newEndpoints | length) > 0 then
+                  "\n## ✅ New Endpoints (" + (.newEndpoints | length | tostring) + ")\n" +
+                  (.newEndpoints | map("- " + .method + " " + .pathUrl + ": " + (.summary // "")) | join("\n"))
+                else "" end
+              ' rest-api-diff.json >> rest-api-diff.md
+              
+            elif [ "$different" = "true" ]; then
+              echo "📝 Non-breaking changes detected ($new_endpoints new endpoints, $missing_endpoints removed, $changed_operations changed)"
+              
+              # Generate non-breaking changes report
+              echo "# REST API Changes" > rest-api-diff.md
+              echo "" >> rest-api-diff.md
+              echo "## Summary" >> rest-api-diff.md
+              
+              jq -r '
+                if (.newEndpoints | length) > 0 then
+                  "### ✅ New Endpoints (" + (.newEndpoints | length | tostring) + ")\n" +
+                  (.newEndpoints | map("- " + .method + " " + .pathUrl + ": " + (.summary // "")) | join("\n"))
+                else "" end,
+                if (.changedOperations | length) > 0 then
+                  "\n### 🔄 Changed Operations (" + (.changedOperations | length | tostring) + ")\n" +
+                  (.changedOperations | map("- " + .method + " " + .pathUrl + ": " + (.summary // "Modified operation")) | join("\n"))
+                else "" end
+              ' rest-api-diff.json >> rest-api-diff.md
+            else
+              echo "✅ No changes detected in REST API"
+              # Don't create diff file for no changes
+            fi
+          else
+            echo "⚠️ OpenAPI diff tool could not process the files"
+            
+            echo "# REST API Analysis Error" > rest-api-diff.md
+            echo "" >> rest-api-diff.md
+            echo "⚠️ **Error occurred while analyzing REST API changes**" >> rest-api-diff.md
+            echo "" >> rest-api-diff.md
+            echo "## Error Output" >> rest-api-diff.md
+            echo "\`\`\`" >> rest-api-diff.md
+            docker run --rm -v "$(pwd):/specs" openapitools/openapi-diff:latest /specs/main-rest-api.json /specs/current-rest-api.json 2>&1 >> rest-api-diff.md || echo "Could not capture error output"
+            echo "\`\`\`" >> rest-api-diff.md
+            
+            # Don't fail the workflow for tool errors
+            echo "::warning::REST API analysis tool error - continuing workflow"
+          fi
+
+      - name: Check REST Metadata API Breaking Changes
+        run: |
+          echo "=== CHECKING REST METADATA API FOR BREAKING CHANGES ==="
+          
+          # Use the Java-based openapi-diff for metadata API as well
+          docker run --rm -v "$(pwd):/specs" openapitools/openapi-diff:latest \
+            --json /specs/rest-metadata-api-diff.json \
+            /specs/main-rest-metadata-api.json /specs/current-rest-metadata-api.json || echo "OpenAPI diff completed with exit code $?"
+          
+          # Check if the output file was created and is valid JSON
+          if [ -f "rest-metadata-api-diff.json" ] && jq empty rest-metadata-api-diff.json 2>/dev/null; then
+            # Check for breaking changes using Java openapi-diff JSON structure
+            incompatible=$(jq -r '.incompatible // false' rest-metadata-api-diff.json)
+            different=$(jq -r '.different // false' rest-metadata-api-diff.json)
+            
+            # Count changes
+            new_endpoints=$(jq -r '.newEndpoints | length' rest-metadata-api-diff.json 2>/dev/null || echo "0")
+            missing_endpoints=$(jq -r '.missingEndpoints | length' rest-metadata-api-diff.json 2>/dev/null || echo "0")
+            changed_operations=$(jq -r '.changedOperations | length' rest-metadata-api-diff.json 2>/dev/null || echo "0")
+            
+            if [ "$incompatible" = "true" ]; then
+              echo "❌ Breaking changes detected in REST Metadata API"
+              
+              # Generate breaking changes report
+              echo "# REST Metadata API Breaking Changes" > rest-metadata-api-diff.md
+              echo "" >> rest-metadata-api-diff.md
+              echo "⚠️ **Breaking changes detected that may affect existing API consumers**" >> rest-metadata-api-diff.md
+              echo "" >> rest-metadata-api-diff.md
+              
+              # Parse and format the changes from Java openapi-diff
+              jq -r '
+                if (.missingEndpoints | length) > 0 then
+                  "## 🚨 Removed Endpoints (" + (.missingEndpoints | length | tostring) + ")\n" +
+                  (.missingEndpoints | map("- **" + .method + " " + .pathUrl + "**: " + (.summary // "")) | join("\n"))
+                else "" end,
+                if (.changedOperations | length) > 0 then
+                  "\n## ⚠️ Changed Operations (" + (.changedOperations | length | tostring) + ")\n" +
+                  (.changedOperations | map("- **" + .method + " " + .pathUrl + "**: " + (.summary // "Modified operation")) | join("\n"))
+                else "" end,
+                if (.newEndpoints | length) > 0 then
+                  "\n## ✅ New Endpoints (" + (.newEndpoints | length | tostring) + ")\n" +
+                  (.newEndpoints | map("- " + .method + " " + .pathUrl + ": " + (.summary // "")) | join("\n"))
+                else "" end
+              ' rest-metadata-api-diff.json >> rest-metadata-api-diff.md
+
+            elif [ "$different" = "true" ]; then
+              echo "📝 Non-breaking changes detected ($new_endpoints new endpoints, $missing_endpoints removed, $changed_operations changed)"
+              
+              # Generate non-breaking changes report
+              echo "# REST Metadata API Changes" > rest-metadata-api-diff.md
+              echo "" >> rest-metadata-api-diff.md
+              echo "## Summary" >> rest-metadata-api-diff.md
+              
+              jq -r '
+                if (.newEndpoints | length) > 0 then
+                  "### ✅ New Endpoints (" + (.newEndpoints | length | tostring) + ")\n" +
+                  (.newEndpoints | map("- " + .method + " " + .pathUrl + ": " + (.summary // "")) | join("\n"))
+                else "" end,
+                if (.changedOperations | length) > 0 then
+                  "\n### 🔄 Changed Operations (" + (.changedOperations | length | tostring) + ")\n" +
+                  (.changedOperations | map("- " + .method + " " + .pathUrl + ": " + (.summary // "Modified operation")) | join("\n"))
+                else "" end
+              ' rest-metadata-api-diff.json >> rest-metadata-api-diff.md
+            else
+              echo "✅ No changes detected in REST Metadata API"
+              # Don't create diff file for no changes
+            fi
+          else
+            echo "⚠️ OpenAPI diff tool could not process the metadata API files"
+            
+            echo "# REST Metadata API Analysis Error" > rest-metadata-api-diff.md
+            echo "" >> rest-metadata-api-diff.md
+            echo "⚠️ **Error occurred while analyzing REST Metadata API changes**" >> rest-metadata-api-diff.md
+            echo "" >> rest-metadata-api-diff.md
+            echo "## Error Output" >> rest-metadata-api-diff.md
+            echo "\`\`\`" >> rest-metadata-api-diff.md
+            docker run --rm -v "$(pwd):/specs" openapitools/openapi-diff:latest /specs/main-rest-metadata-api.json /specs/current-rest-metadata-api.json 2>&1 >> rest-metadata-api-diff.md || echo "Could not capture error output"
+            echo "\`\`\`" >> rest-metadata-api-diff.md
+            
+            # Don't fail the workflow for tool errors
+            echo "::warning::REST Metadata API analysis tool error - continuing workflow"
+          fi
+
+      - name: Comment API Changes on PR
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let hasChanges = false;
+            let comment = '';
+            
+            try {
+              if (fs.existsSync('graphql-schema-diff.md')) {
+                const graphqlDiff = fs.readFileSync('graphql-schema-diff.md', 'utf8');
+                if (graphqlDiff.trim()) {
+                  if (!hasChanges) {
+                    comment = '## 📊 API Changes Report\n\n';
+                    hasChanges = true;
+                  }
+                  comment += '### GraphQL Schema Changes\n' + graphqlDiff + '\n\n';
+                }
+              }
+              
+              if (fs.existsSync('graphql-metadata-diff.md')) {
+                const graphqlMetadataDiff = fs.readFileSync('graphql-metadata-diff.md', 'utf8');
+                if (graphqlMetadataDiff.trim()) {
+                  if (!hasChanges) {
+                    comment = '## 📊 API Changes Report\n\n';
+                    hasChanges = true;
+                  }
+                  comment += '### GraphQL Metadata Schema Changes\n' + graphqlMetadataDiff + '\n\n';
+                }
+              }
+              
+              if (fs.existsSync('rest-api-diff.md')) {
+                const restDiff = fs.readFileSync('rest-api-diff.md', 'utf8');
+                if (restDiff.trim()) {
+                  if (!hasChanges) {
+                    comment = '## 📊 API Changes Report\n\n';
+                    hasChanges = true;
+                  }
+                  comment += restDiff + '\n\n';
+                }
+              }
+              
+              if (fs.existsSync('rest-metadata-api-diff.md')) {
+                const metadataDiff = fs.readFileSync('rest-metadata-api-diff.md', 'utf8');
+                if (metadataDiff.trim()) {
+                  if (!hasChanges) {
+                    comment = '## 📊 API Changes Report\n\n';
+                    hasChanges = true;
+                  }
+                  comment += metadataDiff + '\n\n';
+                }
+              }
+              
+              // Only post comment if there are changes
+              if (hasChanges) {
+                // Check if there are any breaking changes detected
+                let hasBreakingChanges = false;
+                let breakingChangeNote = '';
+                
+                // Check for breaking changes in any of the diff files
+                if (fs.existsSync('rest-api-diff.md')) {
+                  const restDiff = fs.readFileSync('rest-api-diff.md', 'utf8');
+                  if (restDiff.includes('Breaking Changes') || restDiff.includes('🚨') || 
+                      restDiff.includes('Removed Endpoints') || restDiff.includes('Changed Operations')) {
+                    hasBreakingChanges = true;
+                  }
+                }
+                
+                if (fs.existsSync('rest-metadata-api-diff.md')) {
+                  const metadataDiff = fs.readFileSync('rest-metadata-api-diff.md', 'utf8');
+                  if (metadataDiff.includes('Breaking Changes') || metadataDiff.includes('🚨') || 
+                      metadataDiff.includes('Removed Endpoints') || metadataDiff.includes('Changed Operations')) {
+                    hasBreakingChanges = true;
+                  }
+                }
+                
+                // Also check GraphQL changes for breaking changes indicators
+                if (fs.existsSync('graphql-schema-diff.md')) {
+                  const graphqlDiff = fs.readFileSync('graphql-schema-diff.md', 'utf8');
+                  if (graphqlDiff.includes('Breaking changes') || graphqlDiff.includes('BREAKING')) {
+                    hasBreakingChanges = true;
+                  }
+                }
+                
+                if (fs.existsSync('graphql-metadata-diff.md')) {
+                  const graphqlMetadataDiff = fs.readFileSync('graphql-metadata-diff.md', 'utf8');
+                  if (graphqlMetadataDiff.includes('Breaking changes') || graphqlMetadataDiff.includes('BREAKING')) {
+                    hasBreakingChanges = true;
+                  }
+                }
+                
+                // Check PR title for "breaking"
+                const prTitle = "${{ github.event.pull_request.title }}";
+                const titleContainsBreaking = prTitle.toLowerCase().includes('breaking');
+                
+                if (hasBreakingChanges) {
+                  if (titleContainsBreaking) {
+                    breakingChangeNote = '\n\n## ✅ Breaking Change Protocol\n\n' +
+                      '**This PR title contains "breaking" and breaking changes were detected - the CI will fail as expected.**\n\n' +
+                      '📝 **Action Required**: Please add `BREAKING CHANGE:` to your commit message to trigger a major version bump.\n\n' +
+                      'Example:\n```\nfeat: add new API endpoint\n\nBREAKING CHANGE: removed deprecated field from User schema\n```';
+                  } else {
+                    breakingChangeNote = '\n\n## ⚠️ Breaking Change Protocol\n\n' +
+                      '**Breaking changes detected but PR title does not contain "breaking" - CI will pass but action needed.**\n\n' +
+                      '🔄 **Options**:\n' +
+                      '1. **If this IS a breaking change**: Add "breaking" to your PR title and add `BREAKING CHANGE:` to your commit message\n' +
+                      '2. **If this is NOT a breaking change**: The API diff tool may have false positives - please review carefully\n\n' +
+                      'For breaking changes, add to commit message:\n```\nfeat: add new API endpoint\n\nBREAKING CHANGE: removed deprecated field from User schema\n```';
+                  }
+                }
+                
+                const COMMENT_MARKER = '<!-- API_CHANGES_REPORT -->';
+                const commentBody = COMMENT_MARKER + '\n' + comment + '⚠️ **Please review these API changes carefully before merging.**' + breakingChangeNote;
+                
+                // Get all comments to find existing API changes comment
+                const {data: comments} = await github.rest.issues.listComments({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                });
+                
+                // Find our existing comment
+                const botComment = comments.find(comment => comment.body.includes(COMMENT_MARKER));
+                
+                if (botComment) {
+                  // Update existing comment
+                  await github.rest.issues.updateComment({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    comment_id: botComment.id,
+                    body: commentBody
+                  });
+                  console.log('Updated existing API changes comment');
+                } else {
+                  // Create new comment
+                  await github.rest.issues.createComment({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    body: commentBody
+                  });
+                  console.log('Created new API changes comment');
+                }
+              } else {
+                console.log('No API changes detected - skipping PR comment');
+                
+                // Check if there's an existing comment to remove
+                const {data: comments} = await github.rest.issues.listComments({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                });
+                
+                const COMMENT_MARKER = '<!-- API_CHANGES_REPORT -->';
+                const botComment = comments.find(comment => comment.body.includes(COMMENT_MARKER));
+                
+                if (botComment) {
+                  await github.rest.issues.deleteComment({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    comment_id: botComment.id,
+                  });
+                  console.log('Deleted existing API changes comment (no changes detected)');
+                }
+              }
+            } catch (error) {
+              console.log('Could not post comment:', error);
+            }
+
+      - name: Cleanup servers
+        if: always()
+        run: |
+          if [ -f /tmp/current-server.pid ]; then
+            kill $(cat /tmp/current-server.pid) || true
+          fi
+          if [ -f /tmp/main-server.pid ]; then
+            kill $(cat /tmp/main-server.pid) || true
+          fi
+
+      - name: Upload API specifications and diffs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-specifications-and-diffs
+          path: |
+            /tmp/main-server.log
+            /tmp/current-server.log
+            *-api.json
+            *-schema-introspection.json  
+            *-diff.md
+            *-diff.json
+