From b914182b78b40acbaa8cb10209cee8c91e57f0d2 Mon Sep 17 00:00:00 2001 From: shubham yadav <126192924+yadavshubham01@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:32:14 +0530 Subject: [PATCH] Update workflows to optimize CI processes (#7828) This Pull Request addresses the need to optimize our Continuous Integration (CI) workflows for Playwright tests and release processes. The changes implemented aim to reduce unnecessary resource usage by conditionally executing jobs based on relevant file changes and Implement https://github.com/tj-actions/changed-files step ## Changes logs - Updated `ci-test-docker-compose.yaml , ci-chrome-extension.yaml ` to check for changed files before running tests. - Updated `ci-front.yaml , ci-utils.yaml , ci-website.yaml , ci-server.yaml` to check for changed files before running tests. - Enhanced `playwright.yml` to skip unnecessary tests based on file changes. --- .github/workflows/ci-chrome-extension.yaml | 24 +++- .github/workflows/ci-front.yaml | 134 ++++++++++++++++-- .github/workflows/ci-server.yaml | 43 ++++-- .github/workflows/ci-test-docker-compose.yaml | 14 +- .github/workflows/ci-utils.yaml | 14 ++ .github/workflows/ci-website.yaml | 25 +++- .github/workflows/playwright.yml.bak | 16 +++ 7 files changed, 239 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci-chrome-extension.yaml b/.github/workflows/ci-chrome-extension.yaml index a78be9415..071104d31 100644 --- a/.github/workflows/ci-chrome-extension.yaml +++ b/.github/workflows/ci-chrome-extension.yaml @@ -3,13 +3,9 @@ on: push: branches: - main - paths: - - 'package.json' - - 'packages/twenty-chrome-extension/**' + pull_request: - paths: - - 'package.json' - - 'packages/twenty-chrome-extension/**' + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -26,7 +22,23 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v4 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + package.json + packages/twenty-chrome-extension/** + - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Chrome Extension / Run build + if: steps.changed-files.outputs.changed == 'true' run: npx nx build twenty-chrome-extension + + - name: Mark as Valid if No Changes + if: steps.changed-files.outputs.changed != 'true' + run: | + echo "No relevant changes detected. Marking as valid." diff --git a/.github/workflows/ci-front.yaml b/.github/workflows/ci-front.yaml index 506a6dce0..d0a8a4c36 100644 --- a/.github/workflows/ci-front.yaml +++ b/.github/workflows/ci-front.yaml @@ -3,15 +3,9 @@ on: push: branches: - main - paths: - - 'package.json' - - 'packages/twenty-front/**' - - 'packages/twenty-ui/**' + pull_request: - paths: - - 'package.json' - - 'packages/twenty-front/**' - - 'packages/twenty-ui/**' + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -29,20 +23,81 @@ jobs: access_token: ${{ github.token }} - name: Fetch local actions uses: actions/checkout@v4 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + package.json + packages/twenty-front/** + packages/twenty-ui/** + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Diagnostic disk space issue + if: steps.changed-files.outputs.any_changed == 'true' run: df -h - name: Front / Restore Storybook Task Cache + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:frontend tasks: storybook:build - name: Front / Write .env + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-front - name: Front / Build storybook + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:build twenty-front front-sb-test: + + runs-on: ci-8-cores + timeout-minutes: 60 + needs: front-sb-build + strategy: + matrix: + storybook_scope: [pages, modules] + env: + REACT_APP_SERVER_BASE_URL: http://localhost:3000 + NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 + steps: + - name: Fetch local actions + uses: actions/checkout@v4 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-front/** + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' + uses: ./.github/workflows/actions/yarn-install + - name: Install Playwright + if: steps.changed-files.outputs.any_changed == 'true' + run: cd packages/twenty-front && npx playwright install + - name: Front / Restore Storybook Task Cache + if: steps.changed-files.outputs.any_changed == 'true' + uses: ./.github/workflows/actions/task-cache + with: + tag: scope:frontend + tasks: storybook:build + - name: Front / Write .env + if: steps.changed-files.outputs.any_changed == 'true' + run: npx nx reset:env twenty-front + - name: Run storybook tests + if: steps.changed-files.outputs.any_changed == 'true' + run: npx nx storybook:serve-and-test:static twenty-front --configuration=${{ matrix.storybook_scope }} + front-sb-test-shipfox: runs-on: shipfox-8vcpu-ubuntu-2204 timeout-minutes: 60 needs: front-sb-build @@ -55,18 +110,35 @@ jobs: steps: - name: Fetch local actions uses: actions/checkout@v4 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-front/** + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Install Playwright + if: steps.changed-files.outputs.any_changed == 'true' run: cd packages/twenty-front && npx playwright install - name: Front / Restore Storybook Task Cache + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:frontend tasks: storybook:build - name: Front / Write .env + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-front - name: Run storybook tests + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:serve-and-test:static twenty-front --configuration=${{ matrix.storybook_scope }} front-sb-test-performance: runs-on: shipfox-8vcpu-ubuntu-2204 @@ -77,13 +149,28 @@ jobs: steps: - name: Fetch local actions uses: actions/checkout@v4 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-front/** + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Install Playwright + if: steps.changed-files.outputs.any_changed == 'true' run: cd packages/twenty-front && npx playwright install - name: Front / Write .env + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-front - name: Run storybook tests + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:serve-and-test:static:performance twenty-front front-chromatic-deployment: if: contains(github.event.pull_request.labels.*.name, 'run-chromatic') || github.event_name == 'push' @@ -97,19 +184,35 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-front/** + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Front / Restore Storybook Task Cache + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:frontend tasks: storybook:build - name: Front / Write .env + if: steps.changed-files.outputs.any_changed == 'true' run: | cd packages/twenty-front touch .env echo "REACT_APP_SERVER_BASE_URL: $REACT_APP_SERVER_BASE_URL" >> .env - name: Publish to Chromatic + if: steps.changed-files.outputs.any_changed == 'true' run: npx nx run twenty-front:chromatic:ci front-task: runs-on: ubuntu-latest @@ -127,19 +230,34 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-front/** + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed == 'false' + run: echo "No relevant changes. Skipping CI." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Front / Restore ${{ matrix.task }} task cache + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:frontend tasks: ${{ matrix.task }} - name: Reset .env + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:frontend tasks: reset:env - name: Run ${{ matrix.task }} task + if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:frontend diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index 074d63fdd..101e1df3b 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -3,15 +3,9 @@ on: push: branches: - main - paths: - - 'package.json' - - 'packages/twenty-server/**' - - 'packages/twenty-emails/**' + pull_request: - paths: - - 'package.json' - - 'packages/twenty-server/**' - - 'packages/twenty-emails/**' + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -38,22 +32,35 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'package.json, packages/twenty-server/**, packages/twenty-emails/**' + - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Server / Restore Task Cache + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:backend - name: Server / Run lint & typecheck + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend tasks: lint,typecheck - name: Server / Build + if: steps.changed-files.outputs.changed == 'true' run: npx nx build twenty-server - name: Server / Write .env + if: steps.changed-files.outputs.changed == 'true' run: npx nx reset:env twenty-server - name: Worker / Run + if: steps.changed-files.outputs.changed == 'true' run: npx nx run twenty-server:worker:ci server-test: @@ -66,13 +73,23 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'package.json, packages/twenty-server/**, packages/twenty-emails/**' + - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Server / Restore Task Cache + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:backend - name: Server / Run Tests + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend @@ -100,13 +117,23 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'package.json, packages/twenty-server/**, packages/twenty-emails/**' + - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Server / Restore Task Cache + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/task-cache with: tag: scope:backend - name: Server / Run Integration Tests + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend diff --git a/.github/workflows/ci-test-docker-compose.yaml b/.github/workflows/ci-test-docker-compose.yaml index 1496425c8..2ff08a9e1 100644 --- a/.github/workflows/ci-test-docker-compose.yaml +++ b/.github/workflows/ci-test-docker-compose.yaml @@ -1,8 +1,7 @@ name: 'Test Docker Compose' on: pull_request: - paths: - - 'packages/twenty-docker/**' + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -13,8 +12,19 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/twenty-docker/** + docker-compose.yml + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed != 'true' + run: echo "No relevant changes detected. Marking as valid." - name: Run compose + if: steps.changed-files.outputs.any_changed == 'true' run: | echo "Patching docker-compose.yml..." # change image to localbuild using yq diff --git a/.github/workflows/ci-utils.yaml b/.github/workflows/ci-utils.yaml index fccfca98d..6cbd99b28 100644 --- a/.github/workflows/ci-utils.yaml +++ b/.github/workflows/ci-utils.yaml @@ -23,9 +23,16 @@ jobs: if: github.event.action != 'closed' steps: - uses: actions/checkout@v4 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'packages/twenty-utils/**' - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Utils / Run Danger.js + if: steps.changed-files.outputs.changed == 'true' run: cd packages/twenty-utils && npx nx danger:ci env: DANGER_GITHUB_API_TOKEN: ${{ github.token }} @@ -35,9 +42,16 @@ jobs: if: github.event.action == 'closed' && github.event.pull_request.merged == true steps: - uses: actions/checkout@v4 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'packages/twenty-utils/**' - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Run congratulate-dangerfile.js + if: steps.changed-files.outputs.changed == 'true' run: cd packages/twenty-utils && npx nx danger:congratulate env: DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-website.yaml b/.github/workflows/ci-website.yaml index d79345f3b..ce39d66ca 100644 --- a/.github/workflows/ci-website.yaml +++ b/.github/workflows/ci-website.yaml @@ -3,13 +3,10 @@ on: push: branches: - main - paths: - - 'package.json' - - 'packages/twenty-website/**' + pull_request: - paths: - - 'package.json' - - 'packages/twenty-website/**' + + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -27,13 +24,27 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v4 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: 'package.json, packages/twenty-website/**' + - name: Install dependencies + if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install + - name: Website / Run migrations + if: steps.changed-files.outputs.changed == 'true' run: npx nx database:migrate twenty-website env: DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default - name: Website / Build Website + if: steps.changed-files.outputs.changed == 'true' run: npx nx build twenty-website env: - DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default \ No newline at end of file + DATABASE_PG_URL: postgres://twenty:twenty@localhost:5432/default + + - name: Mark as VALID + if: steps.changed-files.outputs.changed != 'true' # If no changes, mark as valid + run: echo "No relevant changes detected. CI is valid." \ No newline at end of file diff --git a/.github/workflows/playwright.yml.bak b/.github/workflows/playwright.yml.bak index cffb50287..fc45955bc 100644 --- a/.github/workflows/playwright.yml.bak +++ b/.github/workflows/playwright.yml.bak @@ -13,11 +13,27 @@ jobs: - uses: actions/setup-node@v4 with: node-version: lts/* + + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v11 + with: + files: | + packages/** # Adjust this to your relevant directories + playwright.config.ts # Include any relevant config files + + - name: Skip if no relevant changes + if: steps.changed-files.outputs.any_changed != 'true' + run: echo "No relevant changes detected. Marking as valid." + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' run: npm install -g yarn && yarn - name: Install Playwright Browsers + if: steps.changed-files.outputs.any_changed == 'true' run: yarn playwright install --with-deps - name: Run Playwright tests + if: steps.changed-files.outputs.any_changed == 'true' run: yarn test:e2e companies - uses: actions/upload-artifact@v4 if: always()