From f8ddc02b8e2949ffa4d027fe01109553ef5d7d50 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:37:28 +0100 Subject: [PATCH] [CI] Refactor changed files integration (#9643) # BEFORE [run](https://github.com/twentyhq/twenty/actions/runs/12806801953) ![image](https://github.com/user-attachments/assets/f0a8ffe3-3fc0-42ca-b2ee-8a980606b5dd) # AFTER [run](https://github.com/twentyhq/twenty/actions/runs/12807034402) ![image](https://github.com/user-attachments/assets/5117c680-6804-416b-a8c8-bf00614ca453) ## Motivations: - less workflow to whitelist as blocking for PRs - less if condition per step cons: - quite verbose - need to manually sync the `ci-NAME-status-check` needs list to any other existing and should be dep jobs ## Version migration Migrated to the latest `changed-files@45` version, getting rid of the `set-output` usage warnings ## Tests runs: With mutation: - [Success flow](https://github.com/twentyhq/twenty/actions/runs/12791958651/job/35661546343) - [server-setup failure flow](https://github.com/twentyhq/twenty/actions/runs/12792225779) - [Other job failure flow](https://github.com/twentyhq/twenty/actions/runs/12792313463), one of the `inner` job failed - [Manual cancel flow](https://github.com/twentyhq/twenty/actions/runs/12792313463) `ci-server-status-check` also has the `cancelled` status - [Matrix failure](https://github.com/twentyhq/twenty/actions/runs/12806883553) Without mutation: - [Nothing to do flow](https://github.com/twentyhq/twenty/actions/runs/12792098384), skipped `inner` job but `ci-server-status-check` still succeeded ## Notes ### Linter We should setup a `yml` prettier and linter for the `.github/worfklows` folder ### Centralized `ci-NAME-status-check` logic Unfortunately I couldn't achieve to either make a `composite` action or a `reusable-workflow`, as I could not access the correct layer to run the `always` but also acessing the `needs` context --- .github/workflows/changed-files.yaml | 27 +++ .github/workflows/ci-chrome-extension.yaml | 37 +++-- .github/workflows/ci-e2e.yaml | 43 ++--- .github/workflows/ci-front.yaml | 120 ++++---------- .github/workflows/ci-server.yaml | 82 +++------ .github/workflows/ci-shared.yaml | 29 ++-- .github/workflows/ci-test-docker-compose.yaml | 155 +++++++++--------- .github/workflows/ci-website.yaml | 37 +++-- 8 files changed, 241 insertions(+), 289 deletions(-) create mode 100644 .github/workflows/changed-files.yaml diff --git a/.github/workflows/changed-files.yaml b/.github/workflows/changed-files.yaml new file mode 100644 index 000000000..b1d028dde --- /dev/null +++ b/.github/workflows/changed-files.yaml @@ -0,0 +1,27 @@ +name: Changed files reusable workflow +on: + workflow_call: + inputs: + files: + required: true + type: string + outputs: + any_changed: + value: ${{ jobs.changed-files.outputs.any_changed }} + +jobs: + changed-files: + timeout-minutes: 5 + runs-on: ubuntu-latest + outputs: + any_changed: ${{ steps.changed-files.outputs.any_changed }} + steps: + - name: Fetch custom Github Actions and base branch history + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check for changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: ${{ inputs.files }} diff --git a/.github/workflows/ci-chrome-extension.yaml b/.github/workflows/ci-chrome-extension.yaml index 63c7094f7..0f77abef4 100644 --- a/.github/workflows/ci-chrome-extension.yaml +++ b/.github/workflows/ci-chrome-extension.yaml @@ -3,15 +3,23 @@ on: push: branches: - main - + pull_request: - + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + package.json + packages/twenty-chrome-extension/** chrome-extension-build: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 15 runs-on: ubuntu-latest env: @@ -25,23 +33,16 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - package.json - packages/twenty-chrome-extension/** - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Chrome Extension / Run build - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx build twenty-chrome-extension - - - name: Mark as Valid if No Changes - if: steps.changed-files.outputs.changed != 'true' - run: | - echo "No relevant changes detected. Marking as valid." + ci-chrome-extension-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: ubuntu-latest + needs: [prerequisites, chrome-extension-build] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index 34bd52862..f62622d43 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -11,9 +11,17 @@ concurrency: cancel-in-progress: true jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + packages/** + playwright.config.ts + .github/workflows/ci-e2e.yaml test: runs-on: ubuntu-latest - if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-e2e')) + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' && ( github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-e2e'))) timeout-minutes: 30 env: NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 @@ -55,29 +63,13 @@ jobs: echo "CPU info:" lscpu - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/** - playwright.config.ts - .github/workflows/ci-e2e.yaml - - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes detected. Marking as valid." - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Build twenty-shared - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx build twenty-shared - name: Setup environment files - if: steps.changed-files.outputs.any_changed == 'true' run: | cp packages/twenty-e2e-testing/.env.example packages/twenty-e2e-testing/.env cp packages/twenty-front/.env.example packages/twenty-front/.env @@ -85,46 +77,38 @@ jobs: npx nx reset:env twenty-server - name: Build frontend - if: steps.changed-files.outputs.any_changed == 'true' run: NODE_ENV=production NODE_OPTIONS="--max-old-space-size=10240" npx nx build twenty-front - name: Build server - if: steps.changed-files.outputs.any_changed == 'true' run: NODE_ENV=production npx nx build twenty-server - name: Create and setup database - if: steps.changed-files.outputs.any_changed == 'true' run: | PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";' PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";' npx nx run twenty-server:database:reset - name: Start server - if: steps.changed-files.outputs.any_changed == 'true' run: | npx nx start twenty-server & echo "Waiting for server to be ready..." timeout 60 bash -c 'until curl -s http://localhost:3000/health; do sleep 2; done' - name: Start frontend - if: steps.changed-files.outputs.any_changed == 'true' run: | npm_config_yes=true npx serve -s packages/twenty-front/build -l 3001 & echo "Waiting for frontend to be ready..." timeout 60 bash -c 'until curl -s http://localhost:3001; do sleep 2; done' - name: Start worker - if: steps.changed-files.outputs.any_changed == 'true' run: | npx nx run twenty-server:worker:ci & echo "Worker started" - name: Install Playwright Browsers - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx setup twenty-e2e-testing - name: Run Playwright tests - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx test twenty-e2e-testing - uses: actions/upload-artifact@v4 @@ -140,3 +124,12 @@ jobs: name: playwright-report path: packages/twenty-e2e-testing/playwright-report/ retention-days: 30 + ci-e2e-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: ubuntu-latest + needs: [prerequisites, test] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-front.yaml b/.github/workflows/ci-front.yaml index 234e41ff1..b8e29083e 100644 --- a/.github/workflows/ci-front.yaml +++ b/.github/workflows/ci-front.yaml @@ -3,7 +3,7 @@ on: push: branches: - main - + pull_request: concurrency: @@ -14,7 +14,17 @@ env: STORYBOOK_BUILD_CACHE_KEY: storybook-build-depot-ubuntu-24.04-8-runner jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + package.json + packages/twenty-front/** + packages/twenty-ui/** + packages/twenty-shared/** front-sb-build: + needs: [prerequisites] + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 30 runs-on: depot-ubuntu-24.04-8 env: @@ -29,39 +39,20 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - package.json - packages/twenty-front/** - packages/twenty-ui/** - packages/twenty-shared/** - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Diagnostic disk space issue - if: steps.changed-files.outputs.any_changed == 'true' run: df -h - name: Restore storybook build cache id: restore-storybook-build-cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache - with: + with: key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }} - name: Front / Write .env - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-front - name: Front / Build storybook - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:build twenty-front - name: Save storybook build cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/save-cache with: key: ${{ steps.restore-storybook-build-cache.outputs.cache-primary-key }} @@ -83,38 +74,21 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-front/** - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Install Playwright - if: steps.changed-files.outputs.any_changed == 'true' run: cd packages/twenty-front && npx playwright install - name: Restore storybook build cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache - with: + with: key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }} - name: Front / Write .env - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-front - name: Run storybook tests - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:serve-and-test:static twenty-front --configuration=${{ matrix.storybook_scope }} --shard=${{ matrix.shard }}/${{ env.SHARD_COUNTER }} --checkCoverage=false - name: Rename coverage file - if: steps.changed-files.outputs.any_changed == 'true' run: mv packages/twenty-front/coverage/storybook/coverage-storybook.json packages/twenty-front/coverage/storybook/coverage-shard-${{matrix.shard}}.json - name: Upload coverage artifact - if: steps.changed-files.outputs.any_changed == 'true' uses: actions/upload-artifact@v4 with: retention-days: 1 @@ -133,31 +107,18 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-front/** - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - uses: actions/download-artifact@v4 - if: steps.changed-files.outputs.any_changed == 'true' with: pattern: coverage-artifacts-${{ matrix.storybook_scope }}-${{ github.run_id }}-* merge-multiple: true path: coverage-artifacts - name: Merge coverage reports - if: steps.changed-files.outputs.any_changed == 'true' run: | mkdir -p ${{ env.PATH_TO_COVERAGE }} npx nyc merge coverage-artifacts ${{ env.PATH_TO_COVERAGE }}/coverage-storybook.json - name: Checking coverage - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx storybook:coverage twenty-front --checkCoverage=true front-chromatic-deployment: timeout-minutes: 30 @@ -172,36 +133,22 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-front/** - - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Restore storybook build cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache - with: + with: key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }} - name: Front / Write .env - if: steps.changed-files.outputs.any_changed == 'true' run: | cd packages/twenty-front touch .env echo "REACT_APP_SERVER_BASE_URL: $REACT_APP_SERVER_BASE_URL" >> .env - name: Publish to Chromatic - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx run twenty-front:chromatic:ci front-task: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 30 runs-on: depot-ubuntu-24.04-8 env: @@ -219,40 +166,41 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-front/** - - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Restore ${{ matrix.task }} cache id: restore-task-cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache - with: + with: key: ${{ env.TASK_CACHE_KEY }} - name: Reset .env - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:frontend tasks: reset:env - name: Run ${{ matrix.task }} task - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:frontend tasks: ${{ matrix.task }} - name: Save ${{ matrix.task }} cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/save-cache with: key: ${{ steps.restore-task-cache.outputs.cache-primary-key }} + ci-front-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: depot-ubuntu-24.04-8 + needs: + [ + prerequisites, + front-task, + front-chromatic-deployment, + merge-reports-and-check-coverage, + front-sb-test, + front-sb-build, + ] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index 58d3c9288..a866bf8db 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -5,7 +5,7 @@ on: - main pull_request: - + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -14,7 +14,17 @@ env: SERVER_SETUP_CACHE_KEY: server-setup jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + package.json + packages/twenty-server/** + packages/twenty-emails/** + packages/twenty-shared/** server-setup: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 30 runs-on: depot-ubuntu-24.04-8 env: @@ -25,8 +35,8 @@ jobs: env: PGUSER_SUPERUSER: postgres PGPASSWORD_SUPERUSER: postgres - ALLOW_NOSSL: "true" - SPILO_PROVIDER: "local" + ALLOW_NOSSL: 'true' + SPILO_PROVIDER: 'local' ports: - 5432:5432 options: >- @@ -43,53 +53,33 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - package.json - packages/twenty-server/** - packages/twenty-emails/** - packages/twenty-shared/** - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Build twenty-shared - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx build twenty-shared - name: Restore server setup id: restore-server-setup-cache - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache with: key: ${{ env.SERVER_SETUP_CACHE_KEY }} - name: Server / Run lint & typecheck - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend tasks: lint,typecheck - name: Server / Build - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx build twenty-server - name: Server / Write .env - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx reset:env twenty-server - name: Server / Create DB - if: steps.changed-files.outputs.any_changed == 'true' run: | PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";' PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";' npx nx run twenty-server:database:init:prod npx nx run twenty-server:database:migrate:prod - name: Worker / Run - if: steps.changed-files.outputs.any_changed == 'true' run: npx nx run twenty-server:worker:ci - name: Server / Check for Pending Migrations - if: steps.changed-files.outputs.any_changed == 'true' run: | METADATA_MIGRATION_OUTPUT=$(npx nx run twenty-server:typeorm migration:generate metadata-migration-check -d src/database/typeorm/metadata/metadata.datasource.ts || true) @@ -108,7 +98,6 @@ jobs: exit 1 fi - name: Save server setup - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/save-cache with: key: ${{ steps.restore-server-setup-cache.outputs.cache-primary-key }} @@ -124,27 +113,13 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - package.json - packages/twenty-server/** - packages/twenty-emails/** - packages/twenty-shared/** - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Restore server setup - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache with: key: ${{ env.SERVER_SETUP_CACHE_KEY }} - - name: Server / Run Tests - if: steps.changed-files.outputs.any_changed == 'true' + - name: Server / Run Tests uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend @@ -160,8 +135,8 @@ jobs: env: PGUSER_SUPERUSER: postgres PGPASSWORD_SUPERUSER: postgres - ALLOW_NOSSL: "true" - SPILO_PROVIDER: "local" + ALLOW_NOSSL: 'true' + SPILO_PROVIDER: 'local' ports: - 5432:5432 options: >- @@ -180,40 +155,35 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - package.json - packages/twenty-server/** - packages/twenty-emails/** - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Update .env.test for billing - if: steps.changed-files.outputs.any_changed == 'true' run: | echo "IS_BILLING_ENABLED=true" >> .env.test echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test echo "BILLING_STRIPE_BASE_PLAN_PRODUCT_ID=test-base-plan-product-id" >> .env.test echo "BILLING_STRIPE_WEBHOOK_SECRET=test-webhook-secret" >> .env.test - name: Restore server setup - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/restore-cache with: key: ${{ env.SERVER_SETUP_CACHE_KEY }} - name: Server / Run Integration Tests - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:backend - tasks: "test:integration:with-db-reset" + tasks: 'test:integration:with-db-reset' - name: Server / Upload reset-logs file if: always() uses: actions/upload-artifact@v4 with: name: reset-logs path: reset-logs.log + ci-server-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: depot-ubuntu-24.04-8 + needs: [prerequisites, server-setup, server-test, server-integration-test] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-shared.yaml b/.github/workflows/ci-shared.yaml index 41c334e05..2a0d76ab4 100644 --- a/.github/workflows/ci-shared.yaml +++ b/.github/workflows/ci-shared.yaml @@ -3,7 +3,7 @@ on: push: branches: - main - + pull_request: concurrency: @@ -11,7 +11,14 @@ concurrency: cancel-in-progress: true jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + packages/twenty-shared/** shared-test: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 30 runs-on: ubuntu-latest env: @@ -28,21 +35,19 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-shared/** - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed == 'false' - run: echo "No relevant changes. Skipping CI." - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Run ${{ matrix.task }} task - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/workflows/actions/nx-affected with: tag: scope:frontend tasks: ${{ matrix.task }} + ci-shared-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: ubuntu-latest + needs: [prerequisites, shared-test] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-test-docker-compose.yaml b/.github/workflows/ci-test-docker-compose.yaml index a9a076da3..47cc6703b 100644 --- a/.github/workflows/ci-test-docker-compose.yaml +++ b/.github/workflows/ci-test-docker-compose.yaml @@ -1,92 +1,97 @@ name: 'Test Docker Compose' on: pull_request: - + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + packages/twenty-docker/** + docker-compose.yml test: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 30 runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: | - packages/twenty-docker/** - docker-compose.yml - - name: Skip if no relevant changes - if: steps.changed-files.outputs.any_changed != 'true' - run: echo "No relevant changes detected. Marking as valid." + - name: Checkout + uses: actions/checkout@v4 + - name: Run compose + run: | + echo "Patching docker-compose.yml..." + # change image to localbuild using yq + yq eval 'del(.services.server.image)' -i docker-compose.yml + yq eval '.services.server.build.context = "../../"' -i docker-compose.yml + yq eval '.services.server.build.dockerfile = "./packages/twenty-docker/twenty/Dockerfile"' -i docker-compose.yml + yq eval '.services.server.restart = "no"' -i docker-compose.yml - - name: Run compose - if: steps.changed-files.outputs.any_changed == 'true' - run: | - echo "Patching docker-compose.yml..." - # change image to localbuild using yq - yq eval 'del(.services.server.image)' -i docker-compose.yml - yq eval '.services.server.build.context = "../../"' -i docker-compose.yml - yq eval '.services.server.build.dockerfile = "./packages/twenty-docker/twenty/Dockerfile"' -i docker-compose.yml - yq eval '.services.server.restart = "no"' -i docker-compose.yml + yq eval 'del(.services.db.image)' -i docker-compose.yml + yq eval '.services.db.build.context = "../../"' -i docker-compose.yml + yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres-spilo/Dockerfile"' -i docker-compose.yml - yq eval 'del(.services.db.image)' -i docker-compose.yml - yq eval '.services.db.build.context = "../../"' -i docker-compose.yml - yq eval '.services.db.build.dockerfile = "./packages/twenty-docker/twenty-postgres-spilo/Dockerfile"' -i docker-compose.yml + echo "Setting up .env file..." + cp .env.example .env + echo "Generating secrets..." + echo "# === Randomly generated secrets ===" >>.env + echo "APP_SECRET=$(openssl rand -base64 32)" >>.env + echo "PGPASSWORD_SUPERUSER=$(openssl rand -base64 32)" >>.env - echo "Setting up .env file..." - cp .env.example .env - echo "Generating secrets..." - echo "# === Randomly generated secrets ===" >>.env - echo "APP_SECRET=$(openssl rand -base64 32)" >>.env - echo "PGPASSWORD_SUPERUSER=$(openssl rand -base64 32)" >>.env + echo "Docker compose up..." + docker compose up -d || { + echo "Docker compose failed to start" + docker compose logs + exit 1 + } + docker compose logs db server -f & + pid=$! - echo "Docker compose up..." - docker compose up -d || { - echo "Docker compose failed to start" - docker compose logs - exit 1 - } - docker compose logs db server -f & - pid=$! + echo "Waiting for database to start..." + count=0 + while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-db-1) = "healthy" ]; do + sleep 1; + count=$((count+1)); + if [ $(docker inspect --format='{{.State.Status}}' twenty-db-1) = "exited" ]; then + echo "Database exited" + docker compose logs db + exit 1 + fi + if [ $count -gt 300 ]; then + echo "Failed to start database after 5 minutes" + docker compose logs db + exit 1 + fi + echo "Still waiting for database... (${count}/60)" + done - echo "Waiting for database to start..." - count=0 - while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-db-1) = "healthy" ]; do - sleep 1; - count=$((count+1)); - if [ $(docker inspect --format='{{.State.Status}}' twenty-db-1) = "exited" ]; then - echo "Database exited" - docker compose logs db - exit 1 - fi - if [ $count -gt 300 ]; then - echo "Failed to start database after 5 minutes" - docker compose logs db - exit 1 - fi - echo "Still waiting for database... (${count}/60)" - done - - echo "Waiting for server to start..." - count=0 - while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-server-1) = "healthy" ]; do - sleep 1; - count=$((count+1)); - if [ $(docker inspect --format='{{.State.Status}}' twenty-server-1) = "exited" ]; then - echo "Server exited" - docker compose logs server - exit 1 - fi - if [ $count -gt 300 ]; then - echo "Failed to start server after 5 minutes" - docker compose logs server - exit 1 - fi - echo "Still waiting for server... (${count}/300s)" - done - working-directory: ./packages/twenty-docker/ + echo "Waiting for server to start..." + count=0 + while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-server-1) = "healthy" ]; do + sleep 1; + count=$((count+1)); + if [ $(docker inspect --format='{{.State.Status}}' twenty-server-1) = "exited" ]; then + echo "Server exited" + docker compose logs server + exit 1 + fi + if [ $count -gt 300 ]; then + echo "Failed to start server after 5 minutes" + docker compose logs server + exit 1 + fi + echo "Still waiting for server... (${count}/300s)" + done + working-directory: ./packages/twenty-docker/ + ci-test-docker-compose-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: ubuntu-latest + needs: [prerequisites, test] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1 diff --git a/.github/workflows/ci-website.yaml b/.github/workflows/ci-website.yaml index ed5247a48..7f12c327d 100644 --- a/.github/workflows/ci-website.yaml +++ b/.github/workflows/ci-website.yaml @@ -5,14 +5,21 @@ on: - main pull_request: - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + prerequisites: + uses: ./.github/workflows/changed-files.yaml + with: + files: | + package.json + packages/twenty-website/** website-build: + needs: prerequisites + if: needs.prerequisites.outputs.any_changed == 'true' timeout-minutes: 3 runs-on: ubuntu-latest services: @@ -21,8 +28,8 @@ jobs: env: PGUSER_SUPERUSER: postgres PGPASSWORD_SUPERUSER: postgres - ALLOW_NOSSL: "true" - SPILO_PROVIDER: "local" + ALLOW_NOSSL: 'true' + SPILO_PROVIDER: 'local' ports: - 5432:5432 options: >- @@ -34,31 +41,27 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed files - id: changed-files - uses: tj-actions/changed-files@v11 - with: - files: 'package.json, packages/twenty-website/**' - name: Install dependencies - if: steps.changed-files.outputs.changed == 'true' uses: ./.github/workflows/actions/yarn-install - name: Server / Create DB - if: steps.changed-files.outputs.any_changed == 'true' run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";' - name: Website / Run migrations - if: steps.changed-files.outputs.changed == 'true' - run: npx nx database:migrate twenty-website + run: npx nx database:migrate twenty-website env: DATABASE_PG_URL: postgres://postgres:postgres@localhost:5432/default - name: Website / Build Website - if: steps.changed-files.outputs.changed == 'true' run: npx nx build twenty-website env: DATABASE_PG_URL: postgres://postgres:postgres@localhost:5432/default - - - name: Mark as VALID - if: steps.changed-files.outputs.changed != 'true' # If no changes, mark as valid - run: echo "No relevant changes detected. CI is valid." \ No newline at end of file + ci-website-status-check: + if: always() && !cancelled() + timeout-minutes: 1 + runs-on: ubuntu-latest + needs: [prerequisites, website-build] + steps: + - name: Fail job if any needs failed + if: contains(needs.*.result, 'failure') + run: exit 1