[CI] Refactor composite action task-cache (#9583)
## Introduction In this previous PR https://github.com/twentyhq/twenty/pull/9448 we've refactored the storybook build caching flow to be using the new [actions/cache](https://github.com/actions/cache) `restore` and `save` functions, which significantly improve caching operations duration. In this way, in this PR, we've standardize both of the `restore` and `save` by refactoring the `task-cache` composite action. By creating two new composite actions `save-cache` and `restore-cache` that centralize the paths to cache and the way to compute the primary key. ## Misc - **If no cache** is hit, then a job duration will long for its task duration and nothing else, previously the cache upload would sometimes take up to 3 mins. - **if cache** is hit, then mainly the only time consuming step is the dependencies installation ( which is theory is also cached, in fact twice. We will be having a look on this issue in an upcoming PR )
This commit is contained in:
32
.github/workflows/actions/restore-cache/action.yml
vendored
Normal file
32
.github/workflows/actions/restore-cache/action.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
name: Restore cache
|
||||||
|
inputs:
|
||||||
|
key:
|
||||||
|
required: true
|
||||||
|
description: Prefix to the cache key
|
||||||
|
type: string
|
||||||
|
additional-paths:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
outputs:
|
||||||
|
cache-primary-key:
|
||||||
|
description: actions/cache/restore cache-primary-key outputs proxy
|
||||||
|
value: ${{ steps.restore-cache.outputs.cache-primary-key }}
|
||||||
|
cache-hit:
|
||||||
|
description: String bool indicating whether cache has been directly or indirectly hit
|
||||||
|
value: ${{ steps.restore-cache.outputs.cache-hit != 'false' || steps.restore-cache.outputs.cache-matched-key != '' }}
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- name: Restore cache
|
||||||
|
uses: actions/cache/restore@v4
|
||||||
|
id: restore-cache
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
.cache
|
||||||
|
.nx/cache
|
||||||
|
node_modules/.cache
|
||||||
|
packages/*/node_modules/.cache
|
||||||
|
${{ inputs.additional-paths }}
|
||||||
|
key: ${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }}
|
||||||
|
restore-keys: ${{ inputs.key }}-${{ github.ref_name }}-
|
||||||
23
.github/workflows/actions/save-cache/action.yml
vendored
Normal file
23
.github/workflows/actions/save-cache/action.yml
vendored
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
name: Save cache
|
||||||
|
inputs:
|
||||||
|
key:
|
||||||
|
required: true
|
||||||
|
description: Primary key to the cache, should be retrieved from `cache-restore` composite action outputs.
|
||||||
|
type: string
|
||||||
|
additional-paths:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: "composite"
|
||||||
|
steps:
|
||||||
|
- name: Save cache
|
||||||
|
uses: actions/cache/save@v4
|
||||||
|
with:
|
||||||
|
key: ${{ inputs.key }}
|
||||||
|
path: |
|
||||||
|
.cache
|
||||||
|
.nx/cache
|
||||||
|
node_modules/.cache
|
||||||
|
packages/*/node_modules/.cache
|
||||||
|
${{ inputs.additional-paths }}
|
||||||
31
.github/workflows/actions/task-cache/action.yaml
vendored
31
.github/workflows/actions/task-cache/action.yaml
vendored
@ -1,31 +0,0 @@
|
|||||||
name: Restore Tasks Cache CI
|
|
||||||
inputs:
|
|
||||||
tag:
|
|
||||||
required: false
|
|
||||||
types: [string]
|
|
||||||
tasks:
|
|
||||||
required: false
|
|
||||||
types: [string]
|
|
||||||
default: all
|
|
||||||
suffix:
|
|
||||||
required: false
|
|
||||||
types: [string]
|
|
||||||
|
|
||||||
runs:
|
|
||||||
using: "composite"
|
|
||||||
steps:
|
|
||||||
- name: Compute tasks key
|
|
||||||
id: tasks-key
|
|
||||||
shell: bash
|
|
||||||
run: echo "key=${{ inputs.tasks }}" | tr , - >> $GITHUB_OUTPUT
|
|
||||||
- name: Restore tasks cache
|
|
||||||
uses: actions/cache@v3
|
|
||||||
with:
|
|
||||||
path: |
|
|
||||||
.cache
|
|
||||||
.nx/cache
|
|
||||||
node_modules/.cache
|
|
||||||
packages/*/node_modules/.cache
|
|
||||||
key: tasks-cache-${{ github.ref_name }}-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-${{ github.sha }}
|
|
||||||
restore-keys: |
|
|
||||||
tasks-cache-${{ github.ref_name }}-${{ inputs.tag }}-${{ steps.tasks-key.outputs.key }}${{ inputs.suffix }}-
|
|
||||||
78
.github/workflows/ci-front.yaml
vendored
78
.github/workflows/ci-front.yaml
vendored
@ -10,6 +10,9 @@ concurrency:
|
|||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
group: ${{ github.workflow }}-${{ github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
STORYBOOK_BUILD_CACHE_KEY: storybook-build-depot-ubuntu-22.04-8-runner
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
front-sb-build:
|
front-sb-build:
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@ -46,17 +49,11 @@ jobs:
|
|||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: df -h
|
run: df -h
|
||||||
- name: Restore storybook build cache
|
- name: Restore storybook build cache
|
||||||
id: storybook-build-cache-restore
|
id: restore-storybook-build-cache
|
||||||
uses: actions/cache/restore@v4
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
with:
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
path: |
|
with:
|
||||||
.cache
|
key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }}
|
||||||
.nx/cache
|
|
||||||
node_modules/.cache
|
|
||||||
packages/*/node_modules/.cache
|
|
||||||
key: storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-${{ github.sha }}
|
|
||||||
restore-keys: |
|
|
||||||
storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-
|
|
||||||
- name: Front / Write .env
|
- name: Front / Write .env
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: npx nx reset:env twenty-front
|
run: npx nx reset:env twenty-front
|
||||||
@ -64,15 +61,10 @@ jobs:
|
|||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: npx nx storybook:build twenty-front
|
run: npx nx storybook:build twenty-front
|
||||||
- name: Save storybook build cache
|
- name: Save storybook build cache
|
||||||
if: steps.storybook-build-cache-restore.outputs.cache-hit != 'true'
|
if: steps.changed-files.outputs.any_changed == 'true' && steps.restore-storybook-build-cache.outputs.cache-hit != 'true'
|
||||||
uses: actions/cache/save@v4
|
uses: ./.github/workflows/actions/save-cache
|
||||||
with:
|
with:
|
||||||
key: ${{ steps.storybook-build-cache-restore.outputs.cache-primary-key }}
|
key: ${{ steps.restore-storybook-build-cache.outputs.cache-primary-key }}
|
||||||
path: |
|
|
||||||
.cache
|
|
||||||
.nx/cache
|
|
||||||
node_modules/.cache
|
|
||||||
packages/*/node_modules/.cache
|
|
||||||
front-sb-test:
|
front-sb-test:
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
runs-on: depot-ubuntu-22.04-8
|
runs-on: depot-ubuntu-22.04-8
|
||||||
@ -107,17 +99,11 @@ jobs:
|
|||||||
- name: Install Playwright
|
- name: Install Playwright
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: cd packages/twenty-front && npx playwright install
|
run: cd packages/twenty-front && npx playwright install
|
||||||
- name: Restore Storybook build cache
|
- name: Restore storybook build cache
|
||||||
uses: actions/cache/restore@v4
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
with:
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
path: |
|
with:
|
||||||
.cache
|
key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }}
|
||||||
.nx/cache
|
|
||||||
node_modules/.cache
|
|
||||||
packages/*/node_modules/.cache
|
|
||||||
key: storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-${{ github.sha }}
|
|
||||||
restore-keys: |
|
|
||||||
storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-
|
|
||||||
- name: Front / Write .env
|
- name: Front / Write .env
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: npx nx reset:env twenty-front
|
run: npx nx reset:env twenty-front
|
||||||
@ -128,6 +114,7 @@ jobs:
|
|||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: mv packages/twenty-front/coverage/storybook/coverage-storybook.json packages/twenty-front/coverage/storybook/coverage-shard-${{matrix.shard}}.json
|
run: mv packages/twenty-front/coverage/storybook/coverage-storybook.json packages/twenty-front/coverage/storybook/coverage-shard-${{matrix.shard}}.json
|
||||||
- name: Upload coverage artifact
|
- name: Upload coverage artifact
|
||||||
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
@ -201,17 +188,10 @@ jobs:
|
|||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/yarn-install
|
uses: ./.github/workflows/actions/yarn-install
|
||||||
- name: Restore storybook build cache
|
- name: Restore storybook build cache
|
||||||
id: storybook-build-cache-restore
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: actions/cache/restore@v4
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
with:
|
with:
|
||||||
path: |
|
key: ${{ env.STORYBOOK_BUILD_CACHE_KEY }}
|
||||||
.cache
|
|
||||||
.nx/cache
|
|
||||||
node_modules/.cache
|
|
||||||
packages/*/node_modules/.cache
|
|
||||||
key: storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-${{ github.sha }}
|
|
||||||
restore-keys: |
|
|
||||||
storybook-build-depot-ubuntu-22.04-8-runner-${{ github.ref_name }}-
|
|
||||||
- name: Front / Write .env
|
- name: Front / Write .env
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: |
|
run: |
|
||||||
@ -226,6 +206,7 @@ jobs:
|
|||||||
runs-on: depot-ubuntu-22.04-8
|
runs-on: depot-ubuntu-22.04-8
|
||||||
env:
|
env:
|
||||||
NX_REJECT_UNKNOWN_LOCAL_CACHE: 0
|
NX_REJECT_UNKNOWN_LOCAL_CACHE: 0
|
||||||
|
TASK_CACHE_KEY: front-task-${{ matrix.task }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
task: [lint, typecheck, test]
|
task: [lint, typecheck, test]
|
||||||
@ -252,12 +233,12 @@ jobs:
|
|||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/yarn-install
|
uses: ./.github/workflows/actions/yarn-install
|
||||||
- name: Front / Restore ${{ matrix.task }} task cache
|
- name: Restore ${{ matrix.task }} cache
|
||||||
|
id: restore-task-cache
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/task-cache
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
with:
|
with:
|
||||||
tag: scope:frontend
|
key: ${{ env.TASK_CACHE_KEY }}
|
||||||
tasks: ${{ matrix.task }}
|
|
||||||
- name: Reset .env
|
- name: Reset .env
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/nx-affected
|
uses: ./.github/workflows/actions/nx-affected
|
||||||
@ -270,3 +251,8 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
tag: scope:frontend
|
tag: scope:frontend
|
||||||
tasks: ${{ matrix.task }}
|
tasks: ${{ matrix.task }}
|
||||||
|
- name: Save ${{ matrix.task }} cache
|
||||||
|
if: steps.changed-files.outputs.any_changed == 'true' && steps.restore-task-cache.outputs.cache-hit != 'true'
|
||||||
|
uses: ./.github/workflows/actions/save-cache
|
||||||
|
with:
|
||||||
|
key: ${{ steps.restore-task-cache.outputs.cache-primary-key }}
|
||||||
|
|||||||
27
.github/workflows/ci-server.yaml
vendored
27
.github/workflows/ci-server.yaml
vendored
@ -10,6 +10,9 @@ concurrency:
|
|||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
group: ${{ github.workflow }}-${{ github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
SERVER_SETUP_CACHE_KEY: server-setup
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
server-setup:
|
server-setup:
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@ -57,11 +60,12 @@ jobs:
|
|||||||
- name: Build twenty-shared
|
- name: Build twenty-shared
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
run: npx nx build twenty-shared
|
run: npx nx build twenty-shared
|
||||||
- name: Server / Restore Task Cache
|
- name: Restore server setup
|
||||||
|
id: restore-server-setup-cache
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/task-cache
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
with:
|
with:
|
||||||
tag: scope:backend
|
key: ${{ env.SERVER_SETUP_CACHE_KEY }}
|
||||||
- name: Server / Run lint & typecheck
|
- name: Server / Run lint & typecheck
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/nx-affected
|
uses: ./.github/workflows/actions/nx-affected
|
||||||
@ -103,6 +107,11 @@ jobs:
|
|||||||
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
- name: Save server setup
|
||||||
|
if: steps.changed-files.outputs.any_changed == 'true' && steps.restore-server-setup-cache.outputs.cache-hit != 'true'
|
||||||
|
uses: ./.github/workflows/actions/save-cache
|
||||||
|
with:
|
||||||
|
key: ${{ steps.restore-server-setup-cache.outputs.cache-primary-key }}
|
||||||
|
|
||||||
server-test:
|
server-test:
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@ -129,11 +138,11 @@ jobs:
|
|||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/yarn-install
|
uses: ./.github/workflows/actions/yarn-install
|
||||||
- name: Server / Restore Task Cache
|
- name: Restore server setup
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/task-cache
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
with:
|
with:
|
||||||
tag: scope:backend
|
key: ${{ env.SERVER_SETUP_CACHE_KEY }}
|
||||||
- name: Server / Run Tests
|
- name: Server / Run Tests
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/nx-affected
|
uses: ./.github/workflows/actions/nx-affected
|
||||||
@ -191,11 +200,11 @@ jobs:
|
|||||||
echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test
|
echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test
|
||||||
echo "BILLING_STRIPE_BASE_PLAN_PRODUCT_ID=test-base-plan-product-id" >> .env.test
|
echo "BILLING_STRIPE_BASE_PLAN_PRODUCT_ID=test-base-plan-product-id" >> .env.test
|
||||||
echo "BILLING_STRIPE_WEBHOOK_SECRET=test-webhook-secret" >> .env.test
|
echo "BILLING_STRIPE_WEBHOOK_SECRET=test-webhook-secret" >> .env.test
|
||||||
- name: Server / Restore Task Cache
|
- name: Restore server setup
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/task-cache
|
uses: ./.github/workflows/actions/restore-cache
|
||||||
with:
|
with:
|
||||||
tag: scope:backend
|
key: ${{ env.SERVER_SETUP_CACHE_KEY }}
|
||||||
- name: Server / Run Integration Tests
|
- name: Server / Run Integration Tests
|
||||||
if: steps.changed-files.outputs.any_changed == 'true'
|
if: steps.changed-files.outputs.any_changed == 'true'
|
||||||
uses: ./.github/workflows/actions/nx-affected
|
uses: ./.github/workflows/actions/nx-affected
|
||||||
|
|||||||
Reference in New Issue
Block a user