[REFACTOR] twenty-shared multi barrel and CJS/ESM build with preconstruct (#11083)

# Introduction

In this PR we've migrated `twenty-shared` from a `vite` app
[libary-mode](https://vite.dev/guide/build#library-mode) to a
[preconstruct](https://preconstruct.tools/) "atomic" application ( in
the future would like to introduce preconstruct to handle of all our
atomic dependencies such as `twenty-emails` `twenty-ui` etc it will be
integrated at the monorepo's root directly, would be to invasive in the
first, starting incremental via `twenty-shared`)

For more information regarding the motivations please refer to nor:
- https://github.com/twentyhq/core-team-issues/issues/587
-
https://github.com/twentyhq/core-team-issues/issues/281#issuecomment-2630949682

close https://github.com/twentyhq/core-team-issues/issues/589
close https://github.com/twentyhq/core-team-issues/issues/590

## How to test
In order to ease the review this PR will ship all the codegen at the
very end, the actual meaning full diff is `+2,411 −114`
In order to migrate existing dependent packages to `twenty-shared` multi
barrel new arch you need to run in local:
```sh
yarn tsx packages/twenty-shared/scripts/migrateFromSingleToMultiBarrelImport.ts && \
npx nx run-many -t lint --fix -p twenty-front twenty-ui twenty-server twenty-emails twenty-shared twenty-zapier
```
Note that `migrateFromSingleToMultiBarrelImport` is idempotent, it's atm
included in the PR but should not be merged. ( such as codegen will be
added before merging this script will be removed )

## Misc
- related opened issue preconstruct
https://github.com/preconstruct/preconstruct/issues/617

## Closed related PR
- https://github.com/twentyhq/twenty/pull/11028
- https://github.com/twentyhq/twenty/pull/10993
- https://github.com/twentyhq/twenty/pull/10960

## Upcoming enhancement: ( in others dedicated PRs )
- 1/ refactor generate barrel to export atomic module instead of `*`
- 2/ generate barrel own package with several files and tests
- 3/ Migration twenty-ui the same way
- 4/ Use `preconstruct` at monorepo global level

## Conclusion
As always any suggestions are welcomed !
This commit is contained in:
Paul Rastoin
2025-03-22 19:16:06 +01:00
committed by GitHub
parent 8a21c19f03
commit 9ad8287dbc
1091 changed files with 3611 additions and 1297 deletions

View File

@ -0,0 +1,253 @@
import prettier from '@prettier/sync';
import * as fs from 'fs';
import path from 'path';
import { Options } from 'prettier';
import slash from 'slash';
// TODO prastoin refactor this file in several one into its dedicated package and make it a TypeScript CLI
const INCLUDED_EXTENSIONS = ['.ts', '.tsx'];
const EXCLUDED_EXTENSIONS = [
'.test.ts',
'.test.tsx',
'.spec.ts',
'.spec.tsx',
'.stories.ts',
'.stories.tsx',
];
const EXCLUDED_DIRECTORIES = [
'__tests__',
'__mocks__',
'__stories__',
'internal',
];
const INDEX_FILENAME = 'index';
const PACKAGE_JSON_FILENAME = 'package.json';
const NX_PROJECT_CONFIGURATION_FILENAME = 'project.json';
const PACKAGE_PATH = path.resolve('packages/twenty-shared');
const SRC_PATH = path.resolve(`${PACKAGE_PATH}/src`);
const PACKAGE_JSON_PATH = path.join(PACKAGE_PATH, PACKAGE_JSON_FILENAME);
const NX_PROJECT_CONFIGURATION_PATH = path.join(
PACKAGE_PATH,
NX_PROJECT_CONFIGURATION_FILENAME,
);
const prettierConfigFile = prettier.resolveConfigFile();
if (prettierConfigFile == null) {
throw new Error('Prettier config file not found');
}
const prettierConfiguration = prettier.resolveConfig(prettierConfigFile);
const prettierFormat = (str: string, parser: Options['parser']) =>
prettier.format(str, {
...prettierConfiguration,
parser,
});
type createTypeScriptFileArgs = {
path: string;
content: string;
filename: string;
};
const createTypeScriptFile = ({
content,
path: filePath,
filename,
}: createTypeScriptFileArgs) => {
const header = `
/*
* _____ _
*|_ _|_ _____ _ __ | |_ _ _
* | | \\ \\ /\\ / / _ \\ '_ \\| __| | | | Auto-generated file
* | | \\ V V / __/ | | | |_| |_| | Any edits to this will be overridden
* |_| \\_/\\_/ \\___|_| |_|\\__|\\__, |
* |___/
*/
`;
const formattedContent = prettierFormat(
`${header}\n${content}\n`,
'typescript',
);
fs.writeFileSync(
path.join(filePath, `${filename}.ts`),
formattedContent,
'utf-8',
);
};
const getLastPathFolder = (path: string) => path.split('/').pop();
const getSubDirectoryPaths = (directoryPath: string): string[] =>
fs
.readdirSync(directoryPath)
.filter((fileOrDirectoryName) => {
const isDirectory = fs
.statSync(path.join(directoryPath, fileOrDirectoryName))
.isDirectory();
if (!isDirectory) {
return false;
}
const isExcludedDirectory =
EXCLUDED_DIRECTORIES.includes(fileOrDirectoryName);
return !isExcludedDirectory;
})
.map((subDirectoryName) => path.join(directoryPath, subDirectoryName));
const getDirectoryPathsRecursive = (directoryPath: string): string[] => [
directoryPath,
...getSubDirectoryPaths(directoryPath).flatMap(getDirectoryPathsRecursive),
];
const getFilesPaths = (directoryPath: string): string[] =>
fs.readdirSync(directoryPath).filter((filePath) => {
const isFile = fs.statSync(path.join(directoryPath, filePath)).isFile();
if (!isFile) {
return false;
}
const isIndexFile = filePath.startsWith(INDEX_FILENAME);
if (isIndexFile) {
return false;
}
const isWhiteListedExtension = INCLUDED_EXTENSIONS.some((extension) =>
filePath.endsWith(extension),
);
const isExcludedExtension = EXCLUDED_EXTENSIONS.every(
(excludedExtension) => !filePath.endsWith(excludedExtension),
);
return isWhiteListedExtension && isExcludedExtension;
});
type ComputeExportLineForGivenFileArgs = {
filePath: string;
moduleDirectory: string; // Rename
directoryPath: string; // Rename
};
const computeExportLineForGivenFile = ({
filePath,
moduleDirectory,
directoryPath,
}: ComputeExportLineForGivenFileArgs) => {
const fileNameWithoutExtension = filePath.split('.').slice(0, -1).join('.');
const pathToImport = slash(
path.relative(
moduleDirectory,
path.join(directoryPath, fileNameWithoutExtension),
),
);
// TODO refactor should extract all exports atomically please refer to https://github.com/twentyhq/core-team-issues/issues/644
return `export * from './${pathToImport}';`;
};
const generateModuleIndexFiles = (moduleDirectories: string[]) => {
return moduleDirectories.map<createTypeScriptFileArgs>((moduleDirectory) => {
const directoryPaths = getDirectoryPathsRecursive(moduleDirectory);
const content = directoryPaths
.flatMap((directoryPath) => {
const directFilesPaths = getFilesPaths(directoryPath);
return directFilesPaths.map((filePath) =>
computeExportLineForGivenFile({
directoryPath,
filePath,
moduleDirectory: moduleDirectory,
}),
);
})
.sort((a, b) => a.localeCompare(b)) // Could be removed as using prettier afterwards anw ?
.join('\n');
return {
content,
path: moduleDirectory,
filename: INDEX_FILENAME,
};
});
};
type JsonUpdate = Record<string, any>;
type WriteInJsonFileArgs = {
content: JsonUpdate;
file: string;
};
const updateJsonFile = ({ content, file }: WriteInJsonFileArgs) => {
const updatedJsonFile = JSON.stringify(content);
const formattedContent = prettierFormat(updatedJsonFile, 'json-stringify');
fs.writeFileSync(file, formattedContent, 'utf-8');
};
const writeInPackageJson = (update: JsonUpdate) => {
const rawJsonFile = fs.readFileSync(PACKAGE_JSON_PATH, 'utf-8');
const initialJsonFile = JSON.parse(rawJsonFile);
updateJsonFile({
file: PACKAGE_JSON_PATH,
content: {
...initialJsonFile,
...update,
},
});
};
const updateNxProjectConfigurationBuildOutputs = (outputs: JsonUpdate) => {
const rawJsonFile = fs.readFileSync(NX_PROJECT_CONFIGURATION_PATH, 'utf-8');
const initialJsonFile = JSON.parse(rawJsonFile);
updateJsonFile({
file: NX_PROJECT_CONFIGURATION_PATH,
content: {
...initialJsonFile,
targets: {
...initialJsonFile.targets,
build: {
...initialJsonFile.targets.build,
outputs,
},
},
},
});
};
const computePackageJsonFilesAndPreconstructConfig = (
moduleDirectories: string[],
) => {
const entrypoints = [...moduleDirectories.map(getLastPathFolder)];
return {
preconstruct: {
entrypoints: [
'./index.ts',
...entrypoints.map((module) => `./${module}/index.ts`),
],
},
files: ['dist', ...entrypoints],
};
};
const computeProjectNxBuildOutputsPath = (moduleDirectories: string[]) => {
const dynamicOutputsPath = moduleDirectories
.map(getLastPathFolder)
.flatMap((barrelName) =>
['package.json', 'dist'].map(
(subPath) => `{projectRoot}/${barrelName}/${subPath}`,
),
);
return ['{projectRoot}/dist', ...dynamicOutputsPath];
};
const main = () => {
const moduleDirectories = getSubDirectoryPaths(SRC_PATH);
const moduleIndexFiles = generateModuleIndexFiles(moduleDirectories);
const packageJsonPreconstructConfigAndFiles =
computePackageJsonFilesAndPreconstructConfig(moduleDirectories);
const nxBuildOutputsPath =
computeProjectNxBuildOutputsPath(moduleDirectories);
updateNxProjectConfigurationBuildOutputs(
nxBuildOutputsPath
);
writeInPackageJson(packageJsonPreconstructConfigAndFiles);
moduleIndexFiles.forEach(createTypeScriptFile);
};
main();

View File

@ -0,0 +1,495 @@
import prettier from '@prettier/sync';
import * as fs from 'fs';
import glob from 'glob';
import * as path from 'path';
import ts from 'typescript';
const prettierConfigFile = prettier.resolveConfigFile();
if (prettierConfigFile == null) {
throw new Error('Prettier config file not found');
}
const prettierConfiguration = prettier.resolveConfig(prettierConfigFile);
type DeclarationOccurence = { kind: string; name: string };
type ExtractedExports = Array<{
file: string;
exports: DeclarationOccurence[];
}>;
type ExtractedImports = Array<{ file: string; imports: string[] }>;
type ExportPerModule = Array<{
moduleName: string;
exports: ExtractedExports[number]['exports'];
}>;
function findAllExports(directoryPath: string): ExtractedExports {
const results: ExtractedExports = [];
const files = getTypeScriptFiles(directoryPath);
for (const file of files) {
const sourceFile = ts.createSourceFile(
file,
fs.readFileSync(file, 'utf8'),
ts.ScriptTarget.Latest,
true,
);
const exports = extractExports(sourceFile);
if (exports.length > 0) {
results.push({
file,
exports,
});
}
}
return results;
}
function findAllImports(directoryPath: string): ExtractedImports {
const results: ExtractedImports = [];
const includeIndex = true;
const files = getTypeScriptFiles(directoryPath, includeIndex);
for (const file of files) {
try {
const sourceFile = ts.createSourceFile(
file,
fs.readFileSync(file, 'utf8'),
ts.ScriptTarget.Latest,
true,
);
const imports = extractImports(sourceFile);
if (imports.length > 0) {
results.push({
file,
imports,
});
}
} catch (e) {
console.log(e);
console.log('Because of file: ', file);
throw e;
}
}
return results;
}
function getTypeScriptFiles(
directoryPath: string,
includeIndex: boolean = false,
): string[] {
const pattern = path.join(directoryPath, '**/*.{ts,tsx}');
const files = glob.sync(pattern);
return files.filter(
(file) =>
!file.endsWith('.d.ts') &&
(includeIndex ? true : !file.endsWith('index.ts')),
);
}
const getKind = (node: ts.VariableStatement) => {
const isConst = (node.declarationList.flags & ts.NodeFlags.Const) !== 0;
if (isConst) {
return 'const';
}
const isLet = (node.declarationList.flags & ts.NodeFlags.Let) !== 0;
if (isLet) {
return 'let';
}
return 'var';
};
function extractExports(sourceFile: ts.SourceFile) {
const exports: DeclarationOccurence[] = [];
function visit(node: ts.Node) {
if (!ts.canHaveModifiers(node)) {
return ts.forEachChild(node, visit);
}
const modifiers = ts.getModifiers(node);
const isExport = modifiers?.some(
(mod) => mod.kind === ts.SyntaxKind.ExportKeyword,
);
if (!isExport) {
return ts.forEachChild(node, visit);
}
switch (true) {
case ts.isTypeAliasDeclaration(node):
exports.push({
kind: 'type',
name: node.name.text,
});
break;
case ts.isInterfaceDeclaration(node):
exports.push({
kind: 'interface',
name: node.name.text,
});
break;
case ts.isEnumDeclaration(node):
exports.push({
kind: 'enum',
name: node.name.text,
});
break;
case ts.isFunctionDeclaration(node) && node.name !== undefined:
exports.push({
kind: 'function',
name: node.name.text,
});
break;
case ts.isVariableStatement(node):
node.declarationList.declarations.forEach((decl) => {
if (ts.isIdentifier(decl.name)) {
const kind = getKind(node);
exports.push({
kind,
name: decl.name.text,
});
}
});
break;
case ts.isClassDeclaration(node) && node.name !== undefined:
exports.push({
kind: 'class',
name: node.name.text,
});
break;
}
return ts.forEachChild(node, visit);
}
visit(sourceFile);
return exports;
}
function extractImports(sourceFile: ts.SourceFile): string[] {
const imports: string[] = [];
function visit(node: ts.Node) {
if (!ts.isImportDeclaration(node)) {
return ts.forEachChild(node, visit);
}
const modulePath = node.moduleSpecifier.getText(sourceFile);
// Quite static
if (modulePath !== `'twenty-shared'`) {
return ts.forEachChild(node, visit);
}
if (!node.importClause) {
return ts.forEachChild(node, visit);
}
if (!node.importClause.namedBindings) {
return ts.forEachChild(node, visit);
}
if (ts.isNamedImports(node.importClause.namedBindings)) {
const namedImports = node.importClause.namedBindings.elements.map(
(element) => {
if (element.propertyName) {
return `${element.propertyName.text} as ${element.name.text}`;
}
return element.name.text;
},
);
// imports.push(`import { ${namedImports} } from ${modulePath}`);
namedImports.forEach((namedImport) => {
imports.push(namedImport);
});
}
ts.forEachChild(node, visit);
}
visit(sourceFile);
return imports;
}
const getSubDirectoryPaths = (directoryPath: string): string[] =>
fs
.readdirSync(directoryPath)
.filter((fileOrDirectoryName) => {
const isDirectory = fs
.statSync(path.join(directoryPath, fileOrDirectoryName))
.isDirectory();
return isDirectory;
})
.map((subDirectoryName) => path.join(directoryPath, subDirectoryName));
const retrievePackageExportsPerModule = (srcPath: string) => {
const subdirectories = getSubDirectoryPaths(srcPath);
return subdirectories.map<ExportPerModule[number]>((moduleDirectory) => {
const moduleExportsPerFile = findAllExports(moduleDirectory);
const moduleName = moduleDirectory.split('/').pop();
if (!moduleName) {
throw new Error(
`Should never occurs moduleName not found ${moduleDirectory}`,
);
}
const flattenExports = Object.values(moduleExportsPerFile).flatMap(
(arr) => arr.exports,
);
return {
moduleName,
exports: flattenExports,
};
});
};
type NewImport = { barrel: string; modules: string[] };
type MappedResolution = {
newImports: Record<string, NewImport>;
file: string;
};
type MapSourceImportToBarrelArgs = {
importsPerFile: ExtractedImports;
exportsPerModule: ExportPerModule;
};
const mapSourceImportToBarrel = ({
exportsPerModule,
importsPerFile,
}: MapSourceImportToBarrelArgs): MappedResolution[] => {
const mappedResolution: MappedResolution[] = [];
for (const fileImport of importsPerFile) {
const { file, imports } = fileImport;
let result: MappedResolution = {
file,
newImports: {},
};
for (const importedDeclaration of imports) {
const findResult = exportsPerModule.find(({ exports }) =>
exports.some((el) => el.name === importedDeclaration),
);
if (findResult === undefined) {
throw new Error(
`Should never occurs no barrel exports ${importedDeclaration}`,
);
}
const { moduleName } = findResult;
if (result.newImports[moduleName]) {
result.newImports[moduleName].modules.push(importedDeclaration);
} else {
result.newImports[moduleName] = {
barrel: moduleName,
modules: [importedDeclaration],
};
}
}
mappedResolution.push(result);
}
return mappedResolution;
};
const retrieveImportFromPackageInSource = (srcPath: string) => {
return findAllImports(srcPath);
};
/**
* Inserts a new import statement at the top of a TypeScript file
* @param filePath Path to the TypeScript file
* @param importSpecifier The module to import from (e.g., 'twenty-shared/utils')
* @param namedImports Array of named imports (e.g., ['useQuery', 'useMutation'])
*/
type InsertImportAtTopArgs = {
filePath: string;
importSpecifier: string;
namedImports: string[];
};
function insertImportAtTop({
filePath,
importSpecifier,
namedImports,
}: InsertImportAtTopArgs): void {
// Read the file content
const sourceText = fs.readFileSync(filePath, 'utf8');
// Create a source file
const sourceFile = ts.createSourceFile(
filePath,
sourceText,
ts.ScriptTarget.Latest,
true,
);
// Build the new import statement
let newImport = `import { ${namedImports.join(', ')} } from '${importSpecifier}';\n`;
// Find the position to insert the import
let insertPos = 0;
// Case 1: File has imports - insert after the last import
let lastImportEnd = 0;
ts.forEachChild(sourceFile, (node) => {
if (
ts.isImportDeclaration(node) ||
ts.isImportEqualsDeclaration(node) ||
(ts.isExpressionStatement(node) &&
ts.isCallExpression(node.expression) &&
node.expression.expression.kind === ts.SyntaxKind.ImportKeyword) // Overkill ?
) {
const end = node.getEnd();
if (end > lastImportEnd) {
lastImportEnd = end;
}
}
});
if (lastImportEnd > 0) {
// Insert after the last import with a newline
insertPos = lastImportEnd;
// Check if there's already a newline after the last import
if (sourceText[insertPos] !== '\n') {
newImport = '\n' + newImport;
}
}
// Insert the new import
const updatedSourceText =
sourceText.substring(0, insertPos) +
newImport +
sourceText.substring(insertPos);
// Write back to file
fs.writeFileSync(
filePath,
prettier.format(updatedSourceText, {
parser: 'typescript',
...prettierConfiguration,
}),
'utf8',
);
}
type RemoveSpecificImports = {
filePath: string;
moduleSpecifier: string;
};
function removeSpecificImports({
filePath,
moduleSpecifier,
}: RemoveSpecificImports) {
const sourceText = fs.readFileSync(filePath, 'utf8');
const sourceFile = ts.createSourceFile(
filePath,
sourceText,
ts.ScriptTarget.Latest,
true,
);
type Replacement = {
start: number;
end: number;
newText: string;
};
let replacement: Replacement | undefined;
function visit(node: ts.Node) {
if (ts.isImportDeclaration(node)) {
const importSource = node.moduleSpecifier
.getText(sourceFile)
.replace(/^['"]|['"]$/g, '');
if (importSource === moduleSpecifier && node.importClause) {
replacement = {
start: node.getFullStart(),
end: node.getEnd(),
newText: '',
};
}
}
ts.forEachChild(node, visit);
}
visit(sourceFile);
let updatedSourceText = sourceText;
if (replacement) {
const { end, newText, start } = replacement;
updatedSourceText =
updatedSourceText.substring(0, start) +
newText +
updatedSourceText.substring(end);
fs.writeFileSync(
filePath,
prettier.format(updatedSourceText, {
parser: 'typescript',
...prettierConfiguration,
}),
'utf8',
);
}
}
const migrateImports = (mappedResolutions: MappedResolution[]) => {
for (const { file, newImports } of mappedResolutions) {
for (const { barrel, modules } of Object.values(newImports)) {
// TODO could refactor to avoid double source file and read
removeSpecificImports({
filePath: file,
moduleSpecifier: 'twenty-shared',
});
insertImportAtTop({
filePath: file,
importSpecifier: `twenty-shared/${barrel}`,
namedImports: modules,
});
}
}
};
const main = () => {
const packageSrcPath = 'packages/twenty-shared/src';
const exportsPerModule = retrievePackageExportsPerModule(packageSrcPath);
const packagesToMigrate = [
'twenty-front',
'twenty-ui',
'twenty-server',
'twenty-emails',
'twenty-zapier',
'twenty-chrome-extension',
];
for (const currPackage of packagesToMigrate) {
console.log(`About to run over ${currPackage}`);
const importsPerFile = retrieveImportFromPackageInSource(
`packages/${currPackage}`,
);
const mappedResolutions = mapSourceImportToBarrel({
exportsPerModule,
importsPerFile,
});
migrateImports(mappedResolutions);
console.log(`${currPackage} migrated`);
}
console.log('SUCCESSFULLY COMPLETED');
};
main();