Migrate url tooling to twenty-shared (#10440)

Migrate and unify URL tooling in twenty-shared.

We now have:
- isValidHostname which follows our own business rules
- a zod schema that can be re-used in different context and leverages is
isValidHostname
- isValidUrl on top of the zod schema
- a getAbsoluteURl and getHostname on top of the zod schema

I have added a LOT of tests to cover all the cases I've found

Also fixes: https://github.com/twentyhq/twenty/issues/10147
This commit is contained in:
Charles Bochet
2025-02-24 18:01:51 +01:00
committed by GitHub
parent d4bdae562f
commit 9046a9ac16
28 changed files with 280 additions and 158 deletions

View File

@ -1,9 +1,8 @@
import { useLinksField } from '@/object-record/record-field/meta-types/hooks/useLinksField';
import { LinksFieldMenuItem } from '@/object-record/record-field/meta-types/input/components/LinksFieldMenuItem';
import { useMemo } from 'react';
import { isDefined } from 'twenty-shared';
import { absoluteUrlSchema, isDefined } from 'twenty-shared';
import { FieldMetadataType } from '~/generated-metadata/graphql';
import { absoluteUrlSchema } from '~/utils/validation-schemas/absoluteUrlSchema';
import { MultiItemFieldInput } from './MultiItemFieldInput';
type LinksFieldInputProps = {

View File

@ -1,7 +1,6 @@
import { z } from 'zod';
import { absoluteUrlSchema } from '~/utils/validation-schemas/absoluteUrlSchema';
import { absoluteUrlSchema } from 'twenty-shared';
import { FieldLinksValue } from '../FieldMetadata';
export const linksSchema = z.object({

View File

@ -1,7 +1,6 @@
import { FieldValidationDefinition } from '@/spreadsheet-import/types';
import { isDefined, isValidUuid } from 'twenty-shared';
import { absoluteUrlSchema, isDefined, isValidUuid } from 'twenty-shared';
import { FieldMetadataType } from '~/generated-metadata/graphql';
import { absoluteUrlSchema } from '~/utils/validation-schemas/absoluteUrlSchema';
export const getSpreadSheetFieldValidationDefinitions = (
type: FieldMetadataType,

View File

@ -7,8 +7,8 @@ import { z } from 'zod';
import { TextInput } from '@/ui/input/components/TextInput';
import { useLingui } from '@lingui/react/macro';
import { isValidHostname } from 'twenty-shared';
import { Button } from 'twenty-ui';
import { isDomain } from '~/utils/is-domain';
const StyledContainer = styled.div`
display: flex;
@ -43,12 +43,15 @@ export const SettingsAccountsBlocklistInput = ({
.trim()
.email(t`Invalid email or domain`)
.or(
z
.string()
.refine(
(value) => value.startsWith('@') && isDomain(value.slice(1)),
t`Invalid email or domain`,
),
z.string().refine(
(value) =>
value.startsWith('@') &&
isValidHostname(value.slice(1), {
allowIp: false,
allowLocalhost: false,
}),
t`Invalid email or domain`,
),
)
.refine(
(value) => !blockedEmailOrDomainList.includes(value),

View File

@ -5,7 +5,7 @@ import { IconChevronRight } from 'twenty-ui';
import { Webhook } from '@/settings/developers/types/webhook/Webhook';
import { TableCell } from '@/ui/layout/table/components/TableCell';
import { TableRow } from '@/ui/layout/table/components/TableRow';
import { getUrlHostname } from '~/utils/url/getUrlHostname';
import { getUrlHostnameOrThrow } from 'twenty-shared';
export const StyledApisFieldTableRow = styled(TableRow)`
grid-template-columns: 1fr 28px;
@ -39,7 +39,7 @@ export const SettingsDevelopersWebhookTableRow = ({
return (
<StyledApisFieldTableRow to={to}>
<StyledUrlTableCell>
{getUrlHostname(fieldItem.targetUrl, { keepPath: true })}
{getUrlHostnameOrThrow(fieldItem.targetUrl)}
</StyledUrlTableCell>
<StyledIconTableCell>
<StyledIconChevronRight

View File

@ -1,18 +1,16 @@
import { useFindOneRecord } from '@/object-record/hooks/useFindOneRecord';
import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
import { useState } from 'react';
import { useUpdateOneRecord } from '@/object-record/hooks/useUpdateOneRecord';
import { useCreateOneRecord } from '@/object-record/hooks/useCreateOneRecord';
import { Webhook } from '@/settings/developers/types/webhook/Webhook';
import { useDebouncedCallback } from 'use-debounce';
import { WebhookOperationType } from '~/pages/settings/developers/webhooks/types/WebhookOperationsType';
import { isDefined } from 'twenty-shared';
import { WEBHOOK_EMPTY_OPERATION } from '~/pages/settings/developers/webhooks/constants/WebhookEmptyOperation';
import { useDeleteOneRecord } from '@/object-record/hooks/useDeleteOneRecord';
import { useFindOneRecord } from '@/object-record/hooks/useFindOneRecord';
import { useUpdateOneRecord } from '@/object-record/hooks/useUpdateOneRecord';
import { Webhook } from '@/settings/developers/types/webhook/Webhook';
import { SettingsPath } from '@/types/SettingsPath';
import { useState } from 'react';
import { getUrlHostnameOrThrow, isDefined, isValidUrl } from 'twenty-shared';
import { useDebouncedCallback } from 'use-debounce';
import { useNavigateSettings } from '~/hooks/useNavigateSettings';
import { isValidUrl } from '~/utils/url/isValidUrl';
import { getUrlHostname } from '~/utils/url/getUrlHostname';
import { WEBHOOK_EMPTY_OPERATION } from '~/pages/settings/developers/webhooks/constants/WebhookEmptyOperation';
import { WebhookOperationType } from '~/pages/settings/developers/webhooks/types/WebhookOperationsType';
type WebhookFormData = {
targetUrl: string;
@ -106,9 +104,7 @@ export const useWebhookUpdateForm = ({
const isTargetUrlValid = isValidUrl(trimmedUrl);
setIsTargetUrlValid(isTargetUrlValid);
if (isTargetUrlValid) {
setTitle(
getUrlHostname(trimmedUrl, { keepPath: true }) || 'New Webhook',
);
setTitle(getUrlHostnameOrThrow(trimmedUrl) || 'New Webhook');
}
}
};
@ -177,9 +173,7 @@ export const useWebhookUpdateForm = ({
operations,
secret: data.secret,
});
setTitle(
getUrlHostname(data.targetUrl, { keepPath: true }) || 'New Webhook',
);
setTitle(getUrlHostnameOrThrow(data.targetUrl) || 'New Webhook');
setLoading(false);
},
});

View File

@ -3,10 +3,12 @@ import { LinkType, RoundedLink, SocialLink } from 'twenty-ui';
import { FieldLinksValue } from '@/object-record/record-field/types/FieldMetadata';
import { ExpandableList } from '@/ui/layout/expandable-list/components/ExpandableList';
import { isDefined } from 'twenty-shared';
import {
getAbsoluteUrlOrThrow,
getUrlHostnameOrThrow,
isDefined,
} from 'twenty-shared';
import { checkUrlType } from '~/utils/checkUrlType';
import { getAbsoluteUrl } from '~/utils/url/getAbsoluteUrl';
import { getUrlHostname } from '~/utils/url/getUrlHostname';
type LinksDisplayProps = {
value?: FieldLinksValue;
@ -26,10 +28,18 @@ export const LinksDisplay = ({ value }: LinksDisplayProps) => {
]
.filter(isDefined)
.map(({ url, label }) => {
const absoluteUrl = getAbsoluteUrl(url);
let absoluteUrl = '';
let hostname = '';
try {
absoluteUrl = getAbsoluteUrlOrThrow(url);
hostname = getUrlHostnameOrThrow(absoluteUrl);
} catch {
absoluteUrl = '';
hostname = '';
}
return {
url: absoluteUrl,
label: label || getUrlHostname(absoluteUrl),
label: label || hostname,
type: checkUrlType(absoluteUrl),
};
}),

View File

@ -1,35 +0,0 @@
import { isDomain } from '~/utils/is-domain';
describe('isDomain', () => {
it(`should return false if null`, () => {
expect(isDomain(null)).toBeFalsy();
});
it(`should return false if undefined`, () => {
expect(isDomain(undefined)).toBeFalsy();
});
it(`should return true if string google`, () => {
expect(isDomain('google')).toBeFalsy();
});
it(`should return true if string google.com`, () => {
expect(isDomain('google.com')).toBeTruthy();
});
it(`should return true if string bbc.co.uk`, () => {
expect(isDomain('bbc.co.uk')).toBeTruthy();
});
it(`should return true if string web.io`, () => {
expect(isDomain('web.io')).toBeTruthy();
});
it(`should return true if string x.com`, () => {
expect(isDomain('x.com')).toBeTruthy();
});
it(`should return true if string 2.com`, () => {
expect(isDomain('2.com')).toBeTruthy();
});
});

View File

@ -1,7 +0,0 @@
import { isDefined } from 'twenty-shared';
export const isDomain = (url: string | undefined | null) =>
isDefined(url) &&
/^((?!-))(xn--)?[a-z0-9][a-z0-9-_]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$/.test(
url,
);

View File

@ -1,25 +0,0 @@
import { getUrlHostname } from '~/utils/url/getUrlHostname';
describe('getUrlHostname', () => {
it("returns the URL's hostname", () => {
expect(getUrlHostname('https://www.example.com')).toBe('example.com');
expect(getUrlHostname('http://subdomain.example.com')).toBe(
'subdomain.example.com',
);
expect(getUrlHostname('https://www.example.com/path')).toBe('example.com');
expect(getUrlHostname('https://www.example.com?query=123')).toBe(
'example.com',
);
expect(getUrlHostname('http://localhost:3000')).toBe('localhost');
expect(getUrlHostname('example.com')).toBe('example.com');
expect(getUrlHostname('www.subdomain.example.com')).toBe(
'subdomain.example.com',
);
});
it('returns an empty string for invalid URLs', () => {
expect(getUrlHostname('?o')).toBe('');
expect(getUrlHostname('')).toBe('');
expect(getUrlHostname('\\')).toBe('');
});
});

View File

@ -1,9 +0,0 @@
import { absoluteUrlSchema } from '~/utils/validation-schemas/absoluteUrlSchema';
export const getAbsoluteUrl = (url: string) => {
try {
return absoluteUrlSchema.parse(url);
} catch {
return '';
}
};

View File

@ -1,13 +0,0 @@
import { getAbsoluteUrl } from '~/utils/url/getAbsoluteUrl';
export const getUrlHostname = (
url: string,
options?: { keepPath?: boolean },
) => {
try {
const parsedUrl = new URL(getAbsoluteUrl(url));
return `${parsedUrl.hostname.replace(/^www\./i, '')}${options?.keepPath && parsedUrl.pathname !== '/' ? parsedUrl.pathname : ''}`;
} catch {
return '';
}
};

View File

@ -1,8 +0,0 @@
export const isValidUrl = (url: string) => {
const urlRegex =
/^(https?:\/\/)?((([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,})|(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(localhost))(:\d+)?(\/[^\s]*)?(\?[^\s]*)?$/;
const urlPattern = new RegExp(urlRegex, 'i');
return !!urlPattern.test(url);
};

View File

@ -1,23 +0,0 @@
import { z } from 'zod';
export const absoluteUrlSchema = z
.string()
.url()
.or(
z
.string()
.transform((value) => {
try {
const url = `https://${value}`.trim();
return isNaN(Number(value.trim())) &&
new URL(url) &&
/\.[a-z]{2,}$/.test(url)
? url
: '';
} catch {
return '';
}
})
.pipe(z.string().url()),
)
.or(z.literal(''));

View File

@ -4,6 +4,7 @@ import { JestConfigWithTsJest, pathsToModuleNameMapper } from 'ts-jest';
const tsConfig = require('./tsconfig.json');
const jestConfig: JestConfigWithTsJest = {
silent: true,
displayName: 'twenty-ui',
preset: '../../jest.preset.js',
testEnvironment: 'jsdom',

View File

@ -3,4 +3,3 @@ export * from './i18n';
export * from './types';
export * from './utils';
export * from './workspace';

View File

@ -1,4 +1,5 @@
export * from './fieldMetadata';
export * from './image';
export * from './strings';
export * from './url';
export * from './validation';

View File

@ -1,4 +1,4 @@
import { absoluteUrlSchema } from '~/utils/validation-schemas/absoluteUrlSchema';
import { absoluteUrlSchema } from '../absoluteUrlSchema';
describe('absoluteUrlSchema', () => {
it('validates an absolute url', () => {
@ -27,6 +27,7 @@ describe('absoluteUrlSchema', () => {
});
it('fails for invalid urls', () => {
expect(absoluteUrlSchema.safeParse('https://2').success).toBe(false);
expect(absoluteUrlSchema.safeParse('?o').success).toBe(false);
expect(absoluteUrlSchema.safeParse('\\').success).toBe(false);
});

View File

@ -0,0 +1,16 @@
import { getAbsoluteUrlOrThrow } from 'src/utils/url/getAbsoluteUrlOrThrow';
describe('getAbsoluteUrlOrThrow', () => {
it("returns the URL's hostname", () => {
expect(getAbsoluteUrlOrThrow('https://www.example.com')).toBe(
'https://www.example.com',
);
});
it('returns an empty string for invalid URLs', () => {
expect(() => getAbsoluteUrlOrThrow('?o')).toThrow('Invalid URL');
expect(() => getAbsoluteUrlOrThrow('')).toThrow('Invalid URL');
expect(() => getAbsoluteUrlOrThrow('\\')).toThrow('Invalid URL');
expect(() => getAbsoluteUrlOrThrow('2')).toThrow('Invalid URL');
});
});

View File

@ -0,0 +1,30 @@
import { getUrlHostnameOrThrow } from 'src/utils/url/getUrlHostnameOrThrow';
describe('getUrlHostnameOrThrow', () => {
it("returns the URL's hostname", () => {
expect(getUrlHostnameOrThrow('https://www.example.com')).toBe(
'www.example.com',
);
expect(getUrlHostnameOrThrow('http://subdomain.example.com')).toBe(
'subdomain.example.com',
);
expect(getUrlHostnameOrThrow('https://www.example.com/path')).toBe(
'www.example.com',
);
expect(getUrlHostnameOrThrow('https://www.example.com?query=123')).toBe(
'www.example.com',
);
expect(getUrlHostnameOrThrow('http://localhost:3000')).toBe('localhost');
expect(getUrlHostnameOrThrow('example.com')).toBe('example.com');
expect(getUrlHostnameOrThrow('www.subdomain.example.com')).toBe(
'www.subdomain.example.com',
);
});
it('returns an empty string for invalid URLs', () => {
expect(() => getUrlHostnameOrThrow('?o')).toThrow('Invalid URL');
expect(() => getUrlHostnameOrThrow('')).toThrow('Invalid URL');
expect(() => getUrlHostnameOrThrow('\\')).toThrow('Invalid URL');
expect(() => getUrlHostnameOrThrow('2')).toThrow('Invalid URL');
});
});

View File

@ -0,0 +1,71 @@
import { isValidHostname } from 'src/utils/url/isValidHostname';
describe('isValidHostname', () => {
it(`should return true if string google`, () => {
expect(isValidHostname('google')).toBeFalsy();
});
it(`should return true if string google.com`, () => {
expect(isValidHostname('google.com')).toBeTruthy();
});
it(`should return true if string bbc.co.uk`, () => {
expect(isValidHostname('bbc.co.uk')).toBeTruthy();
});
it(`should return true if string www.subdomain.example.com`, () => {
expect(isValidHostname('www.subdomain.example.com')).toBeTruthy();
});
it(`should return true if string web.io`, () => {
expect(isValidHostname('web.io')).toBeTruthy();
});
it(`should return true if string x.com`, () => {
expect(isValidHostname('x.com')).toBeTruthy();
});
it(`should return true if string 2.com`, () => {
expect(isValidHostname('2.com')).toBeTruthy();
});
it(`should return true if string localhost`, () => {
expect(isValidHostname('localhost')).toBeTruthy();
});
it(`should return true if string 127.0.0.1`, () => {
expect(isValidHostname('127.0.0.1')).toBeTruthy();
});
it(`should return false if string 2`, () => {
expect(isValidHostname('2')).toBeFalsy();
});
it(`should return false if string contains non-valid characters`, () => {
expect(isValidHostname('subdomain.example.com/path')).toBeFalsy();
});
it(`should return false if string is empty`, () => {
expect(isValidHostname('')).toBeFalsy();
});
it(`should return false if string is one word`, () => {
expect(isValidHostname('subdomain')).toBeFalsy();
});
it(`should return true if string is ip address`, () => {
expect(isValidHostname('192.168.2.0')).toBeTruthy();
});
it(`should return false if string is ip address but allowIp is false`, () => {
expect(isValidHostname('192.168.2.0', { allowIp: false })).toBeFalsy();
});
it(`should return true if string is localhost but allowLocalhost is false`, () => {
expect(isValidHostname('localhost', { allowLocalhost: false })).toBeFalsy();
});
it(`should return true if string is localhost but allowLocalhost is true`, () => {
expect(isValidHostname('localhost', { allowLocalhost: true })).toBeTruthy();
});
});

View File

@ -15,11 +15,15 @@ describe('isValidUrl', () => {
expect(isValidUrl('http://localhost:3000')).toBe(true);
expect(isValidUrl('example.com')).toBe(true);
expect(isValidUrl('www.subdomain.example.com')).toBe(true);
expect(isValidUrl('192.168.2.0')).toBe(true);
expect(isValidUrl('3.com')).toBe(true);
// Falsy
expect(isValidUrl('?o')).toBe(false);
expect(isValidUrl('')).toBe(false);
expect(isValidUrl('\\')).toBe(false);
expect(isValidUrl('wwwexamplecom')).toBe(false);
expect(isValidUrl('2/toto')).toBe(false);
expect(isValidUrl('2')).toBe(false);
});
});

View File

@ -0,0 +1,52 @@
import { isValidHostname } from 'src/utils/url/isValidHostname';
import { z } from 'zod';
const getAbsoluteUrl = (value: string): string => {
if (value.startsWith('http://') || value.startsWith('https://')) {
return value;
}
return `https://${value}`;
};
export const absoluteUrlSchema = z.string().transform((value, ctx) => {
const trimmedValue = value.trim();
const absoluteUrl = getAbsoluteUrl(trimmedValue);
const valueWithoutProtocol = absoluteUrl
.replace('https://', '')
.replace('http://', '');
if (/^\d+(?:\/[a-zA-Z]*)?$/.test(valueWithoutProtocol)) {
// if the hostname is a number, it's not a valid url
// if we let URL() parse it, it will throw cast an IP address and we lose the information
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'domain is not a valid url',
});
return z.NEVER;
}
try {
const url = new URL(absoluteUrl);
if (isValidHostname(url.hostname)) {
return absoluteUrl;
}
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'domain is not a valid url',
});
return z.NEVER;
} catch (error) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'domain is not a valid url',
});
return z.NEVER;
}
});

View File

@ -0,0 +1,9 @@
import { absoluteUrlSchema } from 'src/utils/url/absoluteUrlSchema';
export const getAbsoluteUrlOrThrow = (url: string): string => {
try {
return absoluteUrlSchema.parse(url);
} catch {
throw new Error('Invalid URL');
}
};

View File

@ -0,0 +1,16 @@
import { absoluteUrlSchema } from 'src/utils/url/absoluteUrlSchema';
export const getUrlHostnameOrThrow = (url: string): string => {
const result = absoluteUrlSchema.safeParse(url);
if (!result.success) {
throw new Error('Invalid URL');
}
try {
const url = new URL(result.data);
return url.hostname;
} catch {
throw new Error('Invalid URL');
}
};

View File

@ -0,0 +1,5 @@
export * from './absoluteUrlSchema';
export * from './getAbsoluteUrlOrThrow';
export * from './getUrlHostnameOrThrow';
export * from './isValidHostname';
export * from './isValidUrl';

View File

@ -0,0 +1,26 @@
export const isValidHostname = (
url: string,
options?: {
allowLocalhost?: boolean;
allowIp?: boolean;
},
): boolean => {
const allowIp = options?.allowIp ?? true;
const allowLocalhost = options?.allowLocalhost ?? true;
const regex =
/^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.){1,10}(xn--)?([a-z0-9][a-z0-9-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})$/;
const isMatch = regex.test(url);
const isIp = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(url);
const isLocalhost = url === 'localhost' || url === '127.0.0.1';
if (isLocalhost && !allowLocalhost) {
return false;
}
if (isIp && !allowIp) {
return false;
}
return isMatch || isLocalhost || isIp;
};

View File

@ -0,0 +1,7 @@
import { absoluteUrlSchema } from 'src/utils/url/absoluteUrlSchema';
export const isValidUrl = (url: string): boolean => {
const result = absoluteUrlSchema.safeParse(url);
return result.success;
};