Make Github stars dynamic and improve database init (#5000)
I extracted the init database logic into its own file. You can now run it with yarn database:init. Added database entry for GitHub stars. Do you want me to remove the init route or is it used for something else ? --------- Co-authored-by: Ady Beraud <a.beraud96@gmail.com> Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
Ady Beraud
@ -0,0 +1,21 @@
|
||||
import { graphql } from '@octokit/graphql';
|
||||
|
||||
import { Repository } from '@/github-sync/contributors/types';
|
||||
|
||||
export async function fetchAssignableUsers(
|
||||
query: typeof graphql,
|
||||
): Promise<Set<string>> {
|
||||
const { repository } = await query<Repository>(`
|
||||
query {
|
||||
repository(owner: "twentyhq", name: "twenty") {
|
||||
assignableUsers(first: 100) {
|
||||
nodes {
|
||||
login
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`);
|
||||
|
||||
return new Set(repository.assignableUsers.nodes.map((user) => user.login));
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
import { graphql } from '@octokit/graphql';
|
||||
|
||||
import {
|
||||
IssueNode,
|
||||
PullRequestNode,
|
||||
Repository,
|
||||
} from '@/github-sync/contributors/types';
|
||||
|
||||
export async function fetchIssuesPRs(
|
||||
query: typeof graphql,
|
||||
cursor: string | null = null,
|
||||
isIssues = false,
|
||||
accumulatedData: Array<PullRequestNode | IssueNode> = [],
|
||||
): Promise<Array<PullRequestNode | IssueNode>> {
|
||||
const { repository } = await query<Repository>(
|
||||
`
|
||||
query ($cursor: String) {
|
||||
repository(owner: "twentyhq", name: "twenty") {
|
||||
pullRequests(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
|
||||
nodes {
|
||||
id
|
||||
title
|
||||
body
|
||||
url
|
||||
createdAt
|
||||
updatedAt
|
||||
closedAt
|
||||
mergedAt
|
||||
author {
|
||||
resourcePath
|
||||
login
|
||||
avatarUrl(size: 460)
|
||||
url
|
||||
}
|
||||
labels(first: 100) {
|
||||
nodes {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
}
|
||||
}
|
||||
}
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
}
|
||||
issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${!isIssues}) {
|
||||
nodes {
|
||||
id
|
||||
title
|
||||
body
|
||||
url
|
||||
createdAt
|
||||
updatedAt
|
||||
closedAt
|
||||
author {
|
||||
resourcePath
|
||||
login
|
||||
avatarUrl
|
||||
url
|
||||
}
|
||||
labels(first: 100) {
|
||||
nodes {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
}
|
||||
}
|
||||
}
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`,
|
||||
{ cursor },
|
||||
);
|
||||
|
||||
const newAccumulatedData: Array<PullRequestNode | IssueNode> = [
|
||||
...accumulatedData,
|
||||
...(isIssues ? repository.issues.nodes : repository.pullRequests.nodes),
|
||||
];
|
||||
const pageInfo = isIssues
|
||||
? repository.issues.pageInfo
|
||||
: repository.pullRequests.pageInfo;
|
||||
|
||||
if (pageInfo.hasNextPage) {
|
||||
return fetchIssuesPRs(
|
||||
query,
|
||||
pageInfo.endCursor,
|
||||
isIssues,
|
||||
newAccumulatedData,
|
||||
);
|
||||
} else {
|
||||
return newAccumulatedData;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
import { insertMany } from '@/database/database';
|
||||
import {
|
||||
issueLabelModel,
|
||||
issueModel,
|
||||
labelModel,
|
||||
userModel,
|
||||
} from '@/database/model';
|
||||
import { IssueNode } from '@/github-sync/contributors/types';
|
||||
|
||||
export async function saveIssuesToDB(
|
||||
issues: Array<IssueNode>,
|
||||
assignableUsers: Set<string>,
|
||||
) {
|
||||
for (const issue of issues) {
|
||||
if (issue.author == null) {
|
||||
continue;
|
||||
}
|
||||
await insertMany(
|
||||
userModel,
|
||||
[
|
||||
{
|
||||
id: issue.author.login,
|
||||
avatarUrl: issue.author.avatarUrl,
|
||||
url: issue.author.url,
|
||||
isEmployee: assignableUsers.has(issue.author.login) ? '1' : '0',
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id' },
|
||||
);
|
||||
|
||||
await insertMany(
|
||||
issueModel,
|
||||
[
|
||||
{
|
||||
id: issue.id,
|
||||
title: issue.title,
|
||||
body: issue.body,
|
||||
url: issue.url,
|
||||
createdAt: issue.createdAt,
|
||||
updatedAt: issue.updatedAt,
|
||||
closedAt: issue.closedAt,
|
||||
authorId: issue.author.login,
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id' },
|
||||
);
|
||||
|
||||
for (const label of issue.labels.nodes) {
|
||||
await insertMany(
|
||||
labelModel,
|
||||
[
|
||||
{
|
||||
id: label.id,
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
description: label.description,
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id' },
|
||||
);
|
||||
await insertMany(issueLabelModel, [
|
||||
{
|
||||
pullRequestId: issue.id,
|
||||
labelId: label.id,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
import { insertMany } from '@/database/database';
|
||||
import {
|
||||
labelModel,
|
||||
pullRequestLabelModel,
|
||||
pullRequestModel,
|
||||
userModel,
|
||||
} from '@/database/model';
|
||||
import { PullRequestNode } from '@/github-sync/contributors/types';
|
||||
|
||||
export async function savePRsToDB(
|
||||
prs: Array<PullRequestNode>,
|
||||
assignableUsers: Set<string>,
|
||||
) {
|
||||
for (const pr of prs) {
|
||||
if (pr.author == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await insertMany(
|
||||
userModel,
|
||||
[
|
||||
{
|
||||
id: pr.author.login,
|
||||
avatarUrl: pr.author.avatarUrl,
|
||||
url: pr.author.url,
|
||||
isEmployee: assignableUsers.has(pr.author.login) ? '1' : '0',
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id' },
|
||||
);
|
||||
|
||||
await insertMany(
|
||||
pullRequestModel,
|
||||
[
|
||||
{
|
||||
id: pr.id,
|
||||
title: pr.title,
|
||||
body: pr.body,
|
||||
url: pr.url,
|
||||
createdAt: pr.createdAt,
|
||||
updatedAt: pr.updatedAt,
|
||||
closedAt: pr.closedAt,
|
||||
mergedAt: pr.mergedAt,
|
||||
authorId: pr.author.login,
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id', onConflictUpdateObject: { title: pr.title } },
|
||||
);
|
||||
|
||||
for (const label of pr.labels.nodes) {
|
||||
await insertMany(
|
||||
labelModel,
|
||||
[
|
||||
{
|
||||
id: label.id,
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
description: label.description,
|
||||
},
|
||||
],
|
||||
{ onConflictKey: 'id' },
|
||||
);
|
||||
await insertMany(pullRequestLabelModel, [
|
||||
{
|
||||
pullRequestId: pr.id,
|
||||
labelId: label.id,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,99 @@
|
||||
import { graphql } from '@octokit/graphql';
|
||||
|
||||
import {
|
||||
IssueNode,
|
||||
PullRequestNode,
|
||||
SearchIssuesPRsQuery,
|
||||
} from '@/github-sync/contributors/types';
|
||||
|
||||
export async function searchIssuesPRs(
|
||||
query: typeof graphql,
|
||||
cursor: string | null = null,
|
||||
isIssues = false,
|
||||
accumulatedData: Array<PullRequestNode | IssueNode> = [],
|
||||
): Promise<Array<PullRequestNode | IssueNode>> {
|
||||
const { search } = await query<SearchIssuesPRsQuery>(
|
||||
`
|
||||
query searchPullRequestsAndIssues($cursor: String) {
|
||||
search(query: "repo:twentyhq/twenty ${
|
||||
isIssues ? 'is:issue' : 'is:pr'
|
||||
} updated:>2024-02-27", type: ISSUE, first: 100, after: $cursor) {
|
||||
edges {
|
||||
node {
|
||||
... on PullRequest {
|
||||
id
|
||||
title
|
||||
body
|
||||
url
|
||||
createdAt
|
||||
updatedAt
|
||||
closedAt
|
||||
mergedAt
|
||||
author {
|
||||
resourcePath
|
||||
login
|
||||
avatarUrl(size: 460)
|
||||
url
|
||||
}
|
||||
labels(first: 100) {
|
||||
nodes {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
}
|
||||
}
|
||||
}
|
||||
... on Issue {
|
||||
id
|
||||
title
|
||||
body
|
||||
url
|
||||
createdAt
|
||||
updatedAt
|
||||
closedAt
|
||||
author {
|
||||
resourcePath
|
||||
login
|
||||
avatarUrl
|
||||
url
|
||||
}
|
||||
labels(first: 100) {
|
||||
nodes {
|
||||
id
|
||||
name
|
||||
color
|
||||
description
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
}
|
||||
}
|
||||
`,
|
||||
{
|
||||
cursor,
|
||||
},
|
||||
);
|
||||
const newAccumulatedData: Array<PullRequestNode | IssueNode> = [
|
||||
...accumulatedData,
|
||||
...search.edges.map(({ node }) => node),
|
||||
];
|
||||
const pageInfo = search.pageInfo;
|
||||
|
||||
if (pageInfo.hasNextPage) {
|
||||
return searchIssuesPRs(
|
||||
query,
|
||||
pageInfo.endCursor,
|
||||
isIssues,
|
||||
newAccumulatedData,
|
||||
);
|
||||
} else {
|
||||
return newAccumulatedData;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
export interface LabelNode {
|
||||
id: string;
|
||||
name: string;
|
||||
color: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface AuthorNode {
|
||||
resourcePath: string;
|
||||
login: string;
|
||||
avatarUrl: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface PullRequestNode {
|
||||
id: string;
|
||||
title: string;
|
||||
body: string;
|
||||
url: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
closedAt: string;
|
||||
mergedAt: string;
|
||||
author: AuthorNode;
|
||||
labels: {
|
||||
nodes: LabelNode[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface IssueNode {
|
||||
id: string;
|
||||
title: string;
|
||||
body: string;
|
||||
url: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
closedAt: string;
|
||||
author: AuthorNode;
|
||||
labels: {
|
||||
nodes: LabelNode[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface PageInfo {
|
||||
hasNextPage: boolean;
|
||||
endCursor: string | null;
|
||||
}
|
||||
|
||||
export interface PullRequests {
|
||||
nodes: PullRequestNode[];
|
||||
pageInfo: PageInfo;
|
||||
}
|
||||
|
||||
export interface Issues {
|
||||
nodes: IssueNode[];
|
||||
pageInfo: PageInfo;
|
||||
}
|
||||
|
||||
export interface AssignableUserNode {
|
||||
login: string;
|
||||
}
|
||||
|
||||
export interface AssignableUsers {
|
||||
nodes: AssignableUserNode[];
|
||||
}
|
||||
|
||||
export interface Stargazers {
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
export interface Repository {
|
||||
repository: {
|
||||
pullRequests: PullRequests;
|
||||
issues: Issues;
|
||||
assignableUsers: AssignableUsers;
|
||||
stargazers: Stargazers;
|
||||
};
|
||||
}
|
||||
|
||||
export interface SearchEdgeNode {
|
||||
node: IssueNode | PullRequestNode;
|
||||
}
|
||||
|
||||
export interface SearchEdges {
|
||||
edges: SearchEdgeNode[];
|
||||
pageInfo: PageInfo;
|
||||
}
|
||||
|
||||
export interface SearchIssuesPRsQuery {
|
||||
search: SearchEdges;
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
import { global } from '@apollo/client/utilities/globals';
|
||||
import { graphql } from '@octokit/graphql';
|
||||
|
||||
import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
|
||||
import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs';
|
||||
import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
|
||||
import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
|
||||
import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
|
||||
import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars';
|
||||
|
||||
export const fetchAndSaveGithubData = async () => {
|
||||
if (!global.process.env.GITHUB_TOKEN) {
|
||||
return new Error('No GitHub token provided');
|
||||
}
|
||||
|
||||
console.log('Synching data..');
|
||||
|
||||
const query = graphql.defaults({
|
||||
headers: {
|
||||
Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
|
||||
},
|
||||
});
|
||||
|
||||
await fetchAndSaveGithubStars(query);
|
||||
|
||||
const assignableUsers = await fetchAssignableUsers(query);
|
||||
const fetchedPRs = (await fetchIssuesPRs(
|
||||
query,
|
||||
null,
|
||||
false,
|
||||
[],
|
||||
)) as Array<PullRequestNode>;
|
||||
const fetchedIssues = (await fetchIssuesPRs(
|
||||
query,
|
||||
null,
|
||||
true,
|
||||
[],
|
||||
)) as Array<IssueNode>;
|
||||
|
||||
savePRsToDB(fetchedPRs, assignableUsers);
|
||||
saveIssuesToDB(fetchedIssues, assignableUsers);
|
||||
|
||||
console.log('data synched!');
|
||||
};
|
||||
@ -0,0 +1,27 @@
|
||||
import { graphql } from '@octokit/graphql';
|
||||
|
||||
import { insertMany } from '@/database/database';
|
||||
import { githubStarsModel } from '@/database/model';
|
||||
import { Repository } from '@/github-sync/contributors/types';
|
||||
|
||||
export const fetchAndSaveGithubStars = async (
|
||||
query: typeof graphql,
|
||||
): Promise<void> => {
|
||||
const { repository } = await query<Repository>(`
|
||||
query {
|
||||
repository(owner: "twentyhq", name: "twenty") {
|
||||
stargazers {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
`);
|
||||
|
||||
const numberOfStars = repository.stargazers.totalCount;
|
||||
|
||||
await insertMany(githubStarsModel, [
|
||||
{
|
||||
numberOfStars,
|
||||
},
|
||||
]);
|
||||
};
|
||||
8
packages/twenty-website/src/github-sync/github-sync.ts
Normal file
8
packages/twenty-website/src/github-sync/github-sync.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data';
|
||||
|
||||
export const githubSync = async () => {
|
||||
await fetchAndSaveGithubData();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
githubSync();
|
||||
Reference in New Issue
Block a user