Added GitHub init (#5317)

- Added github:init to allow full import, as opposed to gitHub:sync
which allows partial sync and therefore respecting Github API Limit
quota.

---------

Co-authored-by: Ady Beraud <a.beraud96@gmail.com>

packages/twenty-website/package.json

@@ -8,7 +8,8 @@
     "build": "npx next build",
     "start": "npx next start",
     "lint": "npx next lint",
-    "github:sync": "npx tsx src/github-sync/github-sync.ts",
+    "github:sync": "npx tsx src/github/github-sync.ts --pageLimit 1",
+    "github:init": "npx tsx src/github/github-sync.ts",
     "database:migrate": "npx tsx src/database/migrate-database.ts",
     "database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/drizzle-posgres.config.ts"
   },

packages/twenty-website/src/github-sync/github-sync.ts

@@ -1,8 +0,0 @@
-import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data';
-
-export const githubSync = async () => {
-  await fetchAndSaveGithubData();
-  process.exit(0);
-};
-
-githubSync();

packages/twenty-website/src/github/contributors/fetch-assignable-users.tsx

@@ -1,6 +1,6 @@
 import { graphql } from '@octokit/graphql';
 
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export async function fetchAssignableUsers(
   query: typeof graphql,

packages/twenty-website/src/github/contributors/fetch-issues-prs.tsx

@@ -4,19 +4,24 @@ import {
   IssueNode,
   PullRequestNode,
   Repository,
-} from '@/github-sync/contributors/types';
+} from '@/github/contributors/types';
+
+// TODO: We should implement a true partial sync instead of using pageLimit.
+// Check search-issues-prs.tsx and modify "updated:>2024-02-27" to make it dynamic
 
 export async function fetchIssuesPRs(
   query: typeof graphql,
   cursor: string | null = null,
   isIssues = false,
   accumulatedData: Array<PullRequestNode | IssueNode> = [],
+  pageLimit: number,
+  currentPage = 0,
 ): Promise<Array<PullRequestNode | IssueNode>> {
   const { repository } = await query<Repository>(
     `
       query ($cursor: String) {
         repository(owner: "twentyhq", name: "twenty") {
-          pullRequests(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
+          pullRequests(first: 30, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
             nodes {
               id
               title
@@ -89,12 +94,16 @@ export async function fetchIssuesPRs(
     ? repository.issues.pageInfo
     : repository.pullRequests.pageInfo;
 
-  if (pageInfo.hasNextPage) {
+  const newCurrentPage = currentPage + 1;
+
+  if ((!pageLimit || newCurrentPage < pageLimit) && pageInfo.hasNextPage) {
     return fetchIssuesPRs(
       query,
       pageInfo.endCursor,
       isIssues,
       newAccumulatedData,
+      pageLimit,
+      currentPage + 1,
     );
   } else {
     return newAccumulatedData;

packages/twenty-website/src/github/contributors/save-issues-to-db.tsx

@@ -5,7 +5,7 @@ import {
   labelModel,
   userModel,
 } from '@/database/model';
-import { IssueNode } from '@/github-sync/contributors/types';
+import { IssueNode } from '@/github/contributors/types';
 
 export async function saveIssuesToDB(
   issues: Array<IssueNode>,

packages/twenty-website/src/github/contributors/save-prs-to-db.tsx

@@ -5,7 +5,7 @@ import {
   pullRequestModel,
   userModel,
 } from '@/database/model';
-import { PullRequestNode } from '@/github-sync/contributors/types';
+import { PullRequestNode } from '@/github/contributors/types';
 
 export async function savePRsToDB(
   prs: Array<PullRequestNode>,

packages/twenty-website/src/github/contributors/search-issues-prs.tsx

@@ -4,7 +4,7 @@ import {
   IssueNode,
   PullRequestNode,
   SearchIssuesPRsQuery,
-} from '@/github-sync/contributors/types';
+} from '@/github/contributors/types';
 
 export async function searchIssuesPRs(
   query: typeof graphql,

packages/twenty-website/src/github/fetch-and-save-github-data.ts

@@ -1,15 +1,19 @@
 import { global } from '@apollo/client/utilities/globals';
 import { graphql } from '@octokit/graphql';
 
-import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
+import { fetchAssignableUsers } from '@/github/contributors/fetch-assignable-users';
-import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs';
+import { fetchIssuesPRs } from '@/github/contributors/fetch-issues-prs';
-import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
+import { saveIssuesToDB } from '@/github/contributors/save-issues-to-db';
-import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
+import { savePRsToDB } from '@/github/contributors/save-prs-to-db';
-import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
+import { IssueNode, PullRequestNode } from '@/github/contributors/types';
-import { fetchAndSaveGithubReleases } from '@/github-sync/github-releases/fetch-and-save-github-releases';
+import { fetchAndSaveGithubReleases } from '@/github/github-releases/fetch-and-save-github-releases';
-import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars';
+import { fetchAndSaveGithubStars } from '@/github/github-stars/fetch-and-save-github-stars';
 
-export const fetchAndSaveGithubData = async () => {
+export const fetchAndSaveGithubData = async ({
+  pageLimit,
+}: {
+  pageLimit: number;
+}) => {
   if (!global.process.env.GITHUB_TOKEN) {
     return new Error('No GitHub token provided');
   }
@@ -31,12 +35,14 @@ export const fetchAndSaveGithubData = async () => {
     null,
     false,
     [],
+    pageLimit,
   )) as Array<PullRequestNode>;
   const fetchedIssues = (await fetchIssuesPRs(
     query,
     null,
     true,
     [],
+    pageLimit,
   )) as Array<IssueNode>;
 
   await savePRsToDB(fetchedPRs, assignableUsers);

packages/twenty-website/src/github/github-releases/fetch-and-save-github-releases.tsx

@@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql';
 
 import { insertMany } from '@/database/database';
 import { githubReleasesModel } from '@/database/model';
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export const fetchAndSaveGithubReleases = async (
   query: typeof graphql,

packages/twenty-website/src/github/github-stars/fetch-and-save-github-stars.tsx

@@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql';
 
 import { insertMany } from '@/database/database';
 import { githubStarsModel } from '@/database/model';
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export const fetchAndSaveGithubStars = async (
   query: typeof graphql,

packages/twenty-website/src/github/github-sync.ts

@@ -0,0 +1,15 @@
+import { fetchAndSaveGithubData } from '@/github/fetch-and-save-github-data';
+
+export const githubSync = async () => {
+  const pageLimitFlagIndex = process.argv.indexOf('--pageLimit');
+  let pageLimit = 0;
+
+  if (pageLimitFlagIndex > -1) {
+    pageLimit = parseInt(process.argv[pageLimitFlagIndex + 1], 10);
+  }
+
+  await fetchAndSaveGithubData({ pageLimit });
+  process.exit(0);
+};
+
+githubSync();