Add proper ORM and postgres support (#3978)

* Add postgresql support

* Fixes

* Fix perfs
This commit is contained in:
Charles Bochet
2024-02-14 17:53:50 +01:00
committed by GitHub
parent 94ad0e33ec
commit 4613f64910
24 changed files with 2143 additions and 344 deletions

View File

@ -10,6 +10,9 @@ export const ActivityLog = ({
}: {
data: { value: number; day: string }[];
}) => {
if (!data.length) {
return null;
}
return (
<CardContainer>
<Title>Activity</Title>

View File

@ -90,10 +90,12 @@ export const ProfileCard = ({
<StyledGithubIcon size="M" color="rgba(0,0,0,1)" />
</a>
</h3>
<p className="duration">
Contributing since{' '}
{format(new Date(firstContributionAt), 'MMMM yyyy')}
</p>
{firstContributionAt && (
<p className="duration">
Contributing since{' '}
{format(new Date(firstContributionAt), 'MMMM yyyy')}
</p>
)}
</Details>
</ProfileContainer>
);

View File

@ -1,4 +1,3 @@
import Database from 'better-sqlite3';
import { Metadata } from 'next';
import { Background } from '@/app/components/oss-friends/Background';
@ -9,12 +8,8 @@ import { ProfileCard } from '@/app/developers/contributors/[slug]/components/Pro
import { ProfileInfo } from '@/app/developers/contributors/[slug]/components/ProfileInfo';
import { PullRequests } from '@/app/developers/contributors/[slug]/components/PullRequests';
import { ThankYou } from '@/app/developers/contributors/[slug]/components/ThankYou';
interface Contributor {
login: string;
avatarUrl: string;
pullRequestCount: number;
}
import { findAll } from '@/database/database';
import { pullRequestModel, userModel } from '@/database/model';
export function generateMetadata({
params,
@ -27,134 +22,109 @@ export function generateMetadata({
}
export default async function ({ params }: { params: { slug: string } }) {
const db = new Database('db.sqlite', { readonly: true });
const contributors = await findAll(userModel);
const contributor = db
.prepare(
`
SELECT
u.login,
u.avatarUrl,
(SELECT COUNT(*) FROM pullRequests WHERE authorId = u.id) AS pullRequestCount,
(SELECT COUNT(*) FROM issues WHERE authorId = u.id) AS issuesCount
FROM
users u
WHERE
u.login = :user_id
`,
)
.get({ user_id: params.slug }) as Contributor;
const contributor = contributors.find(
(contributor) => contributor.id === params.slug,
);
const pullRequestActivity = db
.prepare(
`
SELECT
COUNT(*) as value,
DATE(createdAt) as day
FROM
pullRequests
WHERE
authorId = (SELECT id FROM users WHERE login = :user_id)
GROUP BY
DATE(createdAt)
ORDER BY
DATE(createdAt)
`,
)
.all({ user_id: params.slug }) as { value: number; day: string }[];
if (!contributor) {
return;
}
// Latest PRs.
const pullRequestList = db
.prepare(
`
SELECT
id,
title,
body,
url,
createdAt,
updatedAt,
closedAt,
mergedAt,
authorId
FROM
pullRequests
WHERE
authorId = (SELECT id FROM users WHERE login = :user_id)
ORDER BY
DATE(createdAt) DESC
LIMIT
10
`,
)
.all({ user_id: params.slug }) as {
title: string;
createdAt: string;
url: string;
id: string;
mergedAt: string | null;
authorId: string;
}[];
const pullRequests = await findAll(pullRequestModel);
const mergedPullRequests = pullRequests
.filter((pr) => pr.mergedAt !== null)
.filter(
(pr) =>
![
'dependabot',
'cyborch',
'emilienchvt',
'Samox',
'charlesBochet',
'gitstart-app',
'thaisguigon',
'lucasbordeau',
'magrinj',
'Weiko',
'gitstart-twenty',
'bosiraphael',
'martmull',
'FelixMalfait',
'thomtrp',
'Bonapara',
'nimraahmed',
].includes(pr.authorId),
);
const mergedPullRequests = db
.prepare(
`
SELECT * FROM (
SELECT
merged_pr_counts.*,
(RANK() OVER(ORDER BY merged_count) - 1) / CAST( total_authors as float) * 100 as rank_percentage
FROM
(
SELECT
authorId,
COUNT(*) FILTER (WHERE mergedAt IS NOT NULL) as merged_count
FROM
pullRequests pr
JOIN
users u ON pr.authorId = u.id
WHERE
u.isEmployee = FALSE
GROUP BY
authorId
) AS merged_pr_counts
CROSS JOIN
(
SELECT COUNT(DISTINCT authorId) as total_authors
FROM pullRequests pr
JOIN
users u ON pr.authorId = u.id
WHERE
u.isEmployee = FALSE
) AS author_counts
) WHERE authorId = (SELECT id FROM users WHERE login = :user_id)
`,
)
.all({ user_id: params.slug }) as {
merged_count: number;
rank_percentage: number;
}[];
const contributorPullRequests = pullRequests.filter(
(pr) => pr.authorId === contributor.id,
);
const mergedContributorPullRequests = contributorPullRequests.filter(
(pr) => pr.mergedAt !== null,
);
db.close();
const mergedContributorPullRequestsByContributor = mergedPullRequests.reduce(
(acc, pr) => {
acc[pr.authorId] = (acc[pr.authorId] || 0) + 1;
return acc;
},
{},
);
const mergedContributorPullRequestsByContributorArray = Object.entries(
mergedContributorPullRequestsByContributor,
)
.map(([authorId, value]) => ({ authorId, value }))
.sort((a, b) => b.value - a.value);
const contributorRank =
((mergedContributorPullRequestsByContributorArray.findIndex(
(contributor) => contributor.authorId === params.slug,
) +
1) /
contributors.length) *
100;
const pullRequestActivity = contributorPullRequests.reduce((acc, pr) => {
const date = new Date(pr.createdAt).toISOString().split('T')[0];
acc[date] = (acc[date] || 0) + 1;
return acc;
}, []);
const pullRequestActivityArray = Object.entries(pullRequestActivity)
.map(([day, value]) => ({ day, value }))
.sort((a, b) => new Date(a.day).getTime() - new Date(b.day).getTime());
return (
<>
<Background />
<ContentContainer>
<Breadcrumb active={contributor.login} />
<Breadcrumb active={contributor.id} />
<ProfileCard
username={contributor.login}
username={contributor.id}
avatarUrl={contributor.avatarUrl}
firstContributionAt={pullRequestActivity[0].day}
firstContributionAt={pullRequestActivityArray[0]?.day}
/>
<ProfileInfo
mergedPRsCount={mergedPullRequests[0].merged_count}
rank={(100 - Number(mergedPullRequests[0].rank_percentage)).toFixed(
0,
)}
activeDays={pullRequestActivity.length}
mergedPRsCount={mergedContributorPullRequests.length}
rank={Math.ceil(Number(contributorRank)).toFixed(0)}
activeDays={pullRequestActivityArray.length}
/>
<ActivityLog data={pullRequestActivityArray} />
<PullRequests
list={
contributorPullRequests.slice(0, 9) as {
id: string;
title: string;
url: string;
createdAt: string;
mergedAt: string | null;
authorId: string;
}[]
}
/>
<ActivityLog data={pullRequestActivity} />
<PullRequests list={pullRequestList} />
<ThankYou authorId={contributor.login} />
</ContentContainer>
</>

View File

@ -1,23 +0,0 @@
import Database from 'better-sqlite3';
export async function GET(
request: Request,
{ params }: { params: { slug: string } },
) {
const db = new Database('db.sqlite', { readonly: true });
if (
params.slug !== 'users' &&
params.slug !== 'labels' &&
params.slug !== 'pullRequests' &&
params.slug !== 'issues'
) {
return Response.json({ error: 'Invalid table name' }, { status: 400 });
}
const rows = db.prepare('SELECT * FROM ' + params.slug).all();
db.close();
return Response.json(rows);
}

View File

@ -1,7 +1,14 @@
import { graphql } from '@octokit/graphql';
import Database from 'better-sqlite3';
const db = new Database('db.sqlite', { verbose: console.log });
import { insertMany, migrate } from '@/database/database';
import {
issueLabelModel,
issueModel,
labelModel,
pullRequestLabelModel,
pullRequestModel,
userModel,
} from '@/database/model';
interface LabelNode {
id: string;
@ -188,176 +195,132 @@ async function fetchAssignableUsers(): Promise<Set<string>> {
return new Set(repository.assignableUsers.nodes.map((user) => user.login));
}
const initDb = () => {
db.prepare(
`
CREATE TABLE IF NOT EXISTS pullRequests (
id TEXT PRIMARY KEY,
title TEXT,
body TEXT,
url TEXT,
createdAt TEXT,
updatedAt TEXT,
closedAt TEXT,
mergedAt TEXT,
authorId TEXT,
FOREIGN KEY (authorId) REFERENCES users(id)
);
`,
).run();
db.prepare(
`
CREATE TABLE IF NOT EXISTS issues (
id TEXT PRIMARY KEY,
title TEXT,
body TEXT,
url TEXT,
createdAt TEXT,
updatedAt TEXT,
closedAt TEXT,
authorId TEXT,
FOREIGN KEY (authorId) REFERENCES users(id)
);
`,
).run();
db.prepare(
`
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY,
login TEXT,
avatarUrl TEXT,
url TEXT,
isEmployee BOOLEAN
);
`,
).run();
db.prepare(
`
CREATE TABLE IF NOT EXISTS labels (
id TEXT PRIMARY KEY,
name TEXT,
color TEXT,
description TEXT
);
`,
).run();
db.prepare(
`
CREATE TABLE IF NOT EXISTS pullRequestLabels (
pullRequestId TEXT,
labelId TEXT,
FOREIGN KEY (pullRequestId) REFERENCES pullRequests(id),
FOREIGN KEY (labelId) REFERENCES labels(id)
);
`,
).run();
db.prepare(
`
CREATE TABLE IF NOT EXISTS issueLabels (
issueId TEXT,
labelId TEXT,
FOREIGN KEY (issueId) REFERENCES issues(id),
FOREIGN KEY (labelId) REFERENCES labels(id)
);
`,
).run();
};
export async function GET() {
initDb();
await migrate();
// TODO if we ever hit API Rate Limiting
const lastPRCursor = null;
const lastIssueCursor = null;
const assignableUsers = await fetchAssignableUsers();
const prs = (await fetchData(lastPRCursor)) as Array<PullRequestNode>;
const issues = (await fetchData(lastIssueCursor, true)) as Array<IssueNode>;
const fetchedPRs = (await fetchData(lastPRCursor)) as Array<PullRequestNode>;
const fetchedIssues = (await fetchData(
lastIssueCursor,
true,
)) as Array<IssueNode>;
const insertPR = db.prepare(
'INSERT INTO pullRequests (id, title, body, url, createdAt, updatedAt, closedAt, mergedAt, authorId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING',
);
const insertIssue = db.prepare(
'INSERT INTO issues (id, title, body, url, createdAt, updatedAt, closedAt, authorId) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING',
);
const insertUser = db.prepare(
'INSERT INTO users (id, login, avatarUrl, url, isEmployee) VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING',
);
const insertLabel = db.prepare(
'INSERT INTO labels (id, name, color, description) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO NOTHING',
);
const insertPullRequestLabel = db.prepare(
'INSERT INTO pullRequestLabels (pullRequestId, labelId) VALUES (?, ?)',
);
const insertIssueLabel = db.prepare(
'INSERT INTO issueLabels (issueId, labelId) VALUES (?, ?)',
);
for (const pr of prs) {
console.log(pr);
for (const pr of fetchedPRs) {
if (pr.author == null) {
continue;
}
insertUser.run(
pr.author.resourcePath,
pr.author.login,
pr.author.avatarUrl,
pr.author.url,
assignableUsers.has(pr.author.login) ? 1 : 0,
await insertMany(
userModel,
[
{
id: pr.author.login,
avatarUrl: pr.author.avatarUrl,
url: pr.author.url,
isEmployee: assignableUsers.has(pr.author.login) ? '1' : '0',
},
],
{ onConflictKey: 'id' },
);
insertPR.run(
pr.id,
pr.title,
pr.body,
pr.url,
pr.createdAt,
pr.updatedAt,
pr.closedAt,
pr.mergedAt,
pr.author.resourcePath,
await insertMany(
pullRequestModel,
[
{
id: pr.id,
title: pr.title,
body: pr.body,
url: pr.url,
createdAt: pr.createdAt,
updatedAt: pr.updatedAt,
closedAt: pr.closedAt,
mergedAt: pr.mergedAt,
authorId: pr.author.login,
},
],
{ onConflictKey: 'id' },
);
for (const label of pr.labels.nodes) {
insertLabel.run(label.id, label.name, label.color, label.description);
insertPullRequestLabel.run(pr.id, label.id);
await insertMany(
labelModel,
[
{
id: label.id,
name: label.name,
color: label.color,
description: label.description,
},
],
{ onConflictKey: 'id' },
);
await insertMany(pullRequestLabelModel, [
{
pullRequestId: pr.id,
labelId: label.id,
},
]);
}
}
for (const issue of issues) {
for (const issue of fetchedIssues) {
if (issue.author == null) {
continue;
}
insertUser.run(
issue.author.resourcePath,
issue.author.login,
issue.author.avatarUrl,
issue.author.url,
assignableUsers.has(issue.author.login) ? 1 : 0,
await insertMany(
userModel,
[
{
id: issue.author.login,
avatarUrl: issue.author.avatarUrl,
url: issue.author.url,
isEmployee: assignableUsers.has(issue.author.login) ? '1' : '0',
},
],
{ onConflictKey: 'id' },
);
insertIssue.run(
issue.id,
issue.title,
issue.body,
issue.url,
issue.createdAt,
issue.updatedAt,
issue.closedAt,
issue.author.resourcePath,
await insertMany(
issueModel,
[
{
id: issue.id,
title: issue.title,
body: issue.body,
url: issue.url,
createdAt: issue.createdAt,
updatedAt: issue.updatedAt,
closedAt: issue.closedAt,
authorId: issue.author.login,
},
],
{ onConflictKey: 'id' },
);
for (const label of issue.labels.nodes) {
insertLabel.run(label.id, label.name, label.color, label.description);
insertIssueLabel.run(issue.id, label.id);
await insertMany(
labelModel,
[
{
id: label.id,
name: label.name,
color: label.color,
description: label.description,
},
],
{ onConflictKey: 'id' },
);
await insertMany(issueLabelModel, [
{
pullRequestId: issue.id,
labelId: label.id,
},
]);
}
}
db.close();
return new Response('Data synced', { status: 200 });
}

View File

@ -1,41 +1,44 @@
import Database from 'better-sqlite3';
import AvatarGrid from '@/app/components/AvatarGrid';
import { Header } from '@/app/components/developers/contributors/Header';
import { Background } from '@/app/components/oss-friends/Background';
import { ContentContainer } from '@/app/components/oss-friends/ContentContainer';
import { findAll } from '@/database/database';
import { pullRequestModel, userModel } from '@/database/model';
interface Contributor {
login: string;
id: string;
avatarUrl: string;
pullRequestCount: number;
}
const Contributors = async () => {
const db = new Database('db.sqlite', { readonly: true });
const contributors = await findAll(userModel);
const pullRequests = await findAll(pullRequestModel);
const contributors = db
.prepare(
`SELECT
u.login,
u.avatarUrl,
COUNT(pr.id) AS pullRequestCount
FROM
users u
JOIN
pullRequests pr ON u.id = pr.authorId
WHERE
u.isEmployee = FALSE
AND u.login NOT IN ('dependabot', 'cyborch', 'emilienchvt', 'Samox')
GROUP BY
u.id
ORDER BY
pullRequestCount DESC;
`,
const pullRequestByAuthor = pullRequests.reduce((acc, pr) => {
acc[pr.authorId] = acc[pr.authorId] ? acc[pr.authorId] + 1 : 1;
return acc;
}, {});
const fitlerContributors = contributors
.filter((contributor) => contributor.isEmployee === '0')
.filter(
(contributor) =>
![
'dependabot',
'cyborch',
'emilienchvt',
'Samox',
'nimraahmed',
'gitstart-app',
].includes(contributor.id),
)
.all() as Contributor[];
.map((contributor) => {
contributor.pullRequestCount = pullRequestByAuthor[contributor.id] || 0;
db.close();
return contributor;
})
.sort((a, b) => b.pullRequestCount - a.pullRequestCount)
.filter((contributor) => contributor.pullRequestCount > 0);
return (
<>
@ -43,7 +46,7 @@ const Contributors = async () => {
<ContentContainer>
<Header />
<div>
<AvatarGrid users={contributors} />
<AvatarGrid users={fitlerContributors as Contributor[]} />
</div>
</ContentContainer>
</>