Commit dc72393
Changed files (31)
badges
abc-commit
cafe-commit
dead-commit
reactions
stars
thumbs-down
thumbs-up
src
task
comments
issues
user
badges/abc-commit/abc-commit.ts
@@ -1,4 +1,4 @@
-import { Commit, define, Repo } from '#src'
+import { Commit, define, Repository } from '#src'
export default define({
url: import.meta.url,
@@ -45,7 +45,7 @@ export default define({
},
})
-function link(re: RegExp, repo: Repo, commit: Commit) {
+function link(re: RegExp, repo: Repository, commit: Commit) {
const sha = commit.sha.replace(re, '<strong>$1</strong>')
return `- <a href="https://github.com/${repo.owner.login}/${repo.name}/commit/${commit.sha}">${sha}</a>`
}
badges/cafe-commit/cafe-commit.ts
@@ -1,10 +1,10 @@
-import { Commit, define, plural, Repo } from '#src'
+import { Commit, define, plural, Repository } from '#src'
export default define({
url: import.meta.url,
badges: ['cafe-commit'] as const,
present(data, grant) {
- const commits: { repo: Repo; commit: Commit }[] = []
+ const commits: { repo: Repository; commit: Commit }[] = []
for (const repo of data.repos) {
for (const commit of repo.commits) {
badges/dead-commit/dead-commit.ts
@@ -1,10 +1,10 @@
-import { Commit, define, Repo, plural } from '#src'
+import { Commit, define, Repository, plural } from '#src'
export default define({
url: import.meta.url,
badges: ['dead-commit'] as const,
present(data, grant) {
- const commits: { repo: Repo; commit: Commit }[] = []
+ const commits: { repo: Repository; commit: Commit }[] = []
for (const repo of data.repos) {
for (const commit of repo.commits) {
badges/reactions/reactions.ts
@@ -20,7 +20,7 @@ export default define({
if (x.reactions && x.reactions.length > 0) {
const counts = count(x.reactions)
if (counts.CONFUSED > 10) {
- moreThan10.push({ count: counts.THUMBS_UP, url: x.url })
+ moreThan10.push({ count: counts.CONFUSED, url: x.url })
}
}
}
badges/stars/stars.ts
@@ -1,4 +1,4 @@
-import { define, Repo } from '#src'
+import { define, Repository } from '#src'
export default define({
url: import.meta.url,
@@ -17,7 +17,7 @@ export default define({
let totalStars = 0
for (const repo of repos) {
- totalStars += repo.stargazers_count || 0
+ totalStars += repo.stargazers.totalCount
}
if (totalStars >= 100) {
@@ -58,21 +58,21 @@ export default define({
},
})
-function asc(a: Repo, b: Repo) {
- return (a.stargazers_count || 0) - (b.stargazers_count || 0)
+function asc(a: Repository, b: Repository) {
+ return (a.stargazers.totalCount || 0) - (b.stargazers.totalCount || 0)
}
-function withStars(repo: Repo) {
- return (repo.stargazers_count || 0) > 0
+function withStars(repo: Repository) {
+ return repo.stargazers.totalCount > 0
}
-function text(repos: Repo[], max: number): string {
+function text(repos: Repository[], max: number): string {
const lines: string[] = []
let totalStars = 0
for (const repo of repos) {
- totalStars += repo.stargazers_count || 0
+ totalStars += repo.stargazers.totalCount
lines.push(
- `* <a href="https://github.com/${repo.owner.login}/${repo.name}">${repo.owner.login}/${repo.name}: ★${repo.stargazers_count}</a>`,
+ `* <a href="https://github.com/${repo.owner.login}/${repo.name}">${repo.owner.login}/${repo.name}: ★${repo.stargazers.totalCount}</a>`,
)
if (totalStars >= max) break
}
badges/thumbs-down/thumbs-down.ts
@@ -22,12 +22,12 @@ export default define({
]) {
if (x.reactions && x.reactions.length > 0) {
const counts = count(x.reactions)
- if (counts.THUMBS_DOWN > 10) {
- moreThan10.push({ count: counts.THUMBS_DOWN, url: x.url })
+ if (counts.THUMBS_DOWN > 100) {
+ moreThan100.push({ count: counts.THUMBS_DOWN, url: x.url })
} else if (counts.THUMBS_DOWN > 50) {
moreThan50.push({ count: counts.THUMBS_DOWN, url: x.url })
- } else if (counts.THUMBS_DOWN > 100) {
- moreThan100.push({ count: counts.THUMBS_DOWN, url: x.url })
+ } else if (counts.THUMBS_DOWN > 10) {
+ moreThan10.push({ count: counts.THUMBS_DOWN, url: x.url })
}
}
}
badges/thumbs-up/thumbs-up.ts
@@ -22,12 +22,12 @@ export default define({
]) {
if (x.reactions && x.reactions.length > 0) {
const counts = count(x.reactions)
- if (counts.THUMBS_UP > 10) {
- moreThan10.push({ count: counts.THUMBS_UP, url: x.url })
+ if (counts.THUMBS_UP > 100) {
+ moreThan100.push({ count: counts.THUMBS_UP, url: x.url })
} else if (counts.THUMBS_UP > 50) {
moreThan50.push({ count: counts.THUMBS_UP, url: x.url })
- } else if (counts.THUMBS_UP > 100) {
- moreThan100.push({ count: counts.THUMBS_UP, url: x.url })
+ } else if (counts.THUMBS_UP > 10) {
+ moreThan10.push({ count: counts.THUMBS_UP, url: x.url })
}
}
}
src/task/comments/discussion-comments.ts
@@ -1,20 +1,20 @@
import { task } from '../../task.js'
import { paginate } from '../../utils.js'
-import {
- DiscussionCommentsQuery,
- IssueCommentsQuery,
-} from './comments.graphql.js'
+import { DiscussionCommentsQuery } from './comments.graphql.js'
export default task({
name: 'discussion-comments' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
+ run: async ({ octokit, data, batch }, { username }: { username: string }) => {
const discussionComments = paginate(octokit, DiscussionCommentsQuery, {
login: username,
})
data.discussionComments = []
- let reactionsBatch: string[] = []
+ const batchReactions = batch(
+ 'reactions-discussion-comments',
+ 'reactions-batch',
+ )
for await (const resp of discussionComments) {
if (!resp.user?.repositoryDiscussionComments.nodes) {
@@ -23,31 +23,11 @@ export default task({
for (const comment of resp.user.repositoryDiscussionComments.nodes) {
data.discussionComments.push(comment)
- if (comment.reactionsTotal.totalCount > 0) {
- if (reactionsBatch.length > 100) {
- next('reactions-discussion-comments', {
- id: comment.id,
- })
- } else {
- reactionsBatch.push(comment.id)
- if (reactionsBatch.length === 50) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- reactionsBatch = []
- }
- }
- }
+ batchReactions(comment.reactionsTotal.totalCount, comment.id)
}
console.log(
`| discussion comments ${data.discussionComments.length}/${resp.user.repositoryDiscussionComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
)
}
-
- if (reactionsBatch.length > 0) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- }
},
})
src/task/comments/issue-comments.ts
@@ -4,14 +4,14 @@ import { IssueCommentsQuery } from './comments.graphql.js'
export default task({
name: 'issue-comments' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
+ run: async ({ octokit, data, batch }, { username }: { username: string }) => {
const issueComments = paginate(octokit, IssueCommentsQuery, {
login: username,
})
data.issueComments = []
- let reactionsBatch: string[] = []
+ const batchReactions = batch('reactions-issue-comments', 'reactions-batch')
for await (const resp of issueComments) {
if (!resp.user?.issueComments.nodes) {
@@ -20,32 +20,12 @@ export default task({
for (const comment of resp.user.issueComments.nodes) {
data.issueComments.push(comment)
- if (comment.reactionsTotal.totalCount > 0) {
- if (reactionsBatch.length > 100) {
- next('reactions-issue-comments', {
- id: comment.id,
- })
- } else {
- reactionsBatch.push(comment.id)
- if (reactionsBatch.length === 50) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- reactionsBatch = []
- }
- }
- }
+ batchReactions(comment.reactionsTotal.totalCount, comment.id)
}
console.log(
`| issue comments ${data.issueComments.length}/${resp.user.issueComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
)
}
-
- if (reactionsBatch.length > 0) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- }
},
})
src/task/commits/commits-batch.ts
@@ -0,0 +1,30 @@
+import { task } from '../../task.js'
+import { query } from '../../utils.js'
+import { CommitsBatchQuery } from './commits.graphql.js'
+
+export default task({
+ name: 'commits-batch' as const,
+ run: async ({ octokit, data }, { ids }: { ids: string[] }) => {
+ const resp = await query(octokit, CommitsBatchQuery, {
+ ids,
+ author: data.user.id,
+ })
+
+ console.log(
+ `| commits batch ${resp.nodes.length} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+ )
+
+ for (const node of resp.nodes) {
+ if (!node) {
+ throw new Error('Failed to load commits')
+ }
+
+ const repo = data.repos.find((repo) => repo.id == node.id)
+ if (!repo) {
+ throw new Error(`Repo not found: ${node.id}`)
+ }
+
+ repo.commits = node.defaultBranchRef?.target?.history.nodes ?? []
+ }
+ },
+})
src/task/commits/commits.graphql
@@ -1,4 +1,5 @@
fragment Commit on Commit {
+ id
sha: oid
committedDate
message
@@ -23,31 +24,47 @@ fragment Commit on Commit {
}
}
-query CommitsQuery(
- $owner: String!
- $name: String!
- $author: ID!
- $num: Int = 100
- $cursor: String
-) {
- repository(owner: $owner, name: $name) {
- defaultBranchRef {
- target {
- ... on Commit {
- history(first: $num, after: $cursor, author: { id: $author }) {
- totalCount
- nodes {
- ...Commit
- }
- pageInfo {
- hasNextPage
- endCursor
- }
+fragment History on Repository {
+ defaultBranchRef {
+ target {
+ ... on Commit {
+ history(first: $num, after: $cursor, author: { id: $author }) {
+ totalCount
+ nodes {
+ ...Commit
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
}
}
}
}
}
+}
+
+query CommitsQuery($id: ID!, $author: ID!, $num: Int = 100, $cursor: String) {
+ node(id: $id) {
+ ...History
+ }
+ rateLimit {
+ limit
+ cost
+ remaining
+ resetAt
+ }
+}
+
+query CommitsBatchQuery(
+ $ids: [ID!]!
+ $author: ID!
+ $num: Int = 100
+ $cursor: String
+) {
+ nodes(ids: $ids) {
+ id
+ ...History
+ }
rateLimit {
limit
cost
src/task/commits/commits.graphql.ts
@@ -2,6 +2,7 @@
const Commit = `#graphql
fragment Commit on Commit {
+ id
sha: oid
committedDate
message
@@ -27,6 +28,7 @@ fragment Commit on Commit {
}`
export type Commit = {
+ id: string
sha: string
committedDate: string
message: string
@@ -51,27 +53,51 @@ export type Commit = {
}
}
-export const CommitsQuery = `#graphql
-${Commit}
-query CommitsQuery($owner: String!, $name: String!, $author: ID!, $num: Int = 100, $cursor: String) {
- repository(owner: $owner, name: $name) {
- defaultBranchRef {
- target {
- ... on Commit {
- history(first: $num, after: $cursor, author: {id: $author}) {
- totalCount
- nodes {
- ...Commit
- }
- pageInfo {
- hasNextPage
- endCursor
- }
+const History = `#graphql
+fragment History on Repository {
+ defaultBranchRef {
+ target {
+ ... on Commit {
+ history(first: $num, after: $cursor, author: {id: $author}) {
+ totalCount
+ nodes {
+ ...Commit
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
}
}
}
}
}
+}`
+
+export type History = {
+ defaultBranchRef: {
+ target:
+ | ({} & {
+ history: {
+ totalCount: number
+ nodes: Array<{} & Commit> | null
+ pageInfo: {
+ hasNextPage: boolean
+ endCursor: string | null
+ }
+ }
+ })
+ | null
+ | null
+ } | null
+}
+
+export const CommitsQuery = `#graphql
+${Commit}
+${History}
+query CommitsQuery($id: ID!, $author: ID!, $num: Int = 100, $cursor: String) {
+ node(id: $id) {
+ ...History
+ }
rateLimit {
limit
cost
@@ -81,29 +107,47 @@ query CommitsQuery($owner: String!, $name: String!, $author: ID!, $num: Int = 10
}` as string & CommitsQuery
export type CommitsQuery = (vars: {
- owner: string
- name: string
+ id: string
author: string
num?: number | null
cursor?: string | null
}) => {
- repository: {
- defaultBranchRef: {
- target:
- | ({} & {
- history: {
- totalCount: number
- nodes: Array<{} & Commit> | null
- pageInfo: {
- hasNextPage: boolean
- endCursor: string | null
- }
- }
- })
- | null
- | null
- } | null
+ node: ({} & History) | null
+ rateLimit: {
+ limit: number
+ cost: number
+ remaining: number
+ resetAt: string
} | null
+}
+
+export const CommitsBatchQuery = `#graphql
+${Commit}
+${History}
+query CommitsBatchQuery($ids: [ID!]!, $author: ID!, $num: Int = 100, $cursor: String) {
+ nodes(ids: $ids) {
+ id
+ ...History
+ }
+ rateLimit {
+ limit
+ cost
+ remaining
+ resetAt
+ }
+}` as string & CommitsBatchQuery
+
+export type CommitsBatchQuery = (vars: {
+ ids: string[]
+ author: string
+ num?: number | null
+ cursor?: string | null
+}) => {
+ nodes: Array<
+ {
+ id: string
+ } & History
+ >
rateLimit: {
limit: number
cost: number
src/task/commits/commits.ts
@@ -4,52 +4,31 @@ import { CommitsQuery } from './commits.graphql.js'
export default task({
name: 'commits' as const,
- run: async (
- { octokit, data, next },
- { owner, name }: { owner: string; name: string },
- ) => {
- console.log(`Loading commits for ${owner}/${name}`)
+ run: async ({ octokit, data }, { id }: { id: string }) => {
const commits = paginate(octokit, CommitsQuery, {
- owner: owner,
- name: name,
+ id,
author: data.user.id,
})
- const repo = data.repos.find(
- (repo) => repo.owner.login == owner && repo.name == name,
- )
+ const repo = data.repos.find((repo) => repo.id == id)
if (!repo) {
- throw new Error(`Repo not found: ${owner}/${name}`)
+ throw new Error(`Repo not found: ${id}`)
}
+
repo.commits = []
for await (const resp of commits) {
- const { totalCount, nodes } =
- resp.repository?.defaultBranchRef?.target?.history!
-
- if (!nodes) {
+ if (!resp.node?.defaultBranchRef?.target?.history) {
throw new Error('Failed to load commits')
}
- if (totalCount >= 10_000) {
- console.error(
- `Too many commits for ${owner}/${name}: ${totalCount} commits; My-Badges will skip repos with more than 10k commits.`,
- )
- break
- }
-
- const repo = data.repos.find(
- (repo) => repo.owner.login == owner && repo.name == name,
- )
- if (!repo) {
- throw new Error(`Repo not found: ${owner}/${name}`)
- }
-
- for (const commit of nodes) {
+ for (const commit of resp.node.defaultBranchRef.target.history.nodes ??
+ []) {
repo.commits.push(commit)
}
+
console.log(
- `| commits ${repo.commits.length}/${totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+ `| commits ${repo?.owner.login}/${repo?.name} ${repo.commits.length}/${resp.node.defaultBranchRef.target.history.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
)
}
},
src/task/issues/issues.ts
@@ -4,15 +4,15 @@ import { IssuesQuery } from './issues.graphql.js'
export default task({
name: 'issues' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
+ run: async ({ octokit, data, batch }, { username }: { username: string }) => {
const issues = paginate(octokit, IssuesQuery, {
username,
})
data.issues = []
- let reactionsBatch: string[] = []
- let issueTimelineBatch: string[] = []
+ const batchReactions = batch('reactions-issue', 'reactions-batch')
+ const batchIssueTimeline = batch('issue-timeline', 'issue-timeline-batch')
for await (const resp of issues) {
if (!resp.user?.issues.nodes) {
@@ -20,7 +20,7 @@ export default task({
}
console.log(
- `Loading issues ${data.issues.length + resp.user.issues.nodes.length}/${
+ `| issues ${data.issues.length + resp.user.issues.nodes.length}/${
resp.user.issues.totalCount
} (cost: ${resp.rateLimit?.cost}, remaining: ${
resp.rateLimit?.remaining
@@ -28,50 +28,9 @@ export default task({
)
for (const issue of resp.user.issues.nodes) {
data.issues.push(issue)
- if (issue.reactionsTotal.totalCount > 0) {
- if (reactionsBatch.length > 100) {
- next('reactions-issue', {
- id: issue.id,
- })
- } else {
- reactionsBatch.push(issue.id)
- if (reactionsBatch.length === 50) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- reactionsBatch = []
- }
- }
- }
-
- if (issue.timelineItemsTotal.totalCount > 0) {
- if (issue.timelineItemsTotal.totalCount > 100) {
- next('issue-timeline', {
- id: issue.id,
- })
- } else {
- issueTimelineBatch.push(issue.id)
- if (issueTimelineBatch.length === 50) {
- next('issue-timeline-batch', {
- ids: issueTimelineBatch,
- })
- issueTimelineBatch = []
- }
- }
- }
+ batchReactions(issue.reactionsTotal.totalCount, issue.id)
+ batchIssueTimeline(issue.timelineItemsTotal.totalCount, issue.id)
}
}
-
- if (reactionsBatch.length > 0) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- }
-
- if (issueTimelineBatch.length > 0) {
- next('issue-timeline-batch', {
- ids: issueTimelineBatch,
- })
- }
},
})
src/task/pulls/pulls.graphql
@@ -64,7 +64,7 @@ fragment PullRequest on PullRequest {
}
}
-query PullsQuery($username: String!, $num: Int = 100, $cursor: String) {
+query PullsQuery($username: String!, $num: Int = 30, $cursor: String) {
user(login: $username) {
pullRequests(first: $num, after: $cursor) {
totalCount
src/task/pulls/pulls.graphql.ts
@@ -150,7 +150,7 @@ export type PullRequest = {
export const PullsQuery = `#graphql
${PullRequest}
-query PullsQuery($username: String!, $num: Int = 100, $cursor: String) {
+query PullsQuery($username: String!, $num: Int = 30, $cursor: String) {
user(login: $username) {
pullRequests(first: $num, after: $cursor) {
totalCount
src/task/pulls/pulls.ts
@@ -4,14 +4,14 @@ import { PullsQuery } from './pulls.graphql.js'
export default task({
name: 'pulls' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
+ run: async ({ octokit, data, batch }, { username }: { username: string }) => {
const pulls = paginate(octokit, PullsQuery, {
username,
})
data.pulls = []
- let reactionsBatch: string[] = []
+ const batchReactions = batch('reactions-pull', 'reactions-batch')
for await (const resp of pulls) {
if (!resp.user?.pullRequests.nodes) {
@@ -27,28 +27,8 @@ export default task({
)
for (const pull of resp.user.pullRequests.nodes) {
data.pulls.push(pull)
- if (pull.reactionsTotal.totalCount > 0) {
- if (reactionsBatch.length > 100) {
- next('reactions-pull', {
- id: pull.id,
- })
- } else {
- reactionsBatch.push(pull.id)
- if (reactionsBatch.length === 50) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- reactionsBatch = []
- }
- }
- }
+ batchReactions(pull.reactionsTotal.totalCount, pull.id)
}
}
-
- if (reactionsBatch.length > 0) {
- next('reactions-batch', {
- ids: reactionsBatch,
- })
- }
},
})
src/task/repos/repos.graphql
@@ -0,0 +1,63 @@
+fragment Repository on Repository {
+ id
+ name
+ owner {
+ login
+ }
+ url
+ description
+ createdAt
+ updatedAt
+ languages(first: 10, orderBy: { field: SIZE, direction: DESC }) {
+ totalCount
+ nodes {
+ name
+ }
+ }
+ forks {
+ totalCount
+ }
+ stargazers {
+ totalCount
+ }
+ defaultBranchRef {
+ name
+ target {
+ ... on Commit {
+ history(author: { id: $author }) {
+ totalCount
+ }
+ }
+ }
+ }
+}
+
+query ReposQuery(
+ $login: String!
+ $author: ID!
+ $num: Int = 100
+ $cursor: String
+) {
+ user(login: $login) {
+ repositories(
+ first: $num
+ after: $cursor
+ orderBy: { field: CREATED_AT, direction: DESC }
+ ) {
+ totalCount
+ nodes {
+ ...Repository
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ rateLimit {
+ cost
+ remaining
+ resetAt
+ limit
+ }
+}
src/task/repos/repos.graphql.ts
@@ -0,0 +1,122 @@
+// DO NOT EDIT. This is a generated file. Instead of this file, edit "repos.graphql".
+
+const Repository = `#graphql
+fragment Repository on Repository {
+ id
+ name
+ owner {
+ login
+ }
+ url
+ description
+ createdAt
+ updatedAt
+ languages(first: 10, orderBy: {field: SIZE, direction: DESC}) {
+ totalCount
+ nodes {
+ name
+ }
+ }
+ forks {
+ totalCount
+ }
+ stargazers {
+ totalCount
+ }
+ defaultBranchRef {
+ name
+ target {
+ ... on Commit {
+ history(author: {id: $author}) {
+ totalCount
+ }
+ }
+ }
+ }
+}`
+
+export type Repository = {
+ id: string
+ name: string
+ owner: {
+ login: string
+ }
+ url: string
+ description: string | null
+ createdAt: string
+ updatedAt: string
+ languages: {
+ totalCount: number
+ nodes: Array<{
+ name: string
+ }> | null
+ } | null
+ forks: {
+ totalCount: number
+ }
+ stargazers: {
+ totalCount: number
+ }
+ defaultBranchRef: {
+ name: string
+ target:
+ | ({} & {
+ history: {
+ totalCount: number
+ }
+ })
+ | null
+ | null
+ } | null
+}
+
+export const ReposQuery = `#graphql
+${Repository}
+query ReposQuery($login: String!, $author: ID!, $num: Int = 100, $cursor: String) {
+ user(login: $login) {
+ repositories(
+ first: $num
+ after: $cursor
+ orderBy: {field: CREATED_AT, direction: DESC}
+ ) {
+ totalCount
+ nodes {
+ ...Repository
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ rateLimit {
+ cost
+ remaining
+ resetAt
+ limit
+ }
+}` as string & ReposQuery
+
+export type ReposQuery = (vars: {
+ login: string
+ author: string
+ num?: number | null
+ cursor?: string | null
+}) => {
+ user: {
+ repositories: {
+ totalCount: number
+ nodes: Array<{} & Repository> | null
+ pageInfo: {
+ hasNextPage: boolean
+ endCursor: string | null
+ }
+ }
+ } | null
+ rateLimit: {
+ cost: number
+ remaining: number
+ resetAt: string
+ limit: number
+ } | null
+}
src/task/repos/repos.ts
@@ -1,26 +1,40 @@
import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { ReposQuery } from './repos.graphql.js'
export default task({
name: 'repos' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
- const repos = octokit.paginate.iterator('GET /users/{username}/repos', {
- username,
- type: 'all',
- per_page: 100,
- })
+ run: async (
+ { octokit, data, batch },
+ { username, author }: { username: string; author: string },
+ ) => {
+ const repos = paginate(octokit, ReposQuery, { login: username, author })
data.repos = []
+
+ const batchCommits = batch('commits', 'commits-batch', 8)
+
for await (const resp of repos) {
- for (const repo of resp.data) {
- if (repo.name == '-') {
- continue
+ if (!resp.user?.repositories.nodes) {
+ throw new Error('Failed to load repos')
+ }
+
+ for (const repo of resp.user.repositories.nodes) {
+ data.repos.push({ ...repo, commits: [] })
+
+ const commitCount =
+ repo.defaultBranchRef?.target?.history.totalCount ?? 0
+ if (commitCount >= 10_000) {
+ console.error(
+ `Too many commits for ${repo.owner.login}/${repo.name}: ${commitCount} commits; My-Badges will skip repos with more than 10k commits.`,
+ )
+ } else {
+ batchCommits(commitCount, repo.id)
}
- data.repos.push({
- ...repo,
- commits: [],
- })
- next('commits', { owner: repo.owner.login, name: repo.name })
}
+ console.log(
+ `| repos ${data.repos.length}/${resp.user.repositories.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+ )
}
},
})
src/task/stars/stars.graphql
@@ -19,7 +19,7 @@ fragment StarredRepo on Repository {
}
}
-query StarsQuery($login: String!, $num: Int = 100, $cursor: String) {
+query StarsQuery($login: String!, $num: Int = 50, $cursor: String) {
user(login: $login) {
starredRepositories(first: $num, after: $cursor) {
totalCount
src/task/stars/stars.graphql.ts
@@ -45,7 +45,7 @@ export type StarredRepo = {
export const StarsQuery = `#graphql
${StarredRepo}
-query StarsQuery($login: String!, $num: Int = 100, $cursor: String) {
+query StarsQuery($login: String!, $num: Int = 50, $cursor: String) {
user(login: $login) {
starredRepositories(first: $num, after: $cursor) {
totalCount
src/task/stars/stars.ts
@@ -4,7 +4,7 @@ import { StarsQuery } from './stars.graphql.js'
export default task({
name: 'stars' as const,
- run: async ({ octokit, data, next }, { username }: { username: string }) => {
+ run: async ({ octokit, data }, { username }: { username: string }) => {
const stars = paginate(octokit, StarsQuery, {
login: username,
})
src/task/user/user.ts
@@ -14,5 +14,7 @@ export default task({
}
data.user = user
+
+ next('repos', { username, author: user.id })
},
})
src/task/index.ts
@@ -3,6 +3,7 @@ export default [
await import('./repos/repos.js'),
await import('./pulls/pulls.js'),
await import('./commits/commits.js'),
+ await import('./commits/commits-batch.js'),
await import('./issues/issues.js'),
await import('./issue-timeline/issue-timeline.js'),
await import('./issue-timeline/issue-timeline-batch.js'),
src/batch.ts
@@ -0,0 +1,35 @@
+import { TaskName } from './task.js'
+
+export function createBatcher(next: (taskName: TaskName, params: any) => void) {
+ const batches = new Map<TaskName, string[]>()
+
+ function batch(paginate: TaskName, batch: TaskName, maxPerBatch = 50) {
+ return function (count: number, id: string) {
+ if (count == 0) {
+ return
+ }
+ if (count > 100) {
+ next(paginate, { id })
+ } else {
+ let ids = batches.get(batch) ?? []
+ ids.push(id)
+ if (ids.length >= maxPerBatch) {
+ next(batch, { ids })
+ ids = []
+ }
+ batches.set(batch, ids)
+ }
+ }
+ }
+
+ function flush() {
+ for (const [batch, ids] of batches.entries()) {
+ next(batch, { ids })
+ batches.delete(batch)
+ }
+ }
+
+ return { batch, flush }
+}
+
+export type BatchFn = ReturnType<typeof createBatcher>['batch']
src/data.ts
@@ -1,4 +1,3 @@
-import { Endpoints } from '@octokit/types'
import { User } from './task/user/user.graphql.js'
import { PullRequest as PullRequestType } from './task/pulls/pulls.graphql.js'
import { Issue as IssueType } from './task/issues/issues.graphql.js'
@@ -9,23 +8,23 @@ import {
import { StarredRepo } from './task/stars/stars.graphql.js'
import { Commit } from './task/commits/commits.graphql.js'
import { Reaction } from './task/reactions/reactions.graphql.js'
+import { Repository as RepositoryType } from './task/repos/repos.graphql.js'
// Data is collected by tasks, enriched if needed, and saved to disk.
// Use this data to determine which badges to present to the user.
export type Data = {
user: User
starredRepositories: StarredRepo[]
- repos: Repo[]
+ repos: Repository[]
pulls: PullRequest[]
issues: Issue[]
issueComments: IssueComment[]
discussionComments: DiscussionComment[]
}
-export type Repo =
- Endpoints['GET /users/{username}/repos']['response']['data'][0] & {
- commits: Commit[]
- }
+export type Repository = {
+ commits: Commit[]
+} & RepositoryType
export type Issue = {
closedBy?: string
src/index.ts
@@ -1,5 +1,5 @@
export { define } from './badges.js'
-export { Repo } from './data.js'
+export { Repository } from './data.js'
export { User } from './task/user/user.graphql.js'
export { Issue } from './task/issues/issues.graphql.js'
export { PullRequest } from './task/pulls/pulls.graphql.js'
src/present-badges.ts
@@ -10,6 +10,8 @@ export const presentBadges = <P extends Presenter<List>>(
omitBadges: string[],
compact: boolean,
): Badge[] => {
+ const newlyAddedBadges = new Set<ID>()
+
for (const presenter of presenters) {
const newBadges: Badge[] = []
const grant = badgeCollection(newBadges)
@@ -21,6 +23,11 @@ export const presentBadges = <P extends Presenter<List>>(
continue
}
+ // Add new badges to the list of badges.
+ for (const b of newBadges) {
+ newlyAddedBadges.add(b.id)
+ }
+
// Enhance badges with image URLs.
for (const b of newBadges) {
b.image = `https://my-badges.github.io/my-badges/${b.id}.png`
@@ -71,6 +78,9 @@ export const presentBadges = <P extends Presenter<List>>(
)
}
+ // Filter out old badges, keep only granted badges.
+ userBadges = userBadges.filter(b => newlyAddedBadges.has(b.id))
+
return userBadges
}
src/process-tasks.ts
@@ -4,6 +4,9 @@ import { GraphqlResponseError } from '@octokit/graphql'
import { Data } from './data.js'
import { TaskName } from './task.js'
import allTasks from './task/index.js'
+import { createBatcher } from './batch.js'
+
+const MAX_ATTEMPTS = 3
export async function processTasks(
octokit: Octokit,
@@ -43,7 +46,6 @@ export async function processTasks(
let todo: Todo[] = [
{ taskName: 'user', params: { username }, attempts: 0 },
- { taskName: 'repos', params: { username }, attempts: 0 },
{ taskName: 'pulls', params: { username }, attempts: 0 },
{ taskName: 'issues', params: { username }, attempts: 0 },
{ taskName: 'issue-comments', params: { username }, attempts: 0 },
@@ -81,20 +83,22 @@ export async function processTasks(
const next = (taskName: TaskName, params: any) => {
todo.push({ taskName, params, attempts: 0 })
}
+ const { batch, flush } = createBatcher(next)
console.log(
`==> Running task ${taskName}`,
new URLSearchParams(params).toString(),
+ attempts > 0 ? `(attempt: ${attempts + 1})` : '',
)
try {
- await task.run({ octokit, data, next }, params)
+ await task.run({ octokit, data, next, batch }, params)
} catch (e) {
let retry = true
if (e instanceof GraphqlResponseError) {
retry = e.errors?.some((error) => error.type == 'NOT_FOUND') ?? true
}
- if (attempts >= 3 || !retry) {
+ if (attempts >= MAX_ATTEMPTS || !retry) {
console.error(
`!!! Failed to run task ${taskName}`,
new URLSearchParams(params).toString(),
@@ -105,7 +109,8 @@ export async function processTasks(
console.error(
`!!! Failed to run task ${taskName}`,
new URLSearchParams(params).toString(),
- `, retrying... (attempts: ${attempts + 1})`,
+ `retrying`,
+ `(will try ${MAX_ATTEMPTS - attempts} more times)`,
)
console.error(e)
todo.push({ taskName, params, attempts: attempts + 1 })
@@ -113,6 +118,8 @@ export async function processTasks(
}
console.log(`<== Finished ${taskName} (${todo.length} tasks left)`)
+ flush()
+
fs.writeFileSync(dataPath, JSON.stringify(data, null, 2))
fs.writeFileSync(tasksPath, JSON.stringify(todo, null, 2))
}
src/task.ts
@@ -1,6 +1,7 @@
import { Octokit } from 'octokit'
import { Data } from './data.js'
import allTasks from './task/index.js'
+import { BatchFn } from './batch.js'
export type TaskName = (typeof allTasks)[number]['default']['name']
@@ -8,6 +9,7 @@ type Context = {
octokit: Octokit
data: Data
next: (taskName: TaskName, params: any) => void
+ batch: BatchFn
}
type Task<Name extends string> = {