Commit 28466bc

Anton Medvedev <anton@medv.io>
2024-11-29 13:44:15
Refactor collect to tasks
1 parent d00892b
badges/old-issue/old-issue.ts
@@ -20,6 +20,7 @@ export default define({
 
     for (const issue of data.issues.sort(age)) {
       if (!issue.closed) continue
+      if (issue.closedBy != data.user.login) continue
       const createdAt = new Date(issue.createdAt)
       const closedAt = new Date(issue.closedAt!)
       let years = Math.floor(
badges/reactions/reactions.ts
@@ -1,5 +1,5 @@
 import { define } from '#src'
-import { Reactions } from '../../src/collect/comments.graphql.js'
+import { Reactions } from '../../src/task/user-comments/comments.graphql.js'
 
 export default define({
   url: import.meta.url,
src/collect/index.ts
@@ -1,22 +0,0 @@
-import { Endpoints } from '@octokit/types'
-import { User } from './user.graphql.js'
-import { PullRequest } from './pulls.graphql.js'
-import { Issue } from './issues.graphql.js'
-import { DiscussionComment, IssueComment } from './comments.graphql.js'
-import { StarredRepo } from './stars.graphql.js'
-import { Commit } from './commits.graphql.js'
-
-export type Data = {
-  user: User
-  starredRepositories: StarredRepo[]
-  repos: Repo[]
-  pulls: PullRequest[]
-  issues: Issue[]
-  issueComments: IssueComment[]
-  discussionComments: DiscussionComment[]
-}
-
-export type Repo =
-  Endpoints['GET /users/{username}/repos']['response']['data'][0] & {
-    commits: Commit[]
-  }
src/collect/comments.graphql → src/task/comments/comments.graphql
File renamed without changes
src/collect/comments.graphql.ts → src/task/comments/comments.graphql.ts
File renamed without changes
src/task/comments/discussion-comments.ts
@@ -0,0 +1,28 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import {
+  DiscussionCommentsQuery,
+  IssueCommentsQuery,
+} from './comments.graphql.js'
+
+export default task({
+  name: 'discussion-comments' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const discussionComments = paginate(octokit, DiscussionCommentsQuery, {
+      login: username,
+    })
+
+    for await (const resp of discussionComments) {
+      if (!resp.user?.repositoryDiscussionComments.nodes) {
+        throw new Error('Failed to load discussion comments')
+      }
+
+      for (const comment of resp.user.repositoryDiscussionComments.nodes) {
+        data.discussionComments.push(comment)
+      }
+      console.log(
+        `| discussion comments ${data.discussionComments.length}/${resp.user.repositoryDiscussionComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+      )
+    }
+  },
+})
src/task/comments/issue-comments.ts
@@ -0,0 +1,25 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { IssueCommentsQuery } from './comments.graphql.js'
+
+export default task({
+  name: 'issue-comments' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const issueComments = paginate(octokit, IssueCommentsQuery, {
+      login: username,
+    })
+
+    for await (const resp of issueComments) {
+      if (!resp.user?.issueComments.nodes) {
+        throw new Error('Failed to load issue comments')
+      }
+
+      for (const comment of resp.user.issueComments.nodes) {
+        data.issueComments.push(comment)
+      }
+      console.log(
+        `| issue comments ${data.issueComments.length}/${resp.user.issueComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+      )
+    }
+  },
+})
src/collect/commits.graphql → src/task/commits/commits.graphql
File renamed without changes
src/collect/commits.graphql.ts → src/task/commits/commits.graphql.ts
File renamed without changes
src/task/commits/commits.ts
@@ -0,0 +1,50 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { CommitsQuery } from './commits.graphql.js'
+
+export default task({
+  name: 'commits' as const,
+  run: async (
+    { octokit, data, next },
+    { owner, name }: { owner: string; name: string },
+  ) => {
+    console.log(`Loading commits for ${owner}/${name}`)
+    const commits = paginate(octokit, CommitsQuery, {
+      owner: owner,
+      name: name,
+      author: data.user.id,
+    })
+
+    for await (const resp of commits) {
+      const { totalCount, nodes } =
+        resp.repository?.defaultBranchRef?.target?.history!
+
+      if (!nodes) {
+        throw new Error('Failed to load commits')
+      }
+
+      if (totalCount >= 10_000) {
+        console.error(
+          `Too many commits for ${owner}/${name}: ${totalCount} commits; My-Badges will skip repos with more than 10k commits.`,
+        )
+        break
+      }
+
+      const repo = data.repos.find(
+        (repo) => repo.owner.login == owner && repo.name == name,
+      )
+      if (!repo) {
+        throw new Error(`Repo not found: ${owner}/${name}`)
+      }
+
+      for (const commit of nodes) {
+        data.repos
+          .find((repo) => repo.owner.login == owner && repo.name == name)
+          ?.commits.push(commit)
+      }
+      console.log(
+        `| commits ${repo.commits.length}/${totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+      )
+    }
+  },
+})
src/collect/issue-timeline.graphql → src/task/issue-timeline/issue-timeline.graphql
File renamed without changes
src/collect/issue-timeline.graphql.ts → src/task/issue-timeline/issue-timeline.graphql.ts
File renamed without changes
src/task/issue-timeline/issue-timeline.ts
@@ -0,0 +1,38 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { IssueTimelineQuery } from './issue-timeline.graphql.js'
+
+export default task({
+  name: 'issue-timeline' as const,
+  run: async (
+    { octokit, data, next },
+    { owner, name, number }: { owner: string; name: string; number: number },
+  ) => {
+    console.log(`Loading issue timeline for ${name}#${number}`)
+    const timeline = paginate(octokit, IssueTimelineQuery, {
+      owner: owner,
+      name: name,
+      number: number,
+    })
+
+    const issue = data.issues.find((x) => x.number === number)
+    if (!issue) {
+      throw new Error(`Issue ${number} not found`)
+    }
+
+    for await (const resp of timeline) {
+      if (!resp.repository?.issue?.timelineItems.nodes) {
+        throw new Error('Failed to load issue timeline')
+      }
+
+      console.log(
+        `| timeline ${resp.repository.issue.timelineItems.nodes.length}/${resp.repository.issue.timelineItems.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+      )
+      for (const event of resp.repository.issue.timelineItems.nodes) {
+        if (event?.__typename == 'ClosedEvent') {
+          issue.closedBy = event.actor?.login
+        }
+      }
+    }
+  },
+})
src/collect/issues.graphql → src/task/issues/issues.graphql
File renamed without changes
src/collect/issues.graphql.ts → src/task/issues/issues.graphql.ts
File renamed without changes
src/task/issues/issues.ts
@@ -0,0 +1,34 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { IssuesQuery } from './issues.graphql.js'
+
+export default task({
+  name: 'issues' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const issues = paginate(octokit, IssuesQuery, {
+      username,
+    })
+
+    for await (const resp of issues) {
+      if (!resp.user?.issues.nodes) {
+        throw new Error('Failed to load issues')
+      }
+
+      console.log(
+        `Loading issues ${data.issues.length + resp.user.issues.nodes.length}/${
+          resp.user.issues.totalCount
+        } (cost: ${resp.rateLimit?.cost}, remaining: ${
+          resp.rateLimit?.remaining
+        })`,
+      )
+      for (const issue of resp.user.issues.nodes) {
+        data.issues.push(issue)
+        next('issue-timeline', {
+          owner: issue.repository.owner.login,
+          name: issue.repository.name,
+          number: issue.number,
+        })
+      }
+    }
+  },
+})
src/collect/pulls.graphql → src/task/pulls/pulls.graphql
File renamed without changes
src/collect/pulls.graphql.ts → src/task/pulls/pulls.graphql.ts
File renamed without changes
src/task/pulls/pulls.ts
@@ -0,0 +1,29 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { PullsQuery } from './pulls.graphql.js'
+
+export default task({
+  name: 'pulls' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const pulls = paginate(octokit, PullsQuery, {
+      username,
+    })
+
+    for await (const resp of pulls) {
+      if (!resp.user?.pullRequests.nodes) {
+        throw new Error('Failed to load pull requests')
+      }
+
+      console.log(
+        `Loading pull requests ${
+          data.pulls.length + resp.user.pullRequests.nodes.length
+        }/${resp.user.pullRequests.totalCount} (cost: ${
+          resp.rateLimit?.cost
+        }, remaining: ${resp.rateLimit?.remaining})`,
+      )
+      for (const pull of resp.user.pullRequests.nodes) {
+        data.pulls.push(pull)
+      }
+    }
+  },
+})
src/task/repos/repos.ts
@@ -0,0 +1,25 @@
+import { task } from '../../task.js'
+
+export default task({
+  name: 'repos' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const repos = octokit.paginate.iterator('GET /users/{username}/repos', {
+      username,
+      type: 'all',
+      per_page: 100,
+    })
+
+    for await (const resp of repos) {
+      for (const repo of resp.data) {
+        if (repo.name == '-') {
+          continue
+        }
+        data.repos.push({
+          ...repo,
+          commits: [],
+        })
+        next('commits', { owner: repo.owner.login, name: repo.name })
+      }
+    }
+  },
+})
src/collect/stars.graphql → src/task/stars/stars.graphql
File renamed without changes
src/collect/stars.graphql.ts → src/task/stars/stars.graphql.ts
File renamed without changes
src/task/stars/stars.ts
@@ -0,0 +1,24 @@
+import { task } from '../../task.js'
+import { paginate } from '../../utils.js'
+import { StarsQuery } from './stars.graphql.js'
+
+export default task({
+  name: 'stars' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const stars = paginate(octokit, StarsQuery, {
+      login: username,
+    })
+    for await (const resp of stars) {
+      if (!resp.user?.starredRepositories.nodes) {
+        throw new Error('Failed to load stars')
+      }
+
+      for (const repo of resp.user.starredRepositories.nodes) {
+        data.starredRepositories.push(repo)
+      }
+      console.log(
+        `| stars ${data.starredRepositories.length}/${resp.user.starredRepositories.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
+      )
+    }
+  },
+})
src/collect/user.graphql → src/task/user/user.graphql
File renamed without changes
src/collect/user.graphql.ts → src/task/user/user.graphql.ts
File renamed without changes
src/task/user/user.ts
@@ -0,0 +1,18 @@
+import { task } from '../../task.js'
+import { UserQuery } from './user.graphql.js'
+import { query } from '../../utils.js'
+
+export default task({
+  name: 'user' as const,
+  run: async ({ octokit, data, next }, { username }: { username: string }) => {
+    const { user } = await query(octokit, UserQuery, {
+      login: username,
+    })!
+
+    if (!user) {
+      throw new Error('Failed to load user')
+    }
+
+    data.user = user
+  },
+})
src/task/index.ts
@@ -0,0 +1,11 @@
+export default [
+  await import('./user/user.js'),
+  await import('./repos/repos.js'),
+  await import('./pulls/pulls.js'),
+  await import('./commits/commits.js'),
+  await import('./issues/issues.js'),
+  await import('./issue-timeline/issue-timeline.js'),
+  await import('./comments/issue-comments.js'),
+  await import('./comments/discussion-comments.js'),
+  await import('./stars/stars.js'),
+] as const
src/badges.ts
@@ -1,9 +1,9 @@
 import allBadges from '#badges'
 import { linkCommit, linkIssue, linkPull } from './utils.js'
-import { Data } from './collect/index.js'
-import { Commit } from './collect/commits.graphql.js'
-import { PullRequest } from './collect/pulls.graphql.js'
-import { Issue } from './collect/issues.graphql.js'
+import { Data } from './data.js'
+import { Commit } from './task/commits/commits.graphql.js'
+import { PullRequest } from './task/pulls/pulls.graphql.js'
+import { Issue } from './task/issues/issues.graphql.js'
 
 export type Presenters = (typeof allBadges)[number]['default']
 
src/collect.ts
@@ -1,248 +0,0 @@
-import { Octokit } from 'octokit'
-import { Data } from './collect/index.js'
-import { Query, Variables } from 'megaera'
-import { UserQuery } from './collect/user.graphql.js'
-import { CommitsQuery } from './collect/commits.graphql.js'
-import { PullsQuery } from './collect/pulls.graphql.js'
-import { IssuesQuery } from './collect/issues.graphql.js'
-import { IssueTimelineQuery } from './collect/issue-timeline.graphql.js'
-import {
-  DiscussionCommentsQuery,
-  IssueCommentsQuery,
-} from './collect/comments.graphql.js'
-import { StarsQuery } from './collect/stars.graphql.js'
-
-export async function collect(
-  octokit: Octokit,
-  username: string,
-): Promise<Data> {
-  function query<T extends Query>(query: T, variables: Variables<T>) {
-    return octokit.graphql<ReturnType<T>>(query, variables)
-  }
-
-  function paginate<T extends Query>(query: T, variables: Variables<T>) {
-    return octokit.graphql.paginate.iterator<ReturnType<T>>(query, variables)
-  }
-
-  const { user } = await query(UserQuery, {
-    login: username,
-  })!
-
-  if (!user) {
-    throw new Error('Failed to load user')
-  }
-
-  const data: Data = {
-    user: user,
-    starredRepositories: [],
-    repos: [],
-    pulls: [],
-    issues: [],
-    issueComments: [],
-    discussionComments: [],
-  }
-
-  const repos = octokit.paginate.iterator('GET /users/{username}/repos', {
-    username,
-    type: 'all',
-    per_page: 100,
-  })
-
-  for await (const resp of repos) {
-    for (const repo of resp.data) {
-      if (repo.name == '-') {
-        continue
-      }
-      data.repos.push({
-        ...repo,
-        commits: [],
-      })
-    }
-  }
-
-  for (const repo of data.repos) {
-    console.log(`Loading commits for ${repo.owner.login}/${repo.name}`)
-    try {
-      const commits = paginate(CommitsQuery, {
-        owner: repo.owner.login,
-        name: repo.name,
-        author: user.id,
-      })
-
-      for await (const resp of commits) {
-        const { totalCount, nodes } =
-          resp.repository?.defaultBranchRef?.target?.history!
-
-        if (!nodes) {
-          throw new Error('Failed to load commits')
-        }
-
-        if (totalCount >= 10_000) {
-          console.error(
-            `Too many commits for ${repo.owner.login}/${repo.name}: ${totalCount} commits; My-Badges will skip repos with more than 10k commits.`,
-          )
-          break
-        }
-
-        for (const commit of nodes) {
-          repo.commits.push(commit)
-        }
-        console.log(
-          `| commits ${repo.commits.length}/${totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
-        )
-      }
-    } catch (err) {
-      console.error(
-        `Failed to load commits for ${repo.owner.login}/${repo.name}`,
-      )
-      console.error(err)
-    }
-  }
-
-  const pulls = paginate(PullsQuery, {
-    username,
-  })
-  try {
-    for await (const resp of pulls) {
-      if (!resp.user?.pullRequests.nodes) {
-        throw new Error('Failed to load pull requests')
-      }
-
-      console.log(
-        `Loading pull requests ${
-          data.pulls.length + resp.user.pullRequests.nodes.length
-        }/${resp.user.pullRequests.totalCount} (cost: ${
-          resp.rateLimit?.cost
-        }, remaining: ${resp.rateLimit?.remaining})`,
-      )
-      for (const pull of resp.user.pullRequests.nodes) {
-        data.pulls.push(pull)
-      }
-    }
-  } catch (err) {
-    console.error(`Failed to load pull requests`)
-    console.error(err)
-  }
-
-  const issues = paginate(IssuesQuery, {
-    username,
-  })
-  try {
-    for await (const resp of issues) {
-      if (!resp.user?.issues.nodes) {
-        throw new Error('Failed to load issues')
-      }
-
-      console.log(
-        `Loading issues ${data.issues.length + resp.user.issues.nodes.length}/${
-          resp.user.issues.totalCount
-        } (cost: ${resp.rateLimit?.cost}, remaining: ${
-          resp.rateLimit?.remaining
-        })`,
-      )
-      for (const issue of resp.user.issues.nodes) {
-        data.issues.push(issue)
-      }
-    }
-  } catch (err) {
-    console.error(`Failed to load issues`)
-    console.error(err)
-  }
-
-  for (const issue of data.issues) {
-    console.log(
-      `Loading issue timeline for ${issue.repository.name}#${issue.number}`,
-    )
-    try {
-      const timeline = paginate(IssueTimelineQuery, {
-        owner: issue.repository.owner.login,
-        name: issue.repository.name,
-        number: issue.number,
-      })
-      for await (const resp of timeline) {
-        if (!resp.repository?.issue?.timelineItems.nodes) {
-          throw new Error('Failed to load issue timeline')
-        }
-
-        console.log(
-          `| timeline ${resp.repository.issue.timelineItems.nodes.length}/${resp.repository.issue.timelineItems.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
-        )
-        for (const event of resp.repository.issue.timelineItems.nodes) {
-          if (event?.__typename == 'ClosedEvent') {
-            issue.closedAt = event.createdAt
-          }
-        }
-      }
-    } catch (err) {
-      console.error(
-        `Failed to load issue timeline for ${issue.repository.name}#${issue.number}`,
-      )
-      console.error(err)
-    }
-  }
-
-  const issueComments = paginate(IssueCommentsQuery, {
-    login: username,
-  })
-  try {
-    for await (const resp of issueComments) {
-      if (!resp.user?.issueComments.nodes) {
-        throw new Error('Failed to load issue comments')
-      }
-
-      for (const comment of resp.user.issueComments.nodes) {
-        data.issueComments.push(comment)
-      }
-      console.log(
-        `| issue comments ${data.issueComments.length}/${resp.user.issueComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
-      )
-    }
-  } catch (err) {
-    console.error(`Failed to load issue comments`)
-    console.error(err)
-  }
-
-  const discussionComments = paginate(DiscussionCommentsQuery, {
-    login: username,
-  })
-  try {
-    for await (const resp of discussionComments) {
-      if (!resp.user?.repositoryDiscussionComments.nodes) {
-        throw new Error('Failed to load discussion comments')
-      }
-
-      for (const comment of resp.user.repositoryDiscussionComments.nodes) {
-        data.discussionComments.push(comment)
-      }
-      console.log(
-        `| discussion comments ${data.discussionComments.length}/${resp.user.repositoryDiscussionComments.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
-      )
-    }
-  } catch (err) {
-    console.error(`Failed to load discussion comments`)
-    console.error(err)
-  }
-
-  const stars = paginate(StarsQuery, {
-    login: username,
-  })
-  try {
-    for await (const resp of stars) {
-      if (!resp.user?.starredRepositories.nodes) {
-        throw new Error('Failed to load stars')
-      }
-
-      for (const repo of resp.user.starredRepositories.nodes) {
-        data.starredRepositories.push(repo)
-      }
-      console.log(
-        `| stars ${data.starredRepositories.length}/${resp.user.starredRepositories.totalCount} (cost: ${resp.rateLimit?.cost}, remaining: ${resp.rateLimit?.remaining})`,
-      )
-    }
-  } catch (err) {
-    console.error(`Failed to load stars`)
-    console.error(err)
-  }
-
-  return data
-}
src/data.ts
@@ -0,0 +1,31 @@
+import { Endpoints } from '@octokit/types'
+import { User } from './task/user/user.graphql.js'
+import { PullRequest } from './task/pulls/pulls.graphql.js'
+import { Issue as IssueType } from './task/issues/issues.graphql.js'
+import {
+  DiscussionComment,
+  IssueComment,
+} from './task/comments/comments.graphql.js'
+import { StarredRepo } from './task/stars/stars.graphql.js'
+import { Commit } from './task/commits/commits.graphql.js'
+
+// Data is collected by tasks, enriched if needed, and saved to disk.
+// Use this data to determine which badges to present to the user.
+export type Data = {
+  user: User
+  starredRepositories: StarredRepo[]
+  repos: Repo[]
+  pulls: PullRequest[]
+  issues: Issue[]
+  issueComments: IssueComment[]
+  discussionComments: DiscussionComment[]
+}
+
+export type Issue = {
+  closedBy?: string
+} & IssueType
+
+export type Repo =
+  Endpoints['GET /users/{username}/repos']['response']['data'][0] & {
+    commits: Commit[]
+  }
src/get-data.ts
@@ -1,29 +0,0 @@
-import { collect } from './collect.js'
-import fs from 'node:fs'
-import { Octokit } from 'octokit'
-import { Data } from './collect/index.js'
-
-export async function getData(
-  octokit: Octokit,
-  dataPath: string,
-  username: string,
-): Promise<Data> {
-  if (dataPath !== '') {
-    if (!fs.existsSync(dataPath)) {
-      throw new Error('Data file not found')
-    }
-    return JSON.parse(fs.readFileSync(dataPath, 'utf8')) as Data
-  }
-
-  if (!username) {
-    throw new Error('Specify username')
-  }
-
-  const data = await collect(octokit, username)
-  if (!fs.existsSync('data')) {
-    fs.mkdirSync('data')
-  }
-  fs.writeFileSync(`data/${username}.json`, JSON.stringify(data, null, 2))
-
-  return data
-}
src/index.ts
@@ -1,8 +1,8 @@
 export { define } from './badges.js'
-export { Repo } from './collect/index.js'
-export { User } from './collect/user.graphql.js'
-export { Issue } from './collect/issues.graphql.js'
-export { PullRequest } from './collect/pulls.graphql.js'
-export { Commit } from './collect/commits.graphql.js'
+export { Repo } from './data.js'
+export { User } from './task/user/user.graphql.js'
+export { Issue } from './task/issues/issues.graphql.js'
+export { PullRequest } from './task/pulls/pulls.graphql.js'
+export { Commit } from './task/commits/commits.graphql.js'
 
 export { linkCommit, linkIssue, linkPull, latest, plural } from './utils.js'
src/main.ts
@@ -6,10 +6,12 @@ import { Octokit } from 'octokit'
 import { retry } from '@octokit/plugin-retry'
 import { throttling } from '@octokit/plugin-throttling'
 import { presentBadges } from './present-badges.js'
-import { getData } from './get-data.js'
-import { getUserBadges, syncRepo, gitPush, thereAreChanges } from './repo.js'
+import { getUserBadges, gitPush, syncRepo, thereAreChanges } from './repo.js'
 import { updateBadges } from './update-badges.js'
 import { updateReadme } from './update-readme.js'
+import { processTasks } from './process-tasks.js'
+import fs from 'node:fs'
+import { Data } from './data.js'
 
 void (async function main() {
   try {
@@ -67,7 +69,17 @@ void (async function main() {
     })
 
     if (owner && repo) syncRepo(owner, repo, token)
-    const data = await getData(octokit, dataPath, username)
+
+    let data: Data
+    if (dataPath !== '') {
+      data = JSON.parse(fs.readFileSync(dataPath, 'utf8')) as Data
+    } else {
+      let ok: boolean
+      ;[ok, data] = await processTasks(octokit, username)
+      if (!ok) {
+        return
+      }
+    }
 
     let userBadges = getUserBadges()
     userBadges = presentBadges(
src/present-badges.ts
@@ -1,8 +1,6 @@
 import { Badge, Evidence, ID, List, Presenter } from './badges.js'
-import path from 'node:path'
-import { fileURLToPath } from 'node:url'
 import { parseMask } from './utils.js'
-import { Data } from './collect/index.js'
+import { Data } from './data.js'
 
 export const presentBadges = <P extends Presenter<List>>(
   presenters: P[],
src/process-tasks.ts
@@ -0,0 +1,78 @@
+import fs from 'node:fs'
+import { Octokit } from 'octokit'
+import { Data } from './data.js'
+import { TaskName } from './task.js'
+import allTasks from './task/index.js'
+
+export async function processTasks(
+  octokit: Octokit,
+  username: string,
+): Promise<[boolean, Data]> {
+  if (!fs.existsSync('data')) {
+    fs.mkdirSync('data')
+  }
+  const dataPath = `data/${username}.json`
+  const tasksPath = `data/${username}.tasks.json`
+
+  let data: Data = {
+    user: null!,
+    starredRepositories: [],
+    repos: [],
+    pulls: [],
+    issues: [],
+    issueComments: [],
+    discussionComments: [],
+  }
+
+  if (fs.existsSync(dataPath)) {
+    data = JSON.parse(fs.readFileSync(dataPath, 'utf8')) as Data
+  }
+
+  let todo: [TaskName, any][] = [
+    ['user', { username }],
+    ['repos', { username }],
+    ['pulls', { username }],
+    ['issues', { username }],
+    ['issue-timeline', { username, name: 'my-badges', number: 42 }],
+    ['issue-comments', { username }],
+    ['discussion-comments', { username }],
+    ['stars', { username }],
+  ]
+
+  if (fs.existsSync(tasksPath)) {
+    const savedTodo = JSON.parse(fs.readFileSync(tasksPath, 'utf8')) as [
+      TaskName,
+      any,
+    ][]
+    if (savedTodo.length > 0) {
+      todo = savedTodo
+    }
+  }
+
+  while (todo.length > 0) {
+    const [taskName, params] = todo.shift()!
+
+    const task = allTasks.find(({ default: t }) => t.name === taskName)?.default
+    if (!task) {
+      throw new Error(`Unknown task ${taskName}`)
+    }
+
+    const next = (taskName: TaskName, params: any) => {
+      todo.push([taskName, params])
+    }
+
+    console.log(`==> Running task ${taskName}`, params)
+    try {
+      await task.run({ octokit, data, next }, params)
+    } catch (e) {
+      console.error(`!!! Failed to run task ${taskName}`, params)
+      console.error(e)
+    }
+    console.log(`<== Finished ${taskName} (${todo.length} tasks left)`)
+
+    fs.writeFileSync(dataPath, JSON.stringify(data, null, 2))
+    fs.writeFileSync(tasksPath, JSON.stringify(todo, null, 2))
+  }
+
+  return [false, data]
+}
src/repo.ts
@@ -7,6 +7,7 @@ export function syncRepo(owner: string, repo: string, token: string) {
   if (fs.existsSync('repo')) {
     chdir('repo')
     exec('git', ['pull'])
+    chdir('..')
     return
   }
 
src/task.ts
@@ -0,0 +1,20 @@
+import { Octokit } from 'octokit'
+import { Data } from './data.js'
+import allTasks from './task/index.js'
+
+export type TaskName = (typeof allTasks)[number]['default']['name']
+
+type Context = {
+  octokit: Octokit
+  data: Data
+  next: (taskName: TaskName, params: any) => void
+}
+
+type Task<Name extends string> = {
+  name: Name
+  run: (context: Context, params: any) => Promise<void>
+}
+
+export function task<Name extends string>(t: Task<Name>) {
+  return t
+}
src/utils.ts
@@ -1,7 +1,25 @@
 import { spawnSync } from 'node:child_process'
-import { Commit } from './collect/commits.graphql.js'
-import { PullRequest } from './collect/pulls.graphql.js'
-import { Issue } from './collect/issues.graphql.js'
+import { Octokit } from 'octokit'
+import { Query, Variables } from 'megaera'
+import { Commit } from './task/commits/commits.graphql.js'
+import { PullRequest } from './task/pulls/pulls.graphql.js'
+import { Issue } from './task/issues/issues.graphql.js'
+
+export function query<T extends Query>(
+  octokit: Octokit,
+  query: T,
+  variables: Variables<T>,
+) {
+  return octokit.graphql<ReturnType<T>>(query, variables)
+}
+
+export function paginate<T extends Query>(
+  octokit: Octokit,
+  query: T,
+  variables: Variables<T>,
+) {
+  return octokit.graphql.paginate.iterator<ReturnType<T>>(query, variables)
+}
 
 export function linkCommit(commit: Commit): string {
   return `<a href="https://github.com/${commit.repository.owner.login}/${
test/get-data.test.ts
@@ -1,56 +0,0 @@
-import * as assert from 'node:assert'
-import { describe, it } from 'node:test'
-import fs from 'node:fs'
-import path from 'node:path'
-import os from 'node:os'
-import { Octokit } from 'octokit'
-import { getData } from '../src/get-data.js'
-
-const tempy = () => fs.mkdtempSync(path.join(os.tmpdir(), 'tempy-'))
-
-describe('get-badges', () => {
-  const octokit = new Octokit({})
-  const username = 'antongolub'
-  const owner = username
-  const repo = username
-
-  it('getData() reads snapshot data if dataPath specified', async () => {
-    const dataPath = path.join(tempy(), 'data.json')
-    const _data = {
-      user: {
-        id: 'MDQ6VXNlcjUyODgwNDY=',
-        login: 'antongolub',
-        name: 'Anton Golub',
-        createdAt: '2013-08-22T15:51:42Z',
-        starredRepositories: {
-          totalCount: 161,
-        },
-      },
-      repos: [],
-    }
-    fs.writeFileSync(dataPath, JSON.stringify(_data), 'utf-8')
-
-    const data = await getData(octokit, dataPath, username)
-
-    assert.deepEqual(_data, data)
-  })
-
-  it('getData() throws an err if `dataPath` is unreachable', async () => {
-    try {
-      const dataPath = '/foo/bar/baz'
-      await getData(octokit, dataPath, username)
-    } catch (e) {
-      assert.equal((e as Error).message, 'Data file not found')
-    }
-  })
-
-  it('getData() throws an err if `username` is empty', async () => {
-    try {
-      const dataPath = ''
-      const username = ''
-      await getData(octokit, dataPath, username)
-    } catch (e) {
-      assert.equal((e as Error).message, 'Specify username')
-    }
-  })
-})
test/present-badges.test.ts
@@ -2,7 +2,7 @@ import * as assert from 'node:assert'
 import { describe, it } from 'node:test'
 import { presentBadges } from '../src/present-badges.js'
 import { Badge, define } from '../src/badges.js'
-import { Data } from '../src/collect/index.js'
+import { Data } from '../src/data.js'
 
 describe('present-badges', () => {
   const data: Data = {
test/stars.test.ts
@@ -2,7 +2,7 @@ import * as assert from 'node:assert'
 import { describe, it } from 'node:test'
 import starsPresenter from '#badges/stars/stars.js'
 import { badgeCollection } from '../src/present-badges.js'
-import { Data } from '../src/collect/index.js'
+import { Data } from '../src/data.js'
 import { Badge } from '../src/badges.js'
 
 describe('stars', () => {