1import fs from 'node:fs'
  2import { GraphqlResponseError } from '@octokit/graphql'
  3import { Data } from './data.js'
  4import { TaskName } from './task.js'
  5import allTasks from './task/index.js'
  6
  7import { type Context } from './context.js'
  8import { createBatcher } from './batch.js'
  9import { getOctokit } from './utils.js'
 10import { log } from './log.js'
 11
 12const MAX_ATTEMPTS = 3
 13
 14export async function processTasks(
 15  ctx: Pick<
 16    Context,
 17    | 'ghUser'
 18    | 'ghToken'
 19    | 'dataDir'
 20    | 'dataFile'
 21    | 'dataTasks'
 22    | 'taskName'
 23    | 'taskSkip'
 24    | 'taskParams'
 25  >,
 26): Promise<[boolean, Data]> {
 27  const {
 28    ghToken,
 29    ghUser: username,
 30    dataFile,
 31    dataTasks,
 32    taskSkip,
 33    taskName,
 34    taskParams,
 35  } = ctx
 36
 37  if (!ghToken) throw new Error('GitHub token is required for data gathering')
 38  if (!username)
 39    throw new Error('GitHub username is required for data gathering')
 40
 41  const taskSkipSet = new Set(taskSkip?.split(',') || [])
 42  const octokit = getOctokit(ghToken)
 43
 44  let data: Data = {
 45    user: null!,
 46    starredRepositories: [],
 47    repos: [],
 48    pulls: [],
 49    issues: [],
 50    issueComments: [],
 51    discussionComments: [],
 52  }
 53
 54  if (fs.existsSync(dataFile)) {
 55    data = JSON.parse(fs.readFileSync(dataFile, 'utf8')) as Data
 56  }
 57
 58  type Todo = {
 59    taskName: TaskName
 60    params: any
 61    attempts: number
 62  }
 63
 64  let todo: Todo[] = [
 65    { taskName: 'user', params: { username }, attempts: 0 },
 66    { taskName: 'pulls', params: { username }, attempts: 0 },
 67    { taskName: 'issues', params: { username }, attempts: 0 },
 68    { taskName: 'issue-comments', params: { username }, attempts: 0 },
 69    { taskName: 'discussion-comments', params: { username }, attempts: 0 },
 70    { taskName: 'stars', params: { username }, attempts: 0 },
 71  ]
 72
 73  if (taskName && taskParams) {
 74    todo = [
 75      {
 76        taskName: taskName as TaskName,
 77        params: Object.fromEntries(new URLSearchParams(taskParams).entries()),
 78        attempts: 0,
 79      },
 80    ]
 81  } else if (fs.existsSync(dataTasks)) {
 82    const savedTodo = JSON.parse(fs.readFileSync(dataTasks, 'utf8')) as Todo[]
 83    if (savedTodo.length > 0) {
 84      todo = savedTodo
 85    }
 86  }
 87
 88  while (todo.length > 0) {
 89    const { taskName, params, attempts } = todo.shift()!
 90    if (taskSkipSet.has(taskName)) {
 91      log.info(`Skipping task ${taskName}`)
 92      continue
 93    }
 94
 95    const task = allTasks.find(({ default: t }) => t.name === taskName)?.default
 96    if (!task) {
 97      throw new Error(`Unknown task ${taskName}`)
 98    }
 99
100    const next = (taskName: TaskName, params: any) => {
101      todo.push({ taskName, params, attempts: 0 })
102    }
103    const { batch, flush } = createBatcher(next)
104
105    log.info(
106      `==> Running task ${taskName}`,
107      new URLSearchParams(params).toString(),
108      attempts > 0 ? `(attempt: ${attempts + 1})` : '',
109    )
110    try {
111      await task.run({ octokit, data, next, batch }, params)
112    } catch (e) {
113      let retry = true
114      if (e instanceof GraphqlResponseError) {
115        retry = e.errors?.some((error) => error.type == 'NOT_FOUND') ?? true
116      }
117
118      if (attempts >= MAX_ATTEMPTS || !retry) {
119        log.error(
120          `!!! Failed to run task ${taskName}`,
121          new URLSearchParams(params).toString(),
122          `after ${attempts} attempts`,
123        )
124        log.error(e)
125      } else {
126        log.error(
127          `!!! Failed to run task ${taskName}`,
128          new URLSearchParams(params).toString(),
129          `retrying`,
130          `(will try ${MAX_ATTEMPTS - attempts} more times)`,
131        )
132        log.error(e)
133        todo.push({ taskName, params, attempts: attempts + 1 })
134      }
135    }
136    log.info(`<== Finished ${taskName} (${todo.length} tasks left)`)
137
138    flush()
139
140    fs.writeFileSync(dataFile, JSON.stringify(data, null, 2))
141    fs.writeFileSync(dataTasks, JSON.stringify(todo, null, 2))
142  }
143
144  return [true, data]
145}