fix: github collector

dvgamerr-app · Mar 5, 2024 · 9593831 · 9593831
1 parent c9dab02
commit 9593831
Show file tree

Hide file tree

Showing 3 changed files with 108 additions and 45 deletions.
diff --git a/collector/github.js b/collector/github.js
@@ -19,30 +19,47 @@ const fetchGithub = async (method, url, body) => {
     })
     status = res.status
 
-    retry = !res.ok && tries > 0
+    retry = status > 202 && tries > 0
     if (retry) {
       tries--
-      await sleep(500)
+      await sleep(5000)
     } else {
-      data = await res.json()
+      try {
+        data = await res.json()
+      } catch (ex) {
+        data = { payload: await res.text() }
+        logger.debug({ data, tries, status })
+      }
     }
   } while (retry)
   return { data, status }
 }
 
-const ghlocFetch = async (owner, name) => {
-//
-  const res = await fetch(`https://ghloc.vercel.app/api/${owner}/${name}/badge`)
-  const data = await res.json()
-  return { data,  status: res.status }
-}
+// const ghlocFetch = async (owner, name) => {
+// //
+//   const res = await fetch(`https://ghloc.vercel.app/api/${owner}/${name}/badge`)
+//   const data = await res.json()
+//   return { data,  status: res.status }
+// }
 
 module.exports = {
-  ghlocFetch,
   getContributors: async (owner, name) => fetchGithub('GET',`/repos/${owner}/${name}/stats/contributors`),
   getCommitActivity: async (owner, name) => fetchGithub('GET',`/repos/${owner}/${name}/stats/commit_activity`),
   getLanguages: async (owner, name) => fetchGithub('GET',`/repos/${owner}/${name}/languages`),
-  getRepos: async (owner, page) => fetchGithub('GET',`/orgs/${owner}/repos?page=${page}&per_page=100`),
+  getOrgsRepos: async (owner, page) => fetchGithub('GET',`/orgs/${owner}/repos?page=${page}&per_page=100`),
+  getUserRepos: async (page) => fetchGithub('GET',`/user/repos?type=owner&page=${page}&per_page=100`),
 }
 
 // GET /orgs/{org}/repos
+
+
+
+// let page = 0
+// do {
+//   page++
+//   const { data: repos } = await apiGitHub.request('GET /user/repos', { type: 'owner', per_page: 100, page })
+//   orgRepos = orgRepos.concat(repos)
+//   nextPage = repos.length > 0
+// } while(nextPage)
+
+// coding.project = orgRepos.length
diff --git a/collector/index.js b/collector/index.js
@@ -6,20 +6,13 @@ const logger = require('pino')({ level: 'trace' })
 
 const { mergeJsonResponse } = require('./json-merge')
 const {
-  getRepos,
+  getOrgsRepos,
+  getUserRepos,
   getContributors,
   getLanguages,
   getCommitActivity,
 } = require('./github')
 
-
-
-
-// const { existsSync, unlinkSync } = require('fs')
-// const { extname, join } = require('path')
-
-// // const logger = debuger('GEN')
-
 const args = arg({
 	'--github': Boolean,
 	'--wakatime': Boolean,
@@ -57,13 +50,13 @@ const collectReposOrgs = async (orgsRepo = ['dvgamerr-app']) => {
   let repos = []
   let nextPage = false
 
-  const orgsRepoTask = []
+  const repoTask = []
   for (const org of orgsRepo) {
-    orgsRepoTask.push((async () => {
+    repoTask.push((async () => {
       let page = 0
       do {
         page++
-        const { data } = await getRepos(org, page)
+        const { data } = await getOrgsRepos(org, page)
         repos = repos.concat(data)
         nextPage = data.length > 0
         if (nextPage) {
@@ -72,7 +65,23 @@ const collectReposOrgs = async (orgsRepo = ['dvgamerr-app']) => {
       } while(nextPage)
     })())
   }
-  await Promise.all(orgsRepoTask)
+  await Promise.all(repoTask)
+  return repos
+}
+
+const collectRepoOwner = async () => {
+  let repos = []
+  let nextPage = false
+
+  const repoTask = []
+  let page = 0
+  do {
+    page++
+    const { data } = await getUserRepos(page)
+    repos = repos.concat(data)
+    nextPage = data.length > 0
+  } while(nextPage)
+  logger.debug(` - 'owner' repos: ${repos.length}`)
   return repos
 }
 
@@ -85,23 +94,25 @@ const collectGithubProjectStats = async () => {
     private: 3,
     public: 0, // Public Repos
     languages: [],
-    commits: 0
+    commits: 0,
+    loc: 0
   }
 
   // const contribRepos = {}
   const orgRepos = await collectReposOrgs([ 'dvgamerr-app','central-group','nippon-sysits' ])
+  const usrRepos = await collectRepoOwner()
 
-  coding.total = orgRepos.length + coding.private
-  coding.public = orgRepos.filter(e => !e.private).length
+  const repos = orgRepos.concat(usrRepos)
+  coding.total = repos.length + coding.private
+  coding.public = repos.filter(e => !e.private).length
   logger.info('Contributors Task...')
 
-  const orgReposTask = []
-  const displayData = {}
-  for await (const e of orgRepos) {
-    orgReposTask.push((async () => {
+  const repoTask = []
+  for await (const e of repos) {
+    repoTask.push((async () => {
+      logger.trace(` - repos '${e.owner.login}/${e.name}'`)
       const { data: contrib, status: statusContrib } = await getContributors(e.owner.login, e.name)
       if (statusContrib === 200) {
-        if (!displayData['contrib']) displayData['contrib'] = contrib
         for (const con of contrib) {
           if (con.author.login !== 'dvgamerr') continue
           coding.commits += con.total
@@ -110,15 +121,18 @@ const collectGithubProjectStats = async () => {
 
       const { data: langs, status: statusLangs }  = await getLanguages(e.owner.login, e.name)
       if (statusLangs === 200) {
-        if (!displayData['langs']) displayData['langs'] = langs
         coding.languages = [...new Set(coding.languages.concat(Object.keys(langs)))]
       }
+
+      // const { data: loc, status: statusLoc }  = await ghlocFetch(e.owner.login, e.name)
+      // if (statusLoc === 200) {
+      //   coding.loc += parseInt(loc.message)
+      // }
     })())
   }
 
-  logger.info(`Task Github API (${orgReposTask.length}) ...`)
-  console.log(displayData)
-  await Promise.all(orgReposTask)
+  logger.info(`Task Github API (${repoTask.length}) ...`)
+  await Promise.all(repoTask)
 
   // https://ghloc.vercel.app/api/dvgamerr-app/go-hoyolab/badge
 

diff --git a/src/i18n/coding.json b/src/i18n/coding.json
@@ -1,30 +1,62 @@
 {
-  "total": 39,
+  "total": 220,
   "private": 3,
-  "public": 36,
+  "public": 217,
   "languages": [
     "JavaScript",
     "HTML",
+    "PHP",
+    "TSQL",
     "CSS",
+    "Hack",
+    "Ruby",
+    "Shell",
+    "C++",
+    "Astro",
+    "Go",
     "Dockerfile",
+    "Python",
     "Vue",
-    "Go",
-    "Shell",
     "TypeScript",
     "C#",
-    "Astro",
     "SCSS",
+    "Dart",
+    "CMake",
+    "Swift",
+    "C",
+    "Kotlin",
+    "Objective-C",
+    "PowerShell",
+    "Stylus",
+    "AutoHotkey",
+    "Batchfile",
+    "Mathematica",
+    "Makefile",
     "Svelte",
-    "Ruby",
-    "Liquid"
+    "Rust",
+    "SuperCollider",
+    "Smarty",
+    "DataWeave",
+    "VBScript",
+    "Java",
+    "Visual Basic",
+    "CoffeeScript",
+    "Liquid",
+    "ApacheConf",
+    "Jupyter Notebook",
+    "Procfile",
+    "M4",
+    "Nix",
+    "EJS",
+    "Jsonnet"
   ],
-  "commits": 2105,
+  "commits": 3670,
   "experience": "2013-04-29T00:00:00.000+07:00",
   "updated": "2023-04-02T12:44:50.863Z",
   "weekly_seconds": 7317.49946,
   "average_seconds": 7058.645887263014,
   "best_seconds": 44422.207450999995,
-  "loc": 4928566,
+  "loc": 0,
   "daytime": [
     69,
     35,