Import - Programmer sur Github Actions

- Programmé tous les jours à 18h10 UTC ou à la demande (mais en obligeant à n'en avoir qu'un seul qui tourne à la fois) - Adaptation pour qu'il aille chercher le dernier dump uploadé - Écrits les points clefs dans le job summary via `appendToGithubSummary` - Publie les CSV en tant qu'artefact - Remplacement de l'utilisation de `pg_secret_config.mjs` par des variables d'environnements qui viennent de secrets Github ref #893
MTES-MCT · Nov 29, 2024 · 94b53a4 · 94b53a4
1 parent ae6a549
commit 94b53a4
Show file tree

Hide file tree

Showing 9 changed files with 131 additions and 31 deletions.
diff --git a/.github/workflows/sudocuh_import.yml b/.github/workflows/sudocuh_import.yml
@@ -0,0 +1,37 @@
+name: Sudocuh import
+on:
+  schedule:
+    - cron: '10 18 * * *'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  import:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - run: npm ci
+
+      - env:
+          PGPASSWORD: ${{ secrets.DEV_SUPABASE_PASSWORD }}
+          DEV_SUPABASE_ADMIN_KEY: ${{ secrets.DEV_SUPABASE_ADMIN_KEY }}
+          DEV_SUPABASE_PASSWORD: ${{ secrets.DEV_SUPABASE_PASSWORD }}
+          PROD_SUPABASE_ADMIN_KEY: ${{ secrets.PROD_SUPABASE_ADMIN_KEY }}
+          PROD_SUPABASE_PASSWORD: ${{ secrets.PROD_SUPABASE_PASSWORD }}
+        run: npm run daily_dump
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: daily_dump/output
diff --git a/daily_dump/common.mjs b/daily_dump/common.mjs
@@ -0,0 +1,4 @@
+import { appendFileSync } from 'node:fs'
+export function appendToGithubSummary(markdown) {
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown)
+}
diff --git a/daily_dump/database_config.mjs b/daily_dump/database_config.mjs
@@ -0,0 +1,16 @@
+export default {
+  PG_DEV_CONFIG: {
+    env: 'dev',
+    url: 'https://drncrjteathtblggsgxi.supabase.co',
+    host: 'aws-0-eu-west-2.pooler.supabase.com',
+    port: 5432,
+    database: 'postgres'
+  },
+  PG_PROD_CONFIG: {
+    env: 'prod',
+    url: 'https://ixxbyuandbmplfnqtxyw.supabase.co',
+    host: 'aws-0-eu-central-1.pooler.supabase.com',
+    port: 5432,
+    database: 'postgres'
+  }
+}
diff --git a/daily_dump/index.mjs b/daily_dump/index.mjs
@@ -1,22 +1,36 @@
+import { mkdirSync } from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
-import CONFIG from './pg_secret_config.mjs'
+import CONFIG from './database_config.mjs'
 import { downloadDump } from './steps/1-downloadDump.mjs'
 import { sudocuhPlanToDocurba } from './steps/4-interTablesToDocurbaPlan.mjs'
 import { sudocuhScotToDocurba } from './steps/5-interTablesToDocurbaScot.mjs'
 import { updatePerimeterStatus } from './steps/6-setPerimeterStatus.mjs'
 import { migrateDgd } from './steps/8-migrateDgd.mjs'
 import { handleTrigger } from './steps/onOffTriggers.mjs'
-import { clearDev, createOriginalSchema, createSudocuProcessedTables, loadDump, setAllStatus } from './steps/sqlRunner.mjs'
+import {
+  clearDev,
+  createOriginalSchema,
+  createSudocuProcessedTables,
+  loadDump,
+  setAllStatus
+} from './steps/sqlRunner.mjs'
 // import { updateProcedureSec } from './updateProcedureSec.mjs'
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+mkdirSync(path.join(__dirname, 'sudocuh_dumps'), { recursive: true })
+CONFIG.PG_DEV_CONFIG.user = process.env.DEV_SUPABASE_USER
+CONFIG.PG_DEV_CONFIG.admin_key = process.env.DEV_SUPABASE_ADMIN_KEY
+CONFIG.PG_DEV_CONFIG.password = process.env.DEV_SUPABASE_PASSWORD
+CONFIG.PG_PROD_CONFIG.user = process.env.PROD_SUPABASE_USER
+CONFIG.PG_PROD_CONFIG.admin_key = process.env.PROD_SUPABASE_ADMIN_KEY
+CONFIG.PG_PROD_CONFIG.password = process.env.PROD_SUPABASE_PASSWORD
 
 /// /////////////////////
 /// ///// INFO ////////
 /// /////////////////////
-const DUMP_FILENAME = process.argv.at(-1)
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-await downloadDump(CONFIG.PG_PROD_CONFIG, DUMP_FILENAME, __dirname)
+const latestDumpName = await downloadDump(CONFIG.PG_PROD_CONFIG, __dirname)
 
 // You need to setup .pgpass in your home if you don't want to enter the db password on pg_restaure
 // https://tableplus.com/blog/2019/09/how-to-use-pgpass-in-postgresql.html
@@ -29,8 +43,7 @@ await downloadDump(CONFIG.PG_PROD_CONFIG, DUMP_FILENAME, __dirname)
 await clearDev(CONFIG.PG_DEV_CONFIG)
 // // // // // Step 1 - Charge un dump particulier venant de l'export de Andy sur notre storage
 
-await loadDump(CONFIG.PG_DEV_CONFIG, DUMP_FILENAME)
-
+await loadDump(CONFIG.PG_DEV_CONFIG, latestDumpName)
 // // // // // // Step 2 - Créer les tables intermédiaires d'aggregation depuis la donnée Sudocuh
 await createSudocuProcessedTables(CONFIG.PG_DEV_CONFIG)
 // // // // // // Replique un schema de test (Optionnal)
@@ -56,3 +69,7 @@ await updatePerimeterStatus(CONFIG.PG_PROD_CONFIG)
 await migrateDgd(CONFIG.PG_DEV_CONFIG, CONFIG.PG_PROD_CONFIG)
 // // Step 8 - Réactivation du trigger de changement de status sur nouveaux events
 await handleTrigger(CONFIG.PG_PROD_CONFIG, 'enable')
+
+// Pour une raison que nous ne connaissons pas, le process Node ne se termine pas.
+// process.exit le force.
+process.exit(0)
diff --git a/daily_dump/steps/1-downloadDump.mjs b/daily_dump/steps/1-downloadDump.mjs
@@ -2,24 +2,42 @@
 import { createClient } from '@supabase/supabase-js'
 import * as fs from 'node:fs'
 import path from 'node:path'
+import { appendToGithubSummary } from '../common.mjs'
 
-export async function downloadDump(prodConfig, dumpFilename, dirname) {
-  console.log('⬇️ Téléchargement du dump…')
+export async function downloadDump(prodConfig, dirname) {
+  console.log('⬇️ Téléchargement du dernier dump…')
   const supabase = createClient(prodConfig.url, prodConfig.admin_key, {
     auth: { persistSession: false }
   })
+  const { data, errorA } = await supabase.storage
+    .from('dump_sudocu')
+    .list('', { limit: 1, sortBy: { column: 'created_at', order: 'desc' } })
+  if (errorA || data.length < 1) {
+    console.error('❌ Impossible de lister les dumps')
+    console.error(errorA)
+    process.exit(1)
+  }
+
+  const latestDumpName = data[0].name
+  console.log('Dernier dump:', latestDumpName)
+  appendToGithubSummary(`# Dump importé: ${latestDumpName}`)
+
   const { data: dumpBlob, error } = await supabase.storage
     .from('dump_sudocu')
-    .download(dumpFilename)
+    .download(latestDumpName)
 
   if (error) {
-    console.error('❌ Échec du téléchargement de', dumpFilename)
+    console.error('❌ Échec du téléchargement de', latestDumpName)
     console.error(error)
     process.exit(1)
   }
 
   fs.writeFileSync(
-    path.join(dirname, 'sudocuh_dumps', dumpFilename),
+    path.join(dirname, 'sudocuh_dumps', latestDumpName),
     Buffer.from(await dumpBlob.arrayBuffer())
   )
+
+  console.log('⬇️ Téléchargement terminé.')
+
+  return latestDumpName
 }
diff --git a/daily_dump/steps/4-interTablesToDocurbaPlan.mjs b/daily_dump/steps/4-interTablesToDocurbaPlan.mjs
@@ -6,6 +6,7 @@ import DOCUMENTS_TYPES from '../miscs/documentTypes.mjs'
 import PROCEDURES_TYPES from '../miscs/proceduresTypes.mjs'
 import communesReferentiel from '../miscs/referentiels/communes.json' assert {type: 'json'}
 import { updateProcedureSec } from './linkProceduresSecs.mjs'
+import { appendToGithubSummary } from '../common.mjs'
 
 function formatDate(date) {
   const year = date.getFullYear();
@@ -174,6 +175,8 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
   } else {
     console.log('No new projects. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProjects.length} nouveaux projets`)
+
 
   if (addedBufferProcedures.length > 0) {
     const csvNewProcedures = await parser.parse(addedBufferProcedures).promise()
@@ -186,6 +189,7 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
   } else {
     console.log('No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProcedures.length} nouvelles procédures principales`)
 
   // TODO: Get in memory newly added procédures
 
@@ -267,6 +271,7 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
   } else {
     console.log('No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProceduresSec.length} nouvelles procédures secondaires`)
   console.log('End processing for procedures secondaires.')
 
   /// ////////////////////////////////////
@@ -310,6 +315,7 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
   } else {
     console.log('No new perimeters. File wont be written in output')
   }
+  appendToGithubSummary(`- ${perimetersInserted?.length} nouveaux périmètres`)
   console.log('End processing for procedures perimetres.')
 
   /// //////////////////////////////////////////////////////////////////
@@ -361,7 +367,7 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
 //     const { data, error } = await supabase.from('procedures').update({secondary_procedure_of: psUnbinded.secondary_procedure_of}).eq('id', psUnbinded.id)
 //   }
 
-  await updateProcedureSec(proceduresMapping)
+  await updateProcedureSec(proceduresMapping, configSource, configTraget)
 
   /// ////////////////////////////////////////
   /// /////////// UPSERT EVENTS //////////////
@@ -426,6 +432,7 @@ async function sudocuhPlanToDocurba (configSource, configTraget) {
   } else {
     console.log('No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferEvents.length} nouveaux événements`)
   console.log('End processing events.')
 
   return true

diff --git a/daily_dump/steps/5-interTablesToDocurbaScot.mjs b/daily_dump/steps/5-interTablesToDocurbaScot.mjs
@@ -5,7 +5,7 @@ import { createClient } from '@supabase/supabase-js'
 import DOCUMENTS_TYPES from '../miscs/documentTypes.mjs'
 import PROCEDURES_TYPES from '../miscs/proceduresTypes.mjs'
 import communesReferentiel from '../miscs/referentiels/communes.json' assert {type: 'json'}
-
+import { appendToGithubSummary } from '../common.mjs'
 
 
 function formatDate(date) {
@@ -168,6 +168,8 @@ async function sudocuhScotToDocurba (configSource, configTraget) {
   } else {
     console.log('[SCOT] No new projects. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProjects.length} nouveaux projets SCoT`)
+
 
   if (addedBufferProcedures.length > 0) {
     const csvNewProcedures = await parser.parse(addedBufferProcedures).promise()
@@ -180,6 +182,7 @@ async function sudocuhScotToDocurba (configSource, configTraget) {
   } else {
     console.log('[SCOT] No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProcedures.length} nouvealles procédures principales SCoT`)
   console.log('[SCOT] End processing for procedures principales.')
 
   /// ////////////////////////////////////////
@@ -251,6 +254,7 @@ async function sudocuhScotToDocurba (configSource, configTraget) {
   } else {
     console.log('[SCOT] No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferProceduresSec.length} nouvelles procédures secondaires SCoT`)
   console.log('[SCOT] End processing for procedures secondaires.')
 
   /// ////////////////////////////////////
@@ -294,6 +298,7 @@ async function sudocuhScotToDocurba (configSource, configTraget) {
   } else {
     console.log('[SCOT] No new perimeters. File wont be written in output')
   }
+  appendToGithubSummary(`- ${perimetersInserted?.length} nouveaux périmètres SCoT`)
   console.log('[SCOT] End processing for procedures perimetres.')
 
   /// //////////////////////////////////////////////////////////////////
@@ -389,6 +394,7 @@ async function sudocuhScotToDocurba (configSource, configTraget) {
   } else {
     console.log('[SCOT] No new procedures. File wont be written in output')
   }
+  appendToGithubSummary(`- ${addedBufferEvents.length} nouveaux événements SCoT`)
   console.log('[SCOT] End processing events.')
 
   return true

diff --git a/daily_dump/steps/linkProceduresSecs.mjs b/daily_dump/steps/linkProceduresSecs.mjs
@@ -1,6 +1,5 @@
 import fs from 'fs'
 import { createClient } from '@supabase/supabase-js'
-import CONFIG from '../pg_secret_config.mjs'
 
 function formatDate (date) {
   const year = date.getFullYear()
@@ -44,15 +43,15 @@ async function fetchInBatches (supabase, ids, batchSize = 100) {
   return { sudocuIdPpPs, unmatchedIds }
 }
 
-export async function updateProcedureSec (prodProcedures) {
+export async function updateProcedureSec (prodProcedures, configSource, configTraget) {
   console.log('In memory prodProcedures: ', prodProcedures[0])
 
-  const supabaseDev = createClient(CONFIG.PG_DEV_CONFIG.url, CONFIG.PG_DEV_CONFIG.admin_key, {
+  const supabaseDev = createClient(configSource.url, configSource.admin_key, {
     auth: { persistSession: false },
     db: { schema: 'sudocu' }
   })
 
-  const supabase = createClient(CONFIG.PG_PROD_CONFIG.url, CONFIG.PG_PROD_CONFIG.admin_key, {
+  const supabase = createClient(configTraget.url, configTraget.admin_key, {
     auth: { persistSession: false }
   })
 

diff --git a/daily_dump/steps/sqlRunner.mjs b/daily_dump/steps/sqlRunner.mjs
@@ -4,18 +4,14 @@ import { execute } from '@getvim/execute'
 const { Client } = pg
 
 async function clearDev (config) {
-  try {
-    const client = new Client(config)
-    await client.connect()
-    console.log('Clearing Docurba DEV database...')
-    const sql = fs.readFileSync('./daily_dump/steps/sql/miscs/0-clearDevSchema.sql')
-      .toString().replace(/(\r\n|\n|\r)/gm, ' ')
-      .replace(/\s+/g, ' ')
-    await client.query(sql)
-    console.log('Docurba Dev Cleared.')
-  } catch (error) {
-    console.log(error)
-  }
+  const client = new Client(config)
+  await client.connect()
+  console.log('Clearing Docurba DEV database...')
+  const sql = fs.readFileSync('./daily_dump/steps/sql/miscs/0-clearDevSchema.sql')
+    .toString().replace(/(\r\n|\n|\r)/gm, ' ')
+    .replace(/\s+/g, ' ')
+  await client.query(sql)
+  console.log('Docurba Dev Cleared.')
 }
 
 async function loadDump (config, dumpName) {