try run

.github/workflows/deploy.yaml

@@ -91,6 +91,8 @@ jobs:
           curl -o- -L https://slss.io/install | bash
           echo "$HOME/.serverless/bin" >> $GITHUB_PATH
       - name: Deploy with Serverless
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
         run: |
           cd ./deploy
           serverless deploy

projects/scrape-txs/deploy/serverless.yml

@@ -7,6 +7,9 @@ provider:
 functions:
   cronHandler:
     image: ${env:ECR_REPO_URL}:${env:IMAGE_TAG}
+    memorySize: 512
+    environment:
+      DATABASE_URL: ${env:DATABASE_URL}
     events:
       # Invoke Lambda function every day at 7am
       - schedule: cron(0 7 * * ? *)

projects/scrape-txs/src/console.ts

@@ -1,44 +1,51 @@
-import path from 'node:path';
 import dayjs, { Dayjs } from 'dayjs';
 import { program } from 'commander';
 // import { updateYnab } from './lib/ynab';
 import { configSchema } from './lib/config-schema';
 import { bancoIndustrialScrape } from './lib/banco-industrial/scrape';
+import { db } from './lib/db';
 
-const config = configSchema.parse(
-  await Bun.file(path.resolve(__dirname, '../config.json')).json()
-);
+const { data: configJson } = await db
+  .selectFrom('config')
+  .select('data')
+  .where('id', '=', 'general')
+  .executeTakeFirstOrThrow();
+const config = configSchema.parse(configJson);
 
-program.option('-m, --month <months...>', 'Month(s) to scrape');
+console.log('x', config.banks.bancoIndustrialGt);
 
-program.parse();
+// program.option('-m, --month <months...>', 'Month(s) to scrape');
 
-const options = program.opts<{
-  month?: string[];
-}>();
+// program.parse();
 
-const months: Dayjs[] = [];
+// const options = program.opts<{
+//   month?: string[];
+// }>();
 
-if (options.month) {
-  months.push(...options.month.map((month) => dayjs(month)));
-} else {
-  const today = dayjs(new Date());
-  months.unshift(today);
-  if (today.date() <= 10) {
-    months.unshift(today.subtract(1, 'month'));
-  }
-}
+// const months: Dayjs[] = [];
 
-// // console.log('config', config);
+// if (options.month) {
+//   months.push(...options.month.map((month) => dayjs(month)));
+// } else {
+//   const today = dayjs(new Date());
+//   months.unshift(today);
+//   if (today.date() <= 10) {
+//     months.unshift(today.subtract(1, 'month'));
+//   }
+// }
 
-await bancoIndustrialScrape({
-  biConfig: config.banks.bancoIndustrialGt,
-  months,
-});
+// // // console.log('config', config);
 
-// await updateYnab({
-//   ynabConfig: config.ynab,
-//   bankKey: 'bancoIndustrialGt',
-//   bankAccountsWithTransactions: biTransactions,
-//   dryRun: true,
+// await bancoIndustrialScrape({
+//   biConfig: config.banks.bancoIndustrialGt,
+//   months,
 // });
+
+// // await updateYnab({
+// //   ynabConfig: config.ynab,
+// //   bankKey: 'bancoIndustrialGt',
+// //   bankAccountsWithTransactions: biTransactions,
+// //   dryRun: true,
+// // });
+
+await db.destroy();

projects/scrape-txs/src/lambda.ts

@@ -1,10 +1,26 @@
 import type { ScheduledHandler } from 'aws-lambda';
+import dayjs, { type Dayjs } from 'dayjs';
 import { bancoIndustrialScrape } from './lib/banco-industrial/scrape';
+import { db } from './lib/db';
+import { configSchema } from './lib/config-schema';
 
 export const handler: ScheduledHandler = async (_event) => {
-  console.log('hi');
+  const months: Dayjs[] = [];
+  const today = dayjs(new Date());
+  months.unshift(today);
+  if (today.date() <= 10) {
+    months.unshift(today.subtract(1, 'month'));
+  }
+  const { data: configJson } = await db
+    .selectFrom('config')
+    .select('data')
+    .where('id', '=', 'general')
+    .executeTakeFirstOrThrow();
+  const config = configSchema.parse(configJson);
   await bancoIndustrialScrape({
-    // biConfig: config.banks.bancoIndustrialGt,
-    // months,
-  } as any);
+    biConfig: config.banks.bancoIndustrialGt,
+    months,
+    isLambda: true,
+  });
+  await db.destroy();
 };

projects/scrape-txs/src/lib/banco-industrial/scrape.ts

@@ -6,10 +6,6 @@ import Decimal from 'decimal.js';
 import type { AccountType } from '../types';
 import { db, type InsertObject, type DB } from '../db';
 
-function isLambda() {
-  return !!process.env['AWS_LAMBDA_FUNCTION_NAME'];
-}
-
 function waitRandomMs() {
   const randomMilliSeconds =
     Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000;
@@ -23,12 +19,15 @@ function waitRandomMs() {
   });
 }
 
-async function login(auth: {
-  code: string;
-  username: string;
-  password: string;
-}) {
-  const browser = isLambda()
+async function login(
+  auth: {
+    code: string;
+    username: string;
+    password: string;
+  },
+  isLambda: boolean
+) {
+  const browser = isLambda
     ? ((await launchChromium({
         headless: true,
       })) as Browser)
@@ -138,16 +137,18 @@ export type BiConfig = {
 export async function bancoIndustrialScrape({
   biConfig: { auth, accounts },
   months,
+  isLambda,
 }: {
   biConfig: BiConfig;
   months: dayjs.Dayjs[];
+  isLambda: boolean;
 }) {
   console.log(
     `Scraping Banco Industrial GT transactions for months: ${months
       .map((m) => m.format('YYYY-MM'))
       .join(', ')}`
   );
-  const ctx = await login(auth);
+  const ctx = await login(auth, isLambda);
   try {
     const bankTxs: InsertObject<DB, 'bank_txs'>[] = [];
     for (const account of accounts) {

projects/scrape-txs/src/lib/db/codegen.d.ts

@@ -6,6 +6,18 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
 
 export type Int8 = ColumnType<string, bigint | number | string, bigint | number | string>;
 
+export type Json = JsonValue;
+
+export type JsonArray = JsonValue[];
+
+export type JsonObject = {
+  [K in string]?: JsonValue;
+};
+
+export type JsonPrimitive = boolean | number | string | null;
+
+export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
+
 export type Numeric = ColumnType<string, number | string, number | string>;
 
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
@@ -23,6 +35,13 @@ export interface BankTxs {
   tx_key: string;
 }
 
+export interface Config {
+  created_at: Generated<Timestamp>;
+  data: Json;
+  id: string;
+}
+
 export interface DB {
   bank_txs: BankTxs;
+  config: Config;
 }