Skip to content

Commit

Permalink
remove tx_key
Browse files Browse the repository at this point in the history
  • Loading branch information
rhyek committed Mar 23, 2024
1 parent 0c84f0d commit 93f2c6a
Show file tree
Hide file tree
Showing 12 changed files with 150 additions and 396 deletions.
4 changes: 2 additions & 2 deletions projects/scrape-txs/deploy/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ functions:
events:
# Invoke Lambda function every day at 7am Guatemala time
- schedule:
method: scheduler
method: scheduler # use AWS::Scheduler::Schedule (EventBridge rule) instead of AWS::Events::Rule (CloudWatch Events rule)
rate: cron(0 7 * * ? *)
timezone: America/Guatemala
timezone: America/Guatemala # possible with AWS::Scheduler::Schedule
6 changes: 6 additions & 0 deletions projects/scrape-txs/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion projects/scrape-txs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
"name": "scrape-txs",
"type": "module",
"scripts": {
"build": "tsc --project tsconfig.build.json && tsc-alias -p tsconfig.build.json"
"build": "tsc --project tsconfig.build.json && tsc-alias -p tsconfig.build.json",
"start": "node ./dist/console.js"
},
"devDependencies": {
"@tsconfig/node20": "~20.1.2",
Expand All @@ -24,6 +25,7 @@
"pg": "^8.11.3",
"playwright": "1.41.1",
"playwright-aws-lambda": "^0.10.0",
"ts-pattern": "^5.0.8",
"ynab": "^2.2.0",
"zod": "^3.22.4"
}
Expand Down
192 changes: 138 additions & 54 deletions projects/scrape-txs/src/lib/banco-industrial/scrape.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import type dayjs from 'dayjs';
import dayjs from 'dayjs';
import { chromium, type Browser, type Page } from 'playwright';
import { launchChromium } from 'playwright-aws-lambda';
import lodash from 'lodash';
import Decimal from 'decimal.js';
import { isMatching } from 'ts-pattern';
// import lodash from 'lodash';
// import Decimal from 'decimal.js';
import type { AccountType } from '../types';
import { db, type InsertObject, type DB } from '../db';

Expand Down Expand Up @@ -142,6 +143,7 @@ export async function bancoIndustrialScrape({
biConfig: BiConfig;
months: dayjs.Dayjs[];
}) {
const bankKey = 'bancoIndustrialGt';
console.log(
`Scraping Banco Industrial GT transactions for months: ${months
.map((m) => m.format('YYYY-MM'))
Expand All @@ -150,69 +152,151 @@ export async function bancoIndustrialScrape({
const ctx = await login(auth);
try {
const bankTxs: InsertObject<DB, 'bank_txs'>[] = [];
const deleteTxIds: string[] = [];
for (const account of accounts) {
if (account.type === 'checking') {
for (const monthDayJs of months) {
const currentTxs = await db
.selectFrom('bank_txs')
.selectAll()
.where('bank_key', '=', bankKey)
.where('account_number', '=', account.number)
.where('month', '=', monthDayJs.format('YYYY-MM'))
.execute();
const rawTransactions = await getMonetaryAccountTransactions(
ctx.page,
account.number,
monthDayJs
);
bankTxs.push(
...Object.values(
lodash.groupBy(
rawTransactions.map((tx) => {
const [_, dateStr] = tx.date.match(/(\d\d)\s-\s(\d\d)/)!;
const amount =
tx.credit && tx.credit !== ''
? Number(tx.credit)
: -Number(tx.debit);
return {
bank_key: 'bancoIndustrialGt',
tx_key: `bi.gt-${account.number}-${monthDayJs.format(
'YYYYMMDD'
)}-${tx.docNo}`,
account_number: account.number,
month: monthDayJs.format('YYYY-MM'),
date: monthDayJs.date(Number(dateStr)).format('YYYY-MM-DD'),
description: tx.description,
doc_no: tx.docNo,
amount,
};
}),
(tx) => tx.tx_key
)
).map((txs) => ({
...txs[0],
description:
txs.find(
(tx) => tx.description !== 'CONSUMO TARJETA ELECTRON VISA'
)?.description ?? txs[0].description,
amount: txs
.reduce((acc, tx) => acc.add(tx.amount), new Decimal(0))
.toDecimalPlaces(2)
.toNumber(),
}))
);
const _bankTxs = rawTransactions.map((tx) => {
const [_, dateStr] = tx.date.match(/(\d\d)\s-\s(\d\d)/)!;
const amount =
tx.credit && tx.credit !== ''
? Number(tx.credit)
: -Number(tx.debit);
return {
bank_key: 'bancoIndustrialGt',
account_number: account.number,
month: monthDayJs.format('YYYY-MM'),
date: monthDayJs.date(Number(dateStr)).format('YYYY-MM-DD'),
description: tx.description,
doc_no: tx.docNo,
amount,
};
});
const _deleteTxIds = currentTxs
.filter((currentTx) => {
const objToMatch = {
bank_key: currentTx.bank_key,
account_number: currentTx.account_number,
date: dayjs(currentTx.date).format('YYYY-MM-DD'),
doc_no: currentTx.doc_no,
description: currentTx.description,
amount: Number(currentTx.amount),
};
return !_bankTxs.some((bankTx) => isMatching(objToMatch, bankTx));
})
.map((tx) => tx.id);
bankTxs.push(..._bankTxs);
deleteTxIds.push(..._deleteTxIds);
// bankTxs.push(
// ...rawTransactions.map((tx) => {
// const [_, dateStr] = tx.date.match(/(\d\d)\s-\s(\d\d)/)!;
// const amount =
// tx.credit && tx.credit !== ''
// ? Number(tx.credit)
// : -Number(tx.debit);
// return {
// bank_key: 'bancoIndustrialGt',
// tx_key: `bi.gt-${account.number}-${monthDayJs.format(
// 'YYYYMMDD'
// )}-${tx.docNo}-${tx.description}-${amount}`,
// account_number: account.number,
// month: monthDayJs.format('YYYY-MM'),
// date: monthDayJs.date(Number(dateStr)).format('YYYY-MM-DD'),
// description: tx.description,
// doc_no: tx.docNo,
// amount,
// };
// })
// );
// bankTxs.push(
// ...Object.values(
// lodash.groupBy(
// rawTransactions.map((tx) => {
// const [_, dateStr] = tx.date.match(/(\d\d)\s-\s(\d\d)/)!;
// const amount =
// tx.credit && tx.credit !== ''
// ? Number(tx.credit)
// : -Number(tx.debit);
// return {
// bank_key: 'bancoIndustrialGt',
// tx_key: `bi.gt-${account.number}-${monthDayJs.format(
// 'YYYYMMDD'
// )}-${tx.docNo}`,
// account_number: account.number,
// month: monthDayJs.format('YYYY-MM'),
// date: monthDayJs.date(Number(dateStr)).format('YYYY-MM-DD'),
// description: tx.description,
// doc_no: tx.docNo,
// amount,
// };
// }),
// (tx) => tx.tx_key
// )
// ).map((txs) => ({
// ...txs[0],
// description:
// txs.find(
// (tx) => tx.description !== 'CONSUMO TARJETA ELECTRON VISA'
// )?.description ?? txs[0].description,
// amount: txs
// .reduce((acc, tx) => acc.add(tx.amount), new Decimal(0))
// .toDecimalPlaces(2)
// .toNumber(),
// }))
// );
}
}
}
if (bankTxs.length > 0) {
console.log(
`Inserting/updating ${bankTxs.length} Banco Industrial GT transactions...`
);
if (bankTxs.length > 0 || deleteTxIds.length > 0) {
await db.transaction().execute(async (sqlTx) => {
await Promise.all(
bankTxs.map(async (bankTx) => {
await sqlTx
.insertInto('bank_txs')
.values(bankTx)
.onConflict((oc) =>
oc.column('tx_key').doUpdateSet({ amount: bankTx.amount })
)
.execute();
})
);
if (bankTxs.length > 0) {
console.log(
`Inserting/updating ${bankTxs.length} Banco Industrial GT transactions...`
);
await Promise.all(
bankTxs.map(async (bankTx) => {
await sqlTx
.insertInto('bank_txs')
.values(bankTx)
.onConflict((oc) =>
oc
.columns([
'bank_key',
'account_number',
'date',
'doc_no',
'description',
'amount',
])
.doUpdateSet({ amount: bankTx.amount })
)
.execute();
})
);
}
if (deleteTxIds.length > 0) {
console.log(
`Deleting ${deleteTxIds.join(
', '
)} Banco Industrial GT transactions...`
);
await sqlTx
.deleteFrom('bank_txs')
.where('id', 'in', deleteTxIds)
.execute();
}
});
console.log('Done.');
}
Expand Down
1 change: 0 additions & 1 deletion projects/scrape-txs/src/lib/db/codegen.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ export interface BankTxs {
doc_no: string;
id: Generated<Int8>;
month: string;
tx_key: string;
}

export interface Config {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ cd $__dirname

bun kysely-codegen \
--include-pattern="public.*" \
--out-file src/lib/db/codegen.d.ts
--out-file ./codegen.d.ts
4 changes: 0 additions & 4 deletions supabase/.gitignore

This file was deleted.

Loading

0 comments on commit 93f2c6a

Please sign in to comment.