From 6b6422092658175606e84f6b644d18be652f622c Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 08:49:40 -0500 Subject: [PATCH] chore: remove postgres experiment (#1061) I was trying out postgres to see if it would be a good alternative to redis. Unfortunately its slower so its not a good solution. This PR removes the experimental code. --- api/_initdb.ts | 93 ------------------ package-lock.json | 162 +------------------------------- package.json | 1 - src/page-props/common.ts | 3 +- src/util/backend/db-postgres.ts | 56 ----------- src/util/backend/db.ts | 10 -- 6 files changed, 4 insertions(+), 321 deletions(-) delete mode 100644 api/_initdb.ts delete mode 100644 src/util/backend/db-postgres.ts diff --git a/api/_initdb.ts b/api/_initdb.ts deleted file mode 100644 index b5bd914f..00000000 --- a/api/_initdb.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { sql } from '@vercel/postgres'; -import { findAll } from '../src/util/backend/db-redis'; -import type { IncomingMessage, ServerResponse } from 'http'; - -export default async function handler(req: IncomingMessage, res: ServerResponse) { - // This is a temporary function we can use to test - if (process.env.VERCEL_ENV !== 'development') { - res.statusCode = 403; - res.end('403 Forbidden'); - return; - } - - /* - console.log(await sql` - CREATE COLLATION semver ( - LOCALE = 'en-US-u-kn-true', - PROVIDER = 'icu' - ); - `); - - console.log(await sql`drop table if exists "packages";`); - - console.log( - await sql` - CREATE TABLE "packages" ( - "name" VARCHAR(214), - "version" VARCHAR(255) COLLATE semver, - "publishSize" INTEGER, - "installSize" INTEGER, - "publishFiles" INTEGER, - "installFiles" INTEGER, - PRIMARY KEY ("name", "version") - ); -`, - ); -*/ - - const url = new URL(req.url ?? '/', 'http://example.com'); - const pkgName = url.searchParams.get('p'); - if (pkgName) { - const result = await findAll(pkgName); - console.log(`inserting ${Object.keys(result).length} rows`); - - for (let pkg of Object.values(result)) { - console.log(`inserting row ${pkg.name}@${pkg.version}`); - try { - await sql`INSERT INTO "packages" values (${pkg.name}, ${pkg.version}, ${pkg.publishSize}, ${pkg.installSize}, ${pkg.publishFiles}, ${pkg.installFiles});`; - } catch (error) { - if (String(error).includes('duplicate key value')) { - console.log('skipping duplicate key', pkg.name, pkg.version); - } else { - throw error; - } - } - } - console.log('inserting complete!\n'); - // console.log(await sql`SELECT * FROM "packages" ORDER BY version desc;`); - } else { - console.log('Missing "p" query parameter so no packages were inserted'); - } - res.end('success'); -} - -/** - CREATE COLLATION en_natural ( - LOCALE = 'en-US-u-kn-true', - PROVIDER = 'icu' -); - -CREATE TABLE test ( - version varchar(20) collate en_natural -); - -INSERT INTO test values - ('14.1.0'), - ('14.20.0'), - ('14.11.0'), - ('14.10.0'), - ('14.2.0'), - ('14.0.2'), - ('14.0.3-canary.1'), - ('14.0.3-canary.0'), - ('14.0.3-canary.9'), - ('14.0.3-canary.10'), - ('14.0.3-canary.11'), - ('14.0.3-canary.12'), - ('14.0.3'); - -SELECT split_part(version, '-', 1) as one, NULLIF(split_part(version, '-', 2), '') as two -FROM test -ORDER BY split_part(version, '-', 1) desc, NULLIF(split_part(version, '-', 2), '') desc; - - */ diff --git a/package-lock.json b/package-lock.json index fd33fe65..e984a5af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,6 @@ "name": "packagephobia", "license": "MIT", "dependencies": { - "@vercel/postgres": "^0.8.0", "badgen": "^3.2.3", "ioredis": "^5.4.1", "lru-cache": "^10.2.0", @@ -33,32 +32,15 @@ "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, - "node_modules/@neondatabase/serverless": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.7.2.tgz", - "integrity": "sha512-wU3WA2uTyNO7wjPs3Mg0G01jztAxUxzd9/mskMmtPwPTjf7JKWi9AW5/puOGXLxmZ9PVgRFeBVRVYq5nBPhsCg==", - "dependencies": { - "@types/pg": "8.6.6" - } - }, "node_modules/@types/node": { "version": "20.12.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz", "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", + "dev": true, "dependencies": { "undici-types": "~5.26.4" } }, - "node_modules/@types/pg": { - "version": "8.6.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz", - "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, "node_modules/@types/prop-types": { "version": "15.7.12", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz", @@ -90,37 +72,11 @@ "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, - "node_modules/@vercel/postgres": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@vercel/postgres/-/postgres-0.8.0.tgz", - "integrity": "sha512-/QUV9ExwaNdKooRjOQqvrKNVnRvsaXeukPNI5DB1ovUTesglfR/fparw7ngo1KUWWKIVpEj2TRrA+ObRHRdaLg==", - "dependencies": { - "@neondatabase/serverless": "0.7.2", - "bufferutil": "4.0.8", - "utf-8-validate": "6.0.3", - "ws": "8.14.2" - }, - "engines": { - "node": ">=14.6" - } - }, "node_modules/badgen": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/badgen/-/badgen-3.2.3.tgz", "integrity": "sha512-svDuwkc63E/z0ky3drpUppB83s/nlgDciH9m+STwwQoWyq7yCgew1qEfJ+9axkKdNq7MskByptWUN9j1PGMwFA==" }, - "node_modules/bufferutil": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.8.tgz", - "integrity": "sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==", - "hasInstallScript": true, - "dependencies": { - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=6.14.2" - } - }, "node_modules/cluster-key-slot": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", @@ -221,16 +177,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/node-gyp-build": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.0.tgz", - "integrity": "sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/npm": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/npm/-/npm-10.4.0.tgz", @@ -2977,69 +2923,6 @@ "inBundle": true, "license": "ISC" }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/prettier": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", @@ -3151,47 +3034,8 @@ "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, - "node_modules/utf-8-validate": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-6.0.3.tgz", - "integrity": "sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==", - "hasInstallScript": true, - "dependencies": { - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=6.14.2" - } - }, - "node_modules/ws": { - "version": "8.14.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", - "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "engines": { - "node": ">=0.4" - } + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true }, "node_modules/yallist": { "version": "4.0.0", diff --git a/package.json b/package.json index e5bd2a93..9a6bc553 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,6 @@ "author": "styfle", "license": "MIT", "dependencies": { - "@vercel/postgres": "^0.8.0", "badgen": "^3.2.3", "ioredis": "^5.4.1", "lru-cache": "^10.2.0", diff --git a/src/page-props/common.ts b/src/page-props/common.ts index be16e21b..a9d2141f 100644 --- a/src/page-props/common.ts +++ b/src/page-props/common.ts @@ -1,5 +1,4 @@ import { findOne } from '../util/backend/db'; -import { insert as insertPostgres } from '../util/backend/db-postgres'; import { insert as insertRedis } from '../util/backend/db-redis'; import { getAllDistTags } from '../util/npm-api'; import { calculatePackageSize } from '../util/backend/npm-stats'; @@ -45,7 +44,7 @@ export async function getPkgDetails( const end = new Date(); const sec = (end.getTime() - start.getTime()) / 1000; console.log(`Calculated size of ${name}@${version} in ${sec}s`); - await Promise.all([insertRedis(pkgSize), insertPostgres(pkgSize)]); + await insertRedis(pkgSize); } const result = { diff --git a/src/util/backend/db-postgres.ts b/src/util/backend/db-postgres.ts deleted file mode 100644 index b4f6029c..00000000 --- a/src/util/backend/db-postgres.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { sql } from '@vercel/postgres'; -import type { PkgSize } from '../../types'; - -export async function findAll(name: string) { - console.time('findAll (postgres)'); - const { rows } = await sql` - SELECT * - FROM "packages" - WHERE name = ${name}; - `; - console.timeEnd('findAll (postgres)'); - const obj: { [key: string]: PkgSize } = {}; - for (let row of rows) { - const payload = row as PkgSize; - obj[payload.version] = payload; - } - return obj; -} - -export async function findOne(name: string, version: string) { - console.time('findOne (postgres)'); - const { rows } = await sql` - SELECT * - FROM "packages" - WHERE name = ${name} - AND version = ${version}; - `; - console.timeEnd('findOne (postgres)'); - const reply = rows[0]; - - if (!reply) { - return null; - } - - return reply as PkgSize; -} - -export async function insert({ - name, - version, - publishSize, - installSize, - publishFiles, - installFiles, -}: PkgSize) { - console.time('insert (postgres)'); - const reply = await sql` - INSERT INTO "packages" - VALUES (${name}, ${version}, ${publishSize}, ${installSize}, ${publishFiles}, ${installFiles}) - ON CONFLICT ("name", "version") - DO UPDATE SET "publishSize" = ${publishSize}, "installSize" = ${installSize}, "publishFiles" = ${publishFiles}, "installFiles" = ${installFiles} - ; - `; - console.timeEnd('insert (postgres)'); - return reply; -} diff --git a/src/util/backend/db.ts b/src/util/backend/db.ts index 98832f9b..7fba47ce 100644 --- a/src/util/backend/db.ts +++ b/src/util/backend/db.ts @@ -1,24 +1,14 @@ import * as redis from './db-redis'; -import * as postgres from './db-postgres'; import type { PkgSize } from '../../types'; export async function findAll(name: string) { - if (process.env.TRY_POSTGRES === '1') { - return postgres.findAll(name); - } return redis.findAll(name); } export async function findOne(name: string, version: string) { - if (process.env.TRY_POSTGRES === '1') { - return postgres.findOne(name, version); - } return redis.findOne(name, version); } export async function insert(pkg: PkgSize) { - if (process.env.TRY_POSTGRES === '1') { - return postgres.insert(pkg); - } return redis.insert(pkg); }