Support input url option for cli

frontend/packages/cli/src/cli/getInputContent.test.ts

@@ -0,0 +1,80 @@
+import fs from 'node:fs'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { getInputContent } from './getInputContent.js'
+
+vi.mock('node:fs')
+vi.mock('node:https')
+
+describe('getInputContent', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('should read local file content when given a valid file path', async () => {
+    const mockFilePath = '/path/to/local/file.txt'
+    const mockFileContent = 'Local file content'
+
+    vi.spyOn(fs, 'existsSync').mockReturnValue(true)
+    vi.spyOn(fs, 'readFileSync').mockReturnValue(mockFileContent)
+
+    const content = await getInputContent(mockFilePath)
+
+    expect(content).toBe(mockFileContent)
+  })
+
+  it('should throw an error if the local file path is invalid', async () => {
+    const mockFilePath = '/invalid/path/to/file.txt'
+
+    vi.spyOn(fs, 'existsSync').mockReturnValue(false)
+
+    await expect(getInputContent(mockFilePath)).rejects.toThrow(
+      'Invalid input path. Please provide a valid file.',
+    )
+  })
+
+  it('should download raw content from GitHub when given a GitHub blob URL', async () => {
+    const mockGitHubUrl = 'https://github.com/user/repo/blob/main/file.txt'
+    const mockRawUrl =
+      'https://raw.githubusercontent.com/user/repo/main/file.txt'
+    const mockGitHubContent = 'GitHub raw file content'
+
+    const mockFetch = vi
+      .spyOn(global, 'fetch')
+      .mockImplementation(
+        async () => new Response(mockGitHubContent, { status: 200 }),
+      )
+
+    const content = await getInputContent(mockGitHubUrl)
+
+    expect(content).toBe(mockGitHubContent)
+    expect(mockFetch).toHaveBeenCalledWith(mockRawUrl)
+  })
+
+  it('should download content from a regular URL', async () => {
+    const mockUrl = 'https://example.com/file.txt'
+    const mockUrlContent = 'Regular URL file content'
+
+    const mockFetch = vi
+      .spyOn(global, 'fetch')
+      .mockImplementation(
+        async () => new Response(mockUrlContent, { status: 200 }),
+      )
+
+    const content = await getInputContent(mockUrl)
+
+    expect(content).toBe(mockUrlContent)
+    expect(mockFetch).toHaveBeenCalledWith(mockUrl)
+  })
+
+  it('should throw an error when file download fails', async () => {
+    const mockUrl = 'https://example.com/file.txt'
+
+    vi.spyOn(global, 'fetch').mockImplementation(
+      async () => new Response('', { status: 404, statusText: 'Not Found' }),
+    )
+
+    await expect(getInputContent(mockUrl)).rejects.toThrow(
+      'Failed to download file: Not Found',
+    )
+  })
+})

frontend/packages/cli/src/cli/getInputContent.ts

@@ -0,0 +1,49 @@
+import fs from 'node:fs'
+import { URL } from 'node:url'
+
+function isValidUrl(url: string): boolean {
+  try {
+    new URL(url)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function isGitHubFileUrl(url: string): boolean {
+  const parsedUrl = new URL(url)
+  return parsedUrl.hostname === 'github.com' && url.includes('/blob/')
+}
+
+function readLocalFile(filePath: string): string {
+  if (!fs.existsSync(filePath)) {
+    throw new Error('Invalid input path. Please provide a valid file.')
+  }
+  return fs.readFileSync(filePath, 'utf8')
+}
+
+async function downloadGitHubRawContent(githubUrl: string): Promise<string> {
+  const rawFileUrl = githubUrl
+    .replace('github.com', 'raw.githubusercontent.com')
+    .replace('/blob', '')
+  return await downloadFile(rawFileUrl)
+}
+
+async function downloadFile(url: string): Promise<string> {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`Failed to download file: ${response.statusText}`)
+  }
+  const data = await response.text()
+  return data
+}
+
+export async function getInputContent(inputPath: string): Promise<string> {
+  if (!isValidUrl(inputPath)) {
+    return readLocalFile(inputPath)
+  }
+
+  return isGitHubFileUrl(inputPath)
+    ? await downloadGitHubRawContent(inputPath)
+    : await downloadFile(inputPath)
+}

frontend/packages/cli/src/cli/runPreprocess.test.ts

@@ -47,15 +47,6 @@ describe('runPreprocess', () => {
     },
   )
 
-  it('should throw an error if the input file does not exist', async () => {
-    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-distDir-'))
-    const nonExistentPath = path.join(tmpDir, 'non-existent.sql')
-
-    await expect(
-      runPreprocess(nonExistentPath, tmpDir, 'postgres'),
-    ).rejects.toThrow('Invalid input path. Please provide a valid file.')
-  })
-
   it('should throw an error if the format is invalid', async () => {
     const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-distDir-'))
     const inputPath = path.join(tmpDir, 'input.sql')

frontend/packages/cli/src/cli/runPreprocess.ts

@@ -1,22 +1,19 @@
-import fs, { readFileSync } from 'node:fs'
+import fs from 'node:fs'
 import path from 'node:path'
 import {
   type SupportedFormat,
   parse,
   supportedFormatSchema,
 } from '@liam-hq/db-structure/parser'
 import * as v from 'valibot'
+import { getInputContent } from './getInputContent.js'
 
 export async function runPreprocess(
   inputPath: string,
   outputDir: string,
   format: SupportedFormat,
 ) {
-  if (!fs.existsSync(inputPath)) {
-    throw new Error('Invalid input path. Please provide a valid file.')
-  }
-
-  const input = readFileSync(inputPath, 'utf8')
+  const input = await getInputContent(inputPath)
 
   if (!v.safeParse(supportedFormatSchema, format).success) {
     throw new Error(