From 862b552184f1ebe25c807123441ae62b56f56acb Mon Sep 17 00:00:00 2001 From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:37:16 +0100 Subject: [PATCH] JWT auth support (#360) --- .github/workflows/tests.yml | 9 + .scripts/generate_cloud_jwt.ts | 7 + karma.config.jwt.cjs | 40 +++++ package.json | 3 + .../__tests__/integration/auth.test.ts | 24 +-- .../__tests__/unit/config.test.ts | 169 ++++++++++++++---- .../client-common/__tests__/utils/client.ts | 2 +- packages/client-common/__tests__/utils/env.ts | 2 + .../client-common/src/clickhouse_types.ts | 20 +++ packages/client-common/src/client.ts | 23 +-- packages/client-common/src/config.ts | 63 ++++++- packages/client-common/src/connection.ts | 9 +- packages/client-common/src/index.ts | 7 +- .../client-common/src/utils/connection.ts | 15 ++ .../integration/node_jwt_auth.test.ts | 50 ++++++ .../__tests__/unit/node_client.test.ts | 3 +- .../__tests__/unit/node_config.test.ts | 7 +- .../unit/node_create_connection.test.ts | 39 ++-- .../client-node/__tests__/utils/http_stubs.ts | 8 +- packages/client-node/__tests__/utils/jwt.ts | 18 ++ .../src/connection/node_base_connection.ts | 38 ++-- .../src/connection/node_https_connection.ts | 27 ++- packages/client-node/src/index.ts | 3 + .../__tests__/jwt/web_jwt_auth.test.ts | 52 ++++++ .../src/connection/web_connection.ts | 31 +++- packages/client-web/src/index.ts | 3 + webpack.dev.js | 1 + 27 files changed, 555 insertions(+), 118 deletions(-) create mode 100644 .scripts/generate_cloud_jwt.ts create mode 100644 karma.config.jwt.cjs create mode 100644 packages/client-node/__tests__/integration/node_jwt_auth.test.ts create mode 100644 packages/client-node/__tests__/utils/jwt.ts create mode 100644 packages/client-web/__tests__/jwt/web_jwt_auth.test.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58af1093..2c591fbb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -217,6 +217,7 @@ jobs: env: CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }} CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }} + CLICKHOUSE_CLOUD_JWT_SECRET: ${{ secrets.CI_JWT_SIGNING_PRIVATE_KEY }} run: | npm run test:node:integration:cloud_smt @@ -242,6 +243,14 @@ jobs: run: | npm run test:web:integration:cloud_smt + - name: Run JWT auth integration tests + env: + CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }} + CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }} + CLICKHOUSE_CLOUD_JWT_SECRET: ${{ secrets.CI_JWT_SIGNING_PRIVATE_KEY }} + run: | + npm run test:web:integration:cloud_smt:jwt + # With unit + integration + TLS tests + coverage + SonarCloud report, after the rest of the tests. # Needs all integration tests on all environments to pass. # Should use only the current LTS version of Node.js. diff --git a/.scripts/generate_cloud_jwt.ts b/.scripts/generate_cloud_jwt.ts new file mode 100644 index 00000000..fb149f09 --- /dev/null +++ b/.scripts/generate_cloud_jwt.ts @@ -0,0 +1,7 @@ +import { makeJWT } from '../packages/client-node/__tests__/utils/jwt' + +/** Used to generate a JWT token for web testing (can't use `jsonwebtoken` library directly there) + * See `package.json` -> `scripts` -> `test:web:integration:cloud_smt:jwt` */ +;(() => { + console.log(makeJWT()) +})() diff --git a/karma.config.jwt.cjs b/karma.config.jwt.cjs new file mode 100644 index 00000000..512478db --- /dev/null +++ b/karma.config.jwt.cjs @@ -0,0 +1,40 @@ +const webpackConfig = require('./webpack.dev.js') + +const TEST_TIMEOUT_MS = 400_000 + +module.exports = function (config) { + config.set({ + // base path that will be used to resolve all patterns (e.g. files, exclude) + basePath: '', + frameworks: ['webpack', 'jasmine'], + // list of files / patterns to load in the browser + files: ['packages/client-web/__tests__/jwt/*.test.ts'], + exclude: [], + webpack: webpackConfig, + preprocessors: { + 'packages/client-common/**/*.ts': ['webpack', 'sourcemap'], + 'packages/client-web/**/*.ts': ['webpack', 'sourcemap'], + 'packages/client-common/__tests__/jwt/*.ts': ['webpack', 'sourcemap'], + 'packages/client-common/__tests__/utils/*.ts': ['webpack', 'sourcemap'], + }, + reporters: ['mocha'], + port: 9876, + colors: true, + logLevel: config.LOG_INFO, + autoWatch: false, + // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher + browsers: ['ChromeHeadless', 'FirefoxHeadless'], + browserNoActivityTimeout: TEST_TIMEOUT_MS, + browserDisconnectTimeout: TEST_TIMEOUT_MS, + // if true, Karma captures browsers, runs the tests and exits + singleRun: true, + client: { + jasmine: { + random: false, + stopOnSpecFailure: false, + stopSpecOnExpectationFailure: true, + timeoutInterval: TEST_TIMEOUT_MS, + }, + }, + }) +} diff --git a/package.json b/package.json index 2b940f57..936318be 100644 --- a/package.json +++ b/package.json @@ -40,12 +40,14 @@ "test:web:integration:local_cluster": "CLICKHOUSE_TEST_ENVIRONMENT=local_cluster npm run test:web", "test:web:integration:cloud": "CLICKHOUSE_TEST_ENVIRONMENT=cloud npm run test:web", "test:web:integration:cloud_smt": "CLICKHOUSE_TEST_ENVIRONMENT=cloud_smt npm run test:web", + "test:web:integration:cloud_smt:jwt": "CLICKHOUSE_CLOUD_JWT_ACCESS_TOKEN=`ts-node .scripts/generate_cloud_jwt.ts` karma start karma.config.jwt.cjs", "prepare": "husky" }, "devDependencies": { "@faker-js/faker": "^9.0.2", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@types/jasmine": "^5.1.4", + "@types/jsonwebtoken": "^9.0.7", "@types/node": "^22.7.0", "@types/sinon": "^17.0.3", "@types/split2": "^4.2.3", @@ -61,6 +63,7 @@ "jasmine": "^5.3.0", "jasmine-core": "^5.3.0", "jasmine-expect": "^5.0.0", + "jsonwebtoken": "^9.0.2", "karma": "^6.4.4", "karma-chrome-launcher": "^3.2.0", "karma-firefox-launcher": "^2.1.3", diff --git a/packages/client-common/__tests__/integration/auth.test.ts b/packages/client-common/__tests__/integration/auth.test.ts index 0aa2c8cc..5103260f 100644 --- a/packages/client-common/__tests__/integration/auth.test.ts +++ b/packages/client-common/__tests__/integration/auth.test.ts @@ -4,20 +4,22 @@ import { getAuthFromEnv } from '@test/utils/env' import { createTestClient, guid } from '../utils' describe('authentication', () => { - let client: ClickHouseClient + let invalidAuthClient: ClickHouseClient beforeEach(() => { - client = createTestClient({ - username: 'gibberish', - password: 'gibberish', + invalidAuthClient = createTestClient({ + auth: { + username: 'gibberish', + password: 'gibberish', + }, }) }) afterEach(async () => { - await client.close() + await invalidAuthClient.close() }) it('provides authentication error details', async () => { await expectAsync( - client.query({ + invalidAuthClient.query({ query: 'SELECT number FROM system.numbers LIMIT 3', }), ).toBeRejectedWith( @@ -51,13 +53,13 @@ describe('authentication', () => { it('should with with insert and select', async () => { tableName = `simple_table_${guid()}` await createSimpleTable(defaultClient, tableName) - await client.insert({ + await invalidAuthClient.insert({ table: tableName, format: 'JSONEachRow', values, auth, }) - const rs = await client.query({ + const rs = await invalidAuthClient.query({ query: `SELECT * FROM ${tableName} ORDER BY id ASC`, format: 'JSONEachRow', auth, @@ -68,11 +70,11 @@ describe('authentication', () => { it('should work with command and select', async () => { tableName = `simple_table_${guid()}` await createSimpleTable(defaultClient, tableName) - await client.command({ + await invalidAuthClient.command({ query: `INSERT INTO ${tableName} VALUES (1, 'foo', [3, 4])`, auth, }) - const rs = await client.query({ + const rs = await invalidAuthClient.query({ query: `SELECT * FROM ${tableName} ORDER BY id ASC`, format: 'JSONEachRow', auth, @@ -81,7 +83,7 @@ describe('authentication', () => { }) it('should work with exec', async () => { - const { stream } = await client.exec({ + const { stream } = await invalidAuthClient.exec({ query: 'SELECT 42, 144 FORMAT CSV', auth, }) diff --git a/packages/client-common/__tests__/unit/config.test.ts b/packages/client-common/__tests__/unit/config.test.ts index f6bd94ea..7153ea24 100644 --- a/packages/client-common/__tests__/unit/config.test.ts +++ b/packages/client-common/__tests__/unit/config.test.ts @@ -96,8 +96,7 @@ describe('config', () => { pathname: '/my_proxy', request_timeout: 42_000, max_open_connections: 144, - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, database: 'analytics', http_headers: { 'X-CLICKHOUSE-AUTH': 'secret_header', @@ -124,8 +123,10 @@ describe('config', () => { pathname: '/my_proxy', request_timeout: 42_000, max_open_connections: 144, - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + }, database: 'analytics', http_headers: { 'X-CLICKHOUSE-AUTH': 'secret_header', @@ -178,6 +179,60 @@ describe('config', () => { }) // should not be modified }) + it('should be able to use the deprecated username parameter', async () => { + const deprecated: BaseClickHouseClientConfigOptions = { + username: 'bob', + } + const res = prepareConfigWithURL(deprecated, logger, null) + expect(res).toEqual({ + ...defaultConfig, + auth: { + username: 'bob', + // to be "normalized" later by `getConnectionParams` + password: undefined, + }, + }) + expect(deprecated).toEqual({ + username: 'bob', + }) // should not be modified + }) + + it('should be able to use the deprecated password parameter', async () => { + const deprecated: BaseClickHouseClientConfigOptions = { + password: 'secret', + } + const res = prepareConfigWithURL(deprecated, logger, null) + expect(res).toEqual({ + ...defaultConfig, + auth: { + username: undefined, + password: 'secret', + }, + }) + expect(deprecated).toEqual({ + password: 'secret', + }) // should not be modified + }) + + it('should be able to use both deprecated username and password parameters', async () => { + const deprecated: BaseClickHouseClientConfigOptions = { + username: 'bob', + password: 'secret', + } + const res = prepareConfigWithURL(deprecated, logger, null) + expect(res).toEqual({ + ...defaultConfig, + auth: { + username: 'bob', + password: 'secret', + }, + }) + expect(deprecated).toEqual({ + username: 'bob', + password: 'secret', + }) // should not be modified + }) + // tested more thoroughly in the loadConfigOptionsFromURL section; // this is just a validation that everything works together it('should use settings from the URL', async () => { @@ -202,8 +257,10 @@ describe('config', () => { expect(res).toEqual({ ...defaultConfig, url: new URL('https://my.host:8443/'), - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + }, database: 'analytics', application: 'my_app', impl_specific_setting: 42, @@ -298,6 +355,42 @@ describe('config', () => { }) }) + describe('Credentials vs JWT auth parsing', () => { + it('should correctly get JWT access_token from the URL', async () => { + const url = new URL( + 'https://my.host:8443/analytics?' + + ['application=my_app', 'access_token=jwt_secret'].join('&'), + ) + const res = prepareConfigWithURL({ url }, logger, null) + expect(res).toEqual({ + ...defaultConfig, + url: new URL('https://my.host:8443'), + database: 'analytics', + application: 'my_app', + auth: { + access_token: 'jwt_secret', + }, + } as unknown as BaseClickHouseClientConfigOptionsWithURL) + }) + + it('should correctly override credentials auth with JWT if both are present', async () => { + const url = new URL( + 'https://bob:secret@my.host:8443/analytics?' + + ['application=my_app', 'access_token=jwt_secret'].join('&'), + ) + const res = prepareConfigWithURL({ url }, logger, null) + expect(res).toEqual({ + ...defaultConfig, + url: new URL('https://my.host:8443'), + database: 'analytics', + application: 'my_app', + auth: { + access_token: 'jwt_secret', + }, + } as unknown as BaseClickHouseClientConfigOptionsWithURL) + }) + }) + // more detailed tests are in the createUrl section it('should throw when the URL is not valid', async () => { expect(() => prepareConfigWithURL({ url: 'foo' }, logger, null)).toThrow( @@ -324,8 +417,11 @@ describe('config', () => { decompress_response: false, compress_request: false, }, - username: 'default', - password: '', + auth: { + username: 'default', + password: '', + type: 'Credentials', + }, database: 'default', clickhouse_settings: {}, log_writer: jasmine.any(LogWriter), @@ -345,8 +441,10 @@ describe('config', () => { request: true, response: false, }, - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + }, database: 'analytics', clickhouse_settings: { async_insert: 1, @@ -367,8 +465,11 @@ describe('config', () => { compress_request: true, decompress_response: false, }, - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + type: 'Credentials', + }, database: 'analytics', clickhouse_settings: { async_insert: 1, @@ -391,8 +492,10 @@ describe('config', () => { it('should leave the base config as-is when there is nothing from the URL', async () => { const base: BaseClickHouseClientConfigOptions = { url: 'http://localhost:8123', - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + }, } expect(mergeConfigs(base, {}, logger)).toEqual(base) }) @@ -400,17 +503,21 @@ describe('config', () => { it('should take URL values first, then base config for the rest', async () => { const base: BaseClickHouseClientConfigOptions = { url: 'https://my.host:8124', - username: 'bob', - password: 'secret', + auth: { + username: 'bob', + password: 'secret', + }, } const fromURL: BaseClickHouseClientConfigOptions = { - password: 'secret_from_url!', + auth: { password: 'secret_from_url!' }, } const res = mergeConfigs(base, fromURL, logger) expect(res).toEqual({ url: 'https://my.host:8124', - username: 'bob', - password: 'secret_from_url!', + auth: { + username: 'bob', + password: 'secret_from_url!', + }, }) }) @@ -419,14 +526,12 @@ describe('config', () => { url: 'https://my.host:8124', } const fromURL: BaseClickHouseClientConfigOptions = { - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, } const res = mergeConfigs(base, fromURL, logger) expect(res).toEqual({ url: 'https://my.host:8124', - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, }) }) @@ -434,14 +539,12 @@ describe('config', () => { it('should only take the URL values when there is nothing in the base config', async () => { const fromURL: BaseClickHouseClientConfigOptions = { url: 'https://my.host:8443', - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, } const res = mergeConfigs({}, fromURL, logger) expect(res).toEqual({ url: 'https://my.host:8443', - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, }) }) @@ -645,8 +748,7 @@ describe('config', () => { const res = loadConfigOptionsFromURL(url, null) expect(res[0].toString()).toEqual('https://my.host:8124/') // pathname will be attached later. expect(res[1]).toEqual({ - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, database: 'analytics', application: 'my_app', pathname: '/my_proxy', @@ -676,8 +778,7 @@ describe('config', () => { const res = loadConfigOptionsFromURL(url, null) expect(res[0].toString()).toEqual('http://localhost:8124/') expect(res[1]).toEqual({ - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, database: 'analytics', }) }) @@ -819,8 +920,7 @@ describe('config', () => { const res = loadConfigOptionsFromURL(url, handler) expect(res[0].toString()).toEqual('https://my.host:8124/') expect(res[1]).toEqual({ - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, database: 'analytics', application: 'my_app', session_id: 'sticky', @@ -870,8 +970,7 @@ describe('config', () => { const res = loadConfigOptionsFromURL(url, handler) expect(res[0].toString()).toEqual('https://my.host:8124/') expect(res[1]).toEqual({ - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret' }, database: 'analytics', application: 'my_app', session_id: 'sticky', diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts index 59f87957..1544138c 100644 --- a/packages/client-common/__tests__/utils/client.ts +++ b/packages/client-common/__tests__/utils/client.ts @@ -56,7 +56,7 @@ export function createTestClient( if (isCloudTestEnv()) { const cloudConfig: BaseClickHouseClientConfigOptions = { url: `https://${getFromEnv(EnvKeys.host)}:8443`, - password: getFromEnv(EnvKeys.password), + auth: { password: getFromEnv(EnvKeys.password) }, database: databaseName, request_timeout: 60_000, ...logging, diff --git a/packages/client-common/__tests__/utils/env.ts b/packages/client-common/__tests__/utils/env.ts index c1d7955c..bc9b9518 100644 --- a/packages/client-common/__tests__/utils/env.ts +++ b/packages/client-common/__tests__/utils/env.ts @@ -2,6 +2,8 @@ export const EnvKeys = { host: 'CLICKHOUSE_CLOUD_HOST', username: 'CLICKHOUSE_CLOUD_USERNAME', password: 'CLICKHOUSE_CLOUD_PASSWORD', + jwt_access_token: 'CLICKHOUSE_CLOUD_JWT_ACCESS_TOKEN', + jwt_secret: 'CLICKHOUSE_CLOUD_JWT_SECRET', } export function getFromEnv(key: string): string { diff --git a/packages/client-common/src/clickhouse_types.ts b/packages/client-common/src/clickhouse_types.ts index 846888b9..2cb502c2 100644 --- a/packages/client-common/src/clickhouse_types.ts +++ b/packages/client-common/src/clickhouse_types.ts @@ -46,6 +46,26 @@ export interface ProgressRow { progress: ClickHouseSummary } +export type InsertValues = + | ReadonlyArray + | Stream + | InputJSON + | InputJSONObjectEachRow + +export type NonEmptyArray = [T, ...T[]] + +export interface ClickHouseCredentialsAuth { + username?: string + password?: string +} + +/** Supported in ClickHouse Cloud only */ +export interface ClickHouseJWTAuth { + access_token: string +} + +export type ClickHouseAuth = ClickHouseCredentialsAuth | ClickHouseJWTAuth + /** Type guard to use with JSONEachRowWithProgress, checking if the emitted row is a progress row. * @see https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */ export function isProgressRow(row: unknown): row is ProgressRow { diff --git a/packages/client-common/src/client.ts b/packages/client-common/src/client.ts index 627efa2a..7b0c9642 100644 --- a/packages/client-common/src/client.ts +++ b/packages/client-common/src/client.ts @@ -11,7 +11,7 @@ import type { WithResponseHeaders, } from '@clickhouse/client-common' import { type DataFormat, DefaultLogger } from '@clickhouse/client-common' -import type { InputJSON, InputJSONObjectEachRow } from './clickhouse_types' +import type { InsertValues, NonEmptyArray } from './clickhouse_types' import type { ImplementationDetails, ValuesEncoder } from './config' import { getConnectionParams, prepareConfigWithURL } from './config' import type { ConnPingResult } from './connection' @@ -35,13 +35,14 @@ export interface BaseQueryParams { * If it is not set, {@link BaseClickHouseClientConfigOptions.role} will be used. * @default undefined (no override) */ role?: string | Array - /** When defined, overrides the credentials from the {@link BaseClickHouseClientConfigOptions.username} - * and {@link BaseClickHouseClientConfigOptions.password} settings for this particular request. + /** When defined, overrides {@link BaseClickHouseClientConfigOptions.auth} for this particular request. * @default undefined (no override) */ - auth?: { - username: string - password: string - } + auth?: + | { + username: string + password: string + } + | { access_token: string } } export interface QueryParams extends BaseQueryParams { @@ -112,14 +113,6 @@ export type InsertResult = { export type ExecResult = ConnExecResult export type PingResult = ConnPingResult -export type InsertValues = - | ReadonlyArray - | Stream - | InputJSON - | InputJSONObjectEachRow - -type NonEmptyArray = [T, ...T[]] - /** {@link except} field contains a non-empty list of columns to exclude when generating `(* EXCEPT (...))` clause */ export interface InsertColumnsExcept { except: NonEmptyArray diff --git a/packages/client-common/src/config.ts b/packages/client-common/src/config.ts index b2bd013a..d4b6c242 100644 --- a/packages/client-common/src/config.ts +++ b/packages/client-common/src/config.ts @@ -1,5 +1,9 @@ -import type { ResponseHeaders } from './clickhouse_types' -import type { InsertValues } from './client' +import type { + ClickHouseAuth, + ClickHouseCredentialsAuth, + InsertValues, + ResponseHeaders, +} from './clickhouse_types' import type { Connection, ConnectionParams } from './connection' import type { DataFormat } from './data_formatter' import type { Logger } from './logger' @@ -39,12 +43,18 @@ export interface BaseClickHouseClientConfigOptions { * @default false */ request?: boolean } - /** The name of the user on whose behalf requests are made. + /** @deprecated Use {@link auth} object instead. + * The name of the user on whose behalf requests are made. * @default default */ username?: string - /** The user password. + /** @deprecated Use {@link auth} object instead. + * The user password. * @default empty string */ password?: string + /** Username + password or a JWT token to authenticate with ClickHouse. + * JWT token authentication is supported in ClickHouse Cloud only. + * @default username: `default`, password: empty string */ + auth?: ClickHouseAuth /** The name of the application using the JS client. * @default empty string */ application?: string @@ -178,6 +188,19 @@ export function prepareConfigWithURL( baseConfig.http_headers = baseConfig.additional_headers delete baseConfig.additional_headers } + if (baseConfig.username !== undefined || baseConfig.password !== undefined) { + logger.warn({ + module: 'Config', + message: + '"username" and "password" are deprecated. Use "auth" object instead.', + }) + baseConfig.auth = { + username: baseConfig.username, + password: baseConfig.password, + } + delete baseConfig.username + delete baseConfig.password + } let configURL if (baseConfig.host !== undefined) { logger.warn({ @@ -205,7 +228,26 @@ export function getConnectionParams( config: BaseClickHouseClientConfigOptionsWithURL, logger: Logger, ): ConnectionParams { + let auth: ConnectionParams['auth'] + if (config.auth !== undefined) { + if ('access_token' in config.auth) { + auth = { access_token: config.auth.access_token, type: 'JWT' } + } else { + auth = { + username: config.auth.username ?? 'default', + password: config.auth.password ?? '', + type: 'Credentials', + } + } + } else { + auth = { + username: 'default', + password: '', + type: 'Credentials', + } + } return { + auth, url: config.url, application_id: config.application, request_timeout: config.request_timeout ?? 30_000, @@ -214,8 +256,6 @@ export function getConnectionParams( decompress_response: config.compression?.response ?? false, compress_request: config.compression?.request ?? false, }, - username: config.username ?? 'default', - password: config.password ?? '', database: config.database ?? 'default', log_writer: new LogWriter(logger, 'Connection', config.log?.level), keep_alive: { enabled: config.keep_alive?.enabled ?? true }, @@ -300,12 +340,16 @@ export function loadConfigOptionsFromURL( handleExtraURLParams: HandleImplSpecificURLParams | null, ): [URL, BaseClickHouseClientConfigOptions] { let config: BaseClickHouseClientConfigOptions = {} + const auth: ClickHouseCredentialsAuth = {} if (url.username.trim() !== '') { - config.username = url.username + auth.username = url.username } // no trim for password if (url.password !== '') { - config.password = url.password + auth.password = url.password + } + if (Object.keys(auth).length > 0) { + config.auth = auth } if (url.pathname.trim().length > 1) { config.database = url.pathname.slice(1) @@ -397,6 +441,9 @@ export function loadConfigOptionsFromURL( } config.keep_alive.enabled = booleanConfigURLValue({ key, value }) break + case 'access_token': + config.auth = { access_token: value } + break default: paramWasProcessed = false unknownParams.add(key) diff --git a/packages/client-common/src/connection.ts b/packages/client-common/src/connection.ts index 19090d96..537183f5 100644 --- a/packages/client-common/src/connection.ts +++ b/packages/client-common/src/connection.ts @@ -5,19 +5,22 @@ import type { import type { LogWriter } from './logger' import type { ClickHouseSettings } from './settings' +export type ConnectionAuth = + | { username: string; password: string; type: 'Credentials' } + | { access_token: string; type: 'JWT' } + export interface ConnectionParams { url: URL request_timeout: number max_open_connections: number compression: CompressionSettings - username: string - password: string database: string clickhouse_settings: ClickHouseSettings log_writer: LogWriter keep_alive: { enabled: boolean } application_id?: string http_headers?: Record + auth: ConnectionAuth } export interface CompressionSettings { @@ -32,7 +35,7 @@ export interface ConnBaseQueryParams { abort_signal?: AbortSignal session_id?: string query_id?: string - auth?: { username: string; password: string } + auth?: { username: string; password: string } | { access_token: string } role?: string | Array } diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index 78527567..c44ef8f0 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -5,7 +5,6 @@ export { type QueryResult, type ExecParams, type InsertParams, - type InsertValues, ClickHouseClient, type CommandParams, type CommandResult, @@ -54,6 +53,10 @@ export type { WithClickHouseSummary, WithResponseHeaders, ProgressRow, + InsertValues, + ClickHouseAuth, + ClickHouseJWTAuth, + ClickHouseCredentialsAuth, } from './clickhouse_types' export { isProgressRow } from './clickhouse_types' export { @@ -105,6 +108,8 @@ export { transformUrl, withCompressionHeaders, withHttpSettings, + isCredentialsAuth, + isJWTAuth, } from './utils' export { LogWriter, DefaultLogger, type LogWriterParams } from './logger' export { parseError } from './error' diff --git a/packages/client-common/src/utils/connection.ts b/packages/client-common/src/utils/connection.ts index 62e65eac..81cc6a94 100644 --- a/packages/client-common/src/utils/connection.ts +++ b/packages/client-common/src/utils/connection.ts @@ -36,3 +36,18 @@ export function withHttpSettings( export function isSuccessfulResponse(statusCode?: number): boolean { return Boolean(statusCode && 200 <= statusCode && statusCode < 300) } + +export function isJWTAuth(auth: unknown): auth is { access_token: string } { + return auth !== null && typeof auth === 'object' && 'access_token' in auth +} + +export function isCredentialsAuth( + auth: unknown, +): auth is { username: string; password: string } { + return ( + auth !== null && + typeof auth === 'object' && + 'username' in auth && + 'password' in auth + ) +} diff --git a/packages/client-node/__tests__/integration/node_jwt_auth.test.ts b/packages/client-node/__tests__/integration/node_jwt_auth.test.ts new file mode 100644 index 00000000..e0bd6efb --- /dev/null +++ b/packages/client-node/__tests__/integration/node_jwt_auth.test.ts @@ -0,0 +1,50 @@ +import type { ClickHouseClient } from '@clickhouse/client-common' +import { createTestClient, TestEnv, whenOnEnv } from '@test/utils' +import { EnvKeys, getFromEnv } from '@test/utils/env' +import { makeJWT } from '../utils/jwt' + +whenOnEnv(TestEnv.CloudSMT).describe('[Node.js] JWT auth', () => { + let jwtClient: ClickHouseClient + let url: string + let jwt: string + + beforeAll(() => { + url = `https://${getFromEnv(EnvKeys.host)}:8443` + jwt = makeJWT() + }) + afterEach(async () => { + await jwtClient.close() + }) + + it('should work with client configuration', async () => { + jwtClient = createTestClient({ + url, + auth: { + access_token: jwt, + }, + }) + const rs = await jwtClient.query({ + query: 'SELECT 42 AS result', + format: 'JSONEachRow', + }) + expect(await rs.json()).toEqual([{ result: 42 }]) + }) + + it('should override the client instance auth', async () => { + jwtClient = createTestClient({ + url, + auth: { + username: 'gibberish', + password: 'gibberish', + }, + }) + const rs = await jwtClient.query({ + query: 'SELECT 42 AS result', + format: 'JSONEachRow', + auth: { + access_token: jwt, + }, + }) + expect(await rs.json()).toEqual([{ result: 42 }]) + }) +}) diff --git a/packages/client-node/__tests__/unit/node_client.test.ts b/packages/client-node/__tests__/unit/node_client.test.ts index 70aaedc5..fd0523ed 100644 --- a/packages/client-node/__tests__/unit/node_client.test.ts +++ b/packages/client-node/__tests__/unit/node_client.test.ts @@ -37,8 +37,7 @@ describe('[Node.js] createClient', () => { compress_request: false, decompress_response: false, }, - username: 'bob', - password: 'secret', + auth: { username: 'bob', password: 'secret', type: 'Credentials' }, database: 'analytics', clickhouse_settings: {}, log_writer: new LogWriter(new DefaultLogger(), 'Connection'), diff --git a/packages/client-node/__tests__/unit/node_config.test.ts b/packages/client-node/__tests__/unit/node_config.test.ts index d81a43af..0eced39e 100644 --- a/packages/client-node/__tests__/unit/node_config.test.ts +++ b/packages/client-node/__tests__/unit/node_config.test.ts @@ -71,8 +71,11 @@ describe('[Node.js] Config implementation details', () => { compress_request: true, decompress_response: true, }, - username: 'alice', - password: 'qwerty', + auth: { + username: 'alice', + password: 'qwerty', + type: 'Credentials', + }, database: 'default', clickhouse_settings: {}, log_writer: new LogWriter(new TestLogger(), 'MakeConnectionTest'), diff --git a/packages/client-node/__tests__/unit/node_create_connection.test.ts b/packages/client-node/__tests__/unit/node_create_connection.test.ts index f5cec168..80585a9f 100644 --- a/packages/client-node/__tests__/unit/node_create_connection.test.ts +++ b/packages/client-node/__tests__/unit/node_create_connection.test.ts @@ -16,24 +16,32 @@ describe('[Node.js] createConnection', () => { } const tlsParams: NodeConnectionParams['tls'] = undefined + const defaultConnectionParams = { + url: new URL('http://localhost'), + auth: { + username: 'default', + password: 'password', + type: 'Credentials', + }, + } as ConnectionParams + it('should create an instance of HTTP adapter', async () => { + const adapter = createConnection({ + connection_params: defaultConnectionParams, + tls: tlsParams, + keep_alive: keepAliveParams, + http_agent: undefined, + set_basic_auth_header: true, + }) expect(adapter).toBeInstanceOf(NodeHttpConnection) }) - const adapter = createConnection({ - connection_params: { - url: new URL('http://localhost'), - } as ConnectionParams, - tls: tlsParams, - keep_alive: keepAliveParams, - http_agent: undefined, - set_basic_auth_header: true, - }) it('should create an instance of HTTPS adapter', async () => { const adapter = createConnection({ connection_params: { + ...defaultConnectionParams, url: new URL('https://localhost'), - } as ConnectionParams, + }, tls: tlsParams, keep_alive: keepAliveParams, http_agent: undefined, @@ -46,8 +54,9 @@ describe('[Node.js] createConnection', () => { expect(() => createConnection({ connection_params: { + ...defaultConnectionParams, url: new URL('tcp://localhost'), - } as ConnectionParams, + }, tls: tlsParams, keep_alive: keepAliveParams, http_agent: undefined, @@ -59,9 +68,7 @@ describe('[Node.js] createConnection', () => { describe('Custom HTTP agent', () => { it('should create an instance with a custom HTTP agent', async () => { const adapter = createConnection({ - connection_params: { - url: new URL('https://localhost'), - } as ConnectionParams, + connection_params: defaultConnectionParams, tls: tlsParams, keep_alive: keepAliveParams, http_agent: new http.Agent({ @@ -75,9 +82,7 @@ describe('[Node.js] createConnection', () => { it('should create an instance with a custom HTTPS agent', async () => { const adapter = createConnection({ - connection_params: { - url: new URL('https://localhost'), - } as ConnectionParams, + connection_params: defaultConnectionParams, tls: tlsParams, keep_alive: keepAliveParams, http_agent: new https.Agent({ diff --git a/packages/client-node/__tests__/utils/http_stubs.ts b/packages/client-node/__tests__/utils/http_stubs.ts index 4304dedd..fc5ef0a1 100644 --- a/packages/client-node/__tests__/utils/http_stubs.ts +++ b/packages/client-node/__tests__/utils/http_stubs.ts @@ -105,8 +105,7 @@ export function buildHttpConnection(config: Partial) { }, max_open_connections: 10, - username: 'default', - password: '', + auth: { username: 'default', password: '', type: 'Credentials' }, database: 'default', clickhouse_settings: {}, @@ -130,6 +129,11 @@ export class MyTestHttpConnection extends NodeBaseConnection { enabled: false, }, set_basic_auth_header: true, + auth: { + username: 'default', + password: '', + type: 'Credentials', + }, } as NodeConnectionParams, {} as Http.Agent, ) diff --git a/packages/client-node/__tests__/utils/jwt.ts b/packages/client-node/__tests__/utils/jwt.ts new file mode 100644 index 00000000..45d36c2c --- /dev/null +++ b/packages/client-node/__tests__/utils/jwt.ts @@ -0,0 +1,18 @@ +import jwt from 'jsonwebtoken' + +export function makeJWT(): string { + const secret = process.env['CLICKHOUSE_CLOUD_JWT_SECRET'] + if (secret === undefined) { + throw new Error( + 'Environment variable CLICKHOUSE_CLOUD_JWT_SECRET is not set', + ) + } + const payload = { + iss: 'ClickHouse', + sub: 'CI_Test', + aud: '1f7f78b8-da67-480b-8913-726fdd31d2fc', + 'clickhouse:roles': ['default'], + 'clickhouse:grants': [], + } + return jwt.sign(payload, secret, { expiresIn: '15m', algorithm: 'RS256' }) +} diff --git a/packages/client-node/src/connection/node_base_connection.ts b/packages/client-node/src/connection/node_base_connection.ts index 77d45864..9bdc2db0 100644 --- a/packages/client-node/src/connection/node_base_connection.ts +++ b/packages/client-node/src/connection/node_base_connection.ts @@ -15,6 +15,7 @@ import type { LogWriter, ResponseHeaders, } from '@clickhouse/client-common' +import { isCredentialsAuth, isJWTAuth } from '@clickhouse/client-common' import { isSuccessfulResponse, parseError, @@ -84,9 +85,15 @@ export abstract class NodeBaseConnection protected readonly params: NodeConnectionParams, protected readonly agent: Http.Agent, ) { - this.defaultAuthHeader = `Basic ${Buffer.from( - `${params.username}:${params.password}`, - ).toString('base64')}` + if (params.auth.type === 'Credentials') { + this.defaultAuthHeader = `Basic ${Buffer.from( + `${params.auth.username}:${params.auth.password}`, + ).toString('base64')}` + } else if (params.auth.type === 'JWT') { + this.defaultAuthHeader = `Bearer ${params.auth.access_token}` + } else { + throw new Error(`Unknown auth type: ${(params.auth as any).type}`) + } this.defaultHeaders = { ...(params.http_headers ?? {}), // KeepAlive agent for some reason does not set this on its own @@ -258,19 +265,28 @@ export abstract class NodeBaseConnection protected buildRequestHeaders( params?: BaseQueryParams, ): Http.OutgoingHttpHeaders { - if (this.params.set_basic_auth_header) { + if (isJWTAuth(params?.auth)) { return { ...this.defaultHeaders, - Authorization: - params?.auth !== undefined - ? `Basic ${Buffer.from(`${params.auth.username}:${params.auth.password}`).toString('base64')}` - : this.defaultAuthHeader, + Authorization: `Bearer ${params.auth.access_token}`, } - } else { - return { - ...this.defaultHeaders, + } + if (this.params.set_basic_auth_header) { + if (isCredentialsAuth(params?.auth)) { + return { + ...this.defaultHeaders, + Authorization: `Basic ${Buffer.from(`${params.auth.username}:${params.auth.password}`).toString('base64')}`, + } + } else { + return { + ...this.defaultHeaders, + Authorization: this.defaultAuthHeader, + } } } + return { + ...this.defaultHeaders, + } } protected abstract createClientRequest( diff --git a/packages/client-node/src/connection/node_https_connection.ts b/packages/client-node/src/connection/node_https_connection.ts index 0c1b6ee7..a2befdab 100644 --- a/packages/client-node/src/connection/node_https_connection.ts +++ b/packages/client-node/src/connection/node_https_connection.ts @@ -1,4 +1,7 @@ -import type { BaseQueryParams } from '@clickhouse/client-common' +import { + type BaseQueryParams, + isCredentialsAuth, +} from '@clickhouse/client-common' import { withCompressionHeaders } from '@clickhouse/client-common' import type Http from 'http' import Https from 'https' @@ -24,10 +27,24 @@ export class NodeHttpsConnection extends NodeBaseConnection { params?: BaseQueryParams, ): Http.OutgoingHttpHeaders { if (this.params.tls !== undefined) { - const headers: Http.OutgoingHttpHeaders = { - ...this.defaultHeaders, - 'X-ClickHouse-User': params?.auth?.username ?? this.params.username, - 'X-ClickHouse-Key': params?.auth?.password ?? this.params.password, + if (this.params.auth.type === 'JWT') { + throw new Error( + 'JWT auth is not supported with HTTPS connection using custom certificates', + ) + } + let headers: Http.OutgoingHttpHeaders + if (isCredentialsAuth(params?.auth)) { + headers = { + ...this.defaultHeaders, + 'X-ClickHouse-User': params.auth.username, + 'X-ClickHouse-Key': params.auth.password, + } + } else { + headers = { + ...this.defaultHeaders, + 'X-ClickHouse-User': this.params.auth.username, + 'X-ClickHouse-Key': this.params.auth.password, + } } const tlsType = this.params.tls.type switch (tlsType) { diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index c3544aa9..6d854d74 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -63,5 +63,8 @@ export { type ProgressRow, isProgressRow, type RowOrProgress, + type ClickHouseAuth, + type ClickHouseJWTAuth, + type ClickHouseCredentialsAuth, TupleParam, } from '@clickhouse/client-common' diff --git a/packages/client-web/__tests__/jwt/web_jwt_auth.test.ts b/packages/client-web/__tests__/jwt/web_jwt_auth.test.ts new file mode 100644 index 00000000..d2fb40a8 --- /dev/null +++ b/packages/client-web/__tests__/jwt/web_jwt_auth.test.ts @@ -0,0 +1,52 @@ +import type { ClickHouseClient } from '@clickhouse/client-common' +import { createTestClient } from '@test/utils' +import { EnvKeys, getFromEnv } from '@test/utils/env' + +/** Cannot use the jsonwebtoken library to generate the token: it is Node.js only. + * The access token should be generated externally before running the test, + * and set as the CLICKHOUSE_JWT_ACCESS_TOKEN environment variable */ +describe('[Web] JWT auth', () => { + let client: ClickHouseClient + let url: string + let jwt: string + + beforeAll(() => { + url = `https://${getFromEnv(EnvKeys.host)}:8443` + jwt = getFromEnv(EnvKeys.jwt_access_token) + }) + afterEach(async () => { + await client.close() + }) + + it('should work with client configuration', async () => { + client = createTestClient({ + url, + auth: { + access_token: jwt, + }, + }) + const rs = await client.query({ + query: 'SELECT 42 AS result', + format: 'JSONEachRow', + }) + expect(await rs.json()).toEqual([{ result: 42 }]) + }) + + it('should override the client instance auth', async () => { + client = createTestClient({ + url, + auth: { + username: 'gibberish', + password: 'gibberish', + }, + }) + const rs = await client.query({ + query: 'SELECT 42 AS result', + format: 'JSONEachRow', + auth: { + access_token: jwt, + }, + }) + expect(await rs.json()).toEqual([{ result: 42 }]) + }) +}) diff --git a/packages/client-web/src/connection/web_connection.ts b/packages/client-web/src/connection/web_connection.ts index 493d9e19..39c22c9d 100644 --- a/packages/client-web/src/connection/web_connection.ts +++ b/packages/client-web/src/connection/web_connection.ts @@ -10,6 +10,8 @@ import type { ResponseHeaders, } from '@clickhouse/client-common' import { + isCredentialsAuth, + isJWTAuth, isSuccessfulResponse, parseError, toSearchParams, @@ -31,9 +33,18 @@ export type WebConnectionParams = ConnectionParams export class WebConnection implements Connection { private readonly defaultHeaders: Record constructor(private readonly params: WebConnectionParams) { - this.defaultHeaders = { - Authorization: `Basic ${btoa(`${params.username}:${params.password}`)}`, - ...params?.http_headers, + if (params.auth.type === 'JWT') { + this.defaultHeaders = { + Authorization: `Bearer ${params.auth.access_token}`, + ...params?.http_headers, + } + } else if (params.auth.type === 'Credentials') { + this.defaultHeaders = { + Authorization: `Basic ${btoa(`${params.auth.username}:${params.auth.password}`)}`, + ...params?.http_headers, + } + } else { + throw new Error(`Unknown auth type: ${(params.auth as any).type}`) } } @@ -173,12 +184,13 @@ export class WebConnection implements Connection { } try { + const authHeaderOverride = getAuthHeaderOverride(params?.auth) const headers = withCompressionHeaders({ headers: - params?.auth !== undefined + authHeaderOverride !== undefined ? { ...this.defaultHeaders, - Authorization: `Basic ${btoa(`${params.auth.username}:${params.auth.password}`)}`, + Authorization: authHeaderOverride, } : this.defaultHeaders, enable_request_compression: false, @@ -254,6 +266,15 @@ function getResponseHeaders(response: Response): ResponseHeaders { return headers } +function getAuthHeaderOverride(auth: ConnBaseQueryParams['auth']) { + if (isJWTAuth(auth)) { + return `Bearer ${auth.access_token}` + } + if (isCredentialsAuth(auth)) { + return `Basic ${btoa(`${auth.username}:${auth.password}`)}` + } +} + interface RunExecResult { stream: ReadableStream | null query_id: string diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index 021566ac..22b71b5c 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -62,5 +62,8 @@ export { type ProgressRow, isProgressRow, type RowOrProgress, + type ClickHouseAuth, + type ClickHouseJWTAuth, + type ClickHouseCredentialsAuth, TupleParam, } from '@clickhouse/client-common' diff --git a/webpack.dev.js b/webpack.dev.js index 866f3153..c704951e 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -16,6 +16,7 @@ module.exports = merge(common, { CLICKHOUSE_TEST_ENVIRONMENT: process.env.CLICKHOUSE_TEST_ENVIRONMENT, CLICKHOUSE_CLOUD_HOST: process.env.CLICKHOUSE_CLOUD_HOST, CLICKHOUSE_CLOUD_PASSWORD: process.env.CLICKHOUSE_CLOUD_PASSWORD, + CLICKHOUSE_CLOUD_JWT_ACCESS_TOKEN: process.env.CLICKHOUSE_CLOUD_JWT_ACCESS_TOKEN, }), }), ],