diff --git a/Cargo.lock b/Cargo.lock index 24eb3aaf..7d25d8fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -687,6 +687,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -940,6 +950,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1090,6 +1109,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs-err" version = "2.11.0" @@ -1448,6 +1491,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1471,6 +1527,16 @@ dependencies = [ "cc", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1519,6 +1585,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is-terminal" version = "0.4.12" @@ -1790,6 +1862,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1902,6 +1992,50 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.49", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -2078,6 +2212,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "plotters" version = "0.3.5" @@ -2558,6 +2698,46 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -2633,6 +2813,7 @@ dependencies = [ "pyo3-build-config", "ref-cast", "regex", + "reqwest", "risinglight_proto", "rust_decimal", "rustyline", @@ -2769,6 +2950,15 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2822,6 +3012,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2844,6 +3043,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.21" @@ -2884,6 +3106,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.31" @@ -3212,6 +3446,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -3434,6 +3689,16 @@ dependencies = [ "syn 2.0.49", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3691,6 +3956,17 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -3712,6 +3988,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3777,6 +4059,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.91" @@ -4019,6 +4313,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 70345adb..4096b60c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ prost = "0.12" pyo3 = { version = "0.20", features = ["extension-module"], optional = true } ref-cast = "1.0" regex = "1" +reqwest = { version = "0.11", features = ["json"] } risinglight_proto = "0.2" rust_decimal = "1" rustyline = "13" diff --git a/src/executor/mod.rs b/src/executor/mod.rs index a83d1321..1d69133b 100644 --- a/src/executor/mod.rs +++ b/src/executor/mod.rs @@ -104,6 +104,12 @@ pub enum ExecutorError { #[source] csv::Error, ), + #[error("request error: {0}")] + Reqwest( + #[from] + #[source] + reqwest::Error, + ), #[error("value can not be null")] NotNullable, #[error("exceed char/varchar length limit: item length {length} > char/varchar width {width}")] diff --git a/src/executor/system_table_scan.rs b/src/executor/system_table_scan.rs index d4b3c6c2..b47c43b3 100644 --- a/src/executor/system_table_scan.rs +++ b/src/executor/system_table_scan.rs @@ -1,6 +1,8 @@ // Copyright 2024 RisingLight Project Authors. Licensed under Apache-2.0. +use reqwest::Client; use risinglight_proto::rowset::block_statistics::BlockStatisticsType; +use serde::Deserialize; use super::*; use crate::array::*; @@ -25,7 +27,7 @@ impl SystemTableScan { assert_eq!(self.columns.len(), table.all_columns().len()); yield match table.name() { - "contributors" => contributors(), + "contributors" => contributors().await?, "pg_tables" => pg_tables(self.catalog), "pg_attribute" => pg_attribute(self.catalog), "pg_stat" => pg_stat(self.catalog, &*self.storage).await?, @@ -34,74 +36,27 @@ impl SystemTableScan { } } -// TODO: find a better way to maintain the contributors list instead of hard-coding, and get total -// contributors when contributors is more than 100. (per_page max is 100) -// update this funciton with `curl https://api.github.com/repos/risinglightdb/risinglight/contributors?per_page=100 | jq ".[].login"` -fn contributors() -> DataChunk { - let contributors = vec![ - "skyzh", - "wangrunji0408", - "MingjiHan99", - "pleiadesian", - "TennyZhuang", - "xxchan", - "st1page", - "caicancai", - "Fedomn", - "arkbriar", - "likg227", - "lokax", - "zzl200012", - "unconsolable", - "BaymaxHWY", - "alissa-tung", - "ludics", - "Sunt-ing", - "yinfredyue", - "xiaoyong-z", - "Kikkon", - "D2Lark", - "xzhseh", - "ice1000", - "kwannoel", - "GoGim1", - "eliasyaoyc", - "wangqiim", - "silver-ymz", - "adlternative", - "yingjunwu", - "chaixuqing", - "WindowsXp-Beta", - "tabVersion", - "SkyFan2002", - "FANNG1", - "XieJiann", - "yuzi-neko", - "xinchengxx", - "sundy-li", - "nanderstabel", - "jetjinser", - "cadl", - "Gun9niR", - "zehaowei", - "rapiz1", - "LiuYuHui", - "Ted-Jiang", - "Y7n05h", - "RinChanNOWWW", - "noneback", - "chowc", - "xiaguan", - "JayiceZ", - "danipozo", - "PsiACE", - "yeya24", - ]; - [ArrayImpl::new_string(StringArray::from_iter( - contributors.iter().map(|s| Some(*s)).sorted(), +#[allow(clippy::doc_markdown)] +/// Get contributors from GitHub. +async fn contributors() -> Result { + #[derive(Deserialize)] + struct User { + login: String, + } + let client = Client::new(); + let response = client + .get("https://api.github.com/repos/risinglightdb/risinglight/contributors?per_page=100") + // Github API requires a User-Agent header + .header("User-Agent", "RisingLight") + .send() + .await?; + let contributors = response.json::>().await?; + + Ok([ArrayImpl::new_string(StringArray::from_iter( + contributors.iter().map(|s| Some(&s.login)), ))] .into_iter() - .collect() + .collect()) } /// Returns `pg_tables` table. diff --git a/tests/sql/internal.slt b/tests/sql/internal.slt index c2758954..62ec3393 100644 --- a/tests/sql/internal.slt +++ b/tests/sql/internal.slt @@ -1,63 +1,7 @@ query I -select github_id from pg_catalog.contributors; +select count(github_id) > 50 from pg_catalog.contributors; ---- -BaymaxHWY -D2Lark -FANNG1 -Fedomn -GoGim1 -Gun9niR -JayiceZ -Kikkon -LiuYuHui -MingjiHan99 -PsiACE -RinChanNOWWW -SkyFan2002 -Sunt-ing -Ted-Jiang -TennyZhuang -WindowsXp-Beta -XieJiann -Y7n05h -adlternative -alissa-tung -arkbriar -cadl -caicancai -chaixuqing -chowc -danipozo -eliasyaoyc -ice1000 -jetjinser -kwannoel -likg227 -lokax -ludics -nanderstabel -noneback -pleiadesian -rapiz1 -silver-ymz -skyzh -st1page -sundy-li -tabVersion -unconsolable -wangqiim -wangrunji0408 -xiaguan -xiaoyong-z -xinchengxx -xxchan -xzhseh -yeya24 -yinfredyue -yingjunwu -yuzi-neko -zehaowei -zzl200012 +true statement error -select github_i from pg_catalog.contributors; \ No newline at end of file +select github_i from pg_catalog.contributors;