diff --git a/Cargo.lock b/Cargo.lock index c81d802..1c4b6e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" [[package]] name = "arrayref" @@ -321,7 +321,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.4.0", + "indexmap 2.5.0", "lexical-core", "num", "serde", @@ -450,18 +450,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -640,9 +640,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" [[package]] name = "byteorder" @@ -694,9 +694,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.14" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "jobserver", "libc", @@ -748,9 +748,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -758,9 +758,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -777,7 +777,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -866,9 +866,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" @@ -888,9 +888,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -975,9 +975,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1023,7 +1023,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "log", "num_cpus", @@ -1208,7 +1208,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "log", "paste", @@ -1237,7 +1237,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "hex", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "log", "paste", @@ -1295,7 +1295,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "log", "once_cell", @@ -1369,7 +1369,7 @@ dependencies = [ "delta_kernel_derive", "either", "fix-hidden-lifetime-bug", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.13.0", "lazy_static", "parquet", @@ -1394,7 +1394,7 @@ checksum = "6502fa0ba72fd1f782ccebba8f4c8b9a07c7591559e39d3d05b7ead94690a13f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1442,7 +1442,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.13.0", "lazy_static", "libc", @@ -1485,6 +1485,7 @@ dependencies = [ "datafusion-common", "deltalake", "directories", + "env_logger", "futures", "itertools 0.13.0", "lazy_static", @@ -1555,6 +1556,29 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1713,7 +1737,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1800,7 +1824,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.4.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1819,7 +1843,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1997,16 +2021,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", "hyper 1.4.1", "hyper-util", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls", @@ -2027,9 +2051,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -2096,9 +2120,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2111,7 +2135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2134,9 +2158,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "is_terminal_polyfill" @@ -2659,7 +2683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.4.0", + "indexmap 2.5.0", ] [[package]] @@ -2717,7 +2741,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2806,7 +2830,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2830,9 +2854,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", @@ -2848,9 +2872,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", @@ -2865,15 +2889,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2917,9 +2941,9 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba6a365afbe5615999275bea2446b970b10a41102500e27ce7678d50d978303" +checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" dependencies = [ "bitflags 2.6.0", "cassowary", @@ -3011,7 +3035,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "rustls-pemfile", "rustls-pki-types", "serde", @@ -3069,18 +3093,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags 2.6.0", "errno", @@ -3105,9 +3129,22 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -3134,9 +3171,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -3166,11 +3203,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3216,29 +3253,29 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -3406,7 +3443,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3440,7 +3477,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3462,9 +3499,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -3522,7 +3559,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3572,9 +3609,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -3605,7 +3642,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3621,9 +3658,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -3632,9 +3669,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -3670,7 +3707,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -3756,7 +3793,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3956,7 +3993,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4015,7 +4052,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -4049,7 +4086,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4343,7 +4380,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 392d79b..9c75f72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ datafusion = "41.0.0" datafusion-common = "41.0.0" deltalake = { version = "0.19.0", features = ["datafusion"], optional = true } directories = "5.0.1" +env_logger = "0.11.5" futures = "0.3.30" itertools = "0.13.0" lazy_static = "1.4.0" diff --git a/README.md b/README.md index 37c0ac2..1fddce2 100644 --- a/README.md +++ b/README.md @@ -68,14 +68,24 @@ Some of the current and planned features are: ## `dft` CLI -The `dft` CLI is a scriptable engine for executing queries from files. It is used in a similar manner to `datafusion-cli` but with the added benefit of being able to query from multiple data sources. +The `dft` CLI is a scriptable interface to the `tui` engine for executing +queries from files or the command line. The CLI is used in a similar manner to +`datafusion-cli` but with the added benefit of supporting multiple pre-integrated +data sources. -For example you can run the contents of `query.sql` with +### Example: Run the contents of `query.sql` ```shell $ dft -f query.sql ``` +### Example: Run a query from the command line + +```shell +$ dft -c "SELECT 1+2" +``` + + ## User Guide ### Installation diff --git a/src/app/execution.rs b/src/app/execution.rs index 38832d4..6ac009b 100644 --- a/src/app/execution.rs +++ b/src/app/execution.rs @@ -23,6 +23,7 @@ use datafusion::execution::session_state::SessionStateBuilder; use datafusion::execution::TaskContext; use datafusion::physical_plan::{execute_stream, visit_execution_plan, ExecutionPlanVisitor}; use datafusion::prelude::*; +use datafusion::sql::parser::Statement; use datafusion::{arrow::util::pretty::pretty_format_batches, physical_plan::ExecutionPlan}; #[cfg(feature = "deltalake")] use deltalake::delta_datafusion::DeltaTableFactory; @@ -121,10 +122,25 @@ impl ExecutionContext { &self.session_ctx } + /// Executes the specified parsed DataFusion statement and prints the result to stdout + pub async fn execute_statement(&self, statement: Statement) -> Result<()> { + let plan = self + .session_ctx + .state() + .statement_to_plan(statement) + .await?; + let df = self.session_ctx.execute_logical_plan(plan).await?; + self.execute_stream_dataframe(df).await + } + /// Executes the specified query and prints the result to stdout pub async fn execute_stream_sql(&self, query: &str) -> Result<()> { - let df = self.session_ctx.sql(query).await.unwrap(); - let physical_plan = df.create_physical_plan().await.unwrap(); + let df = self.session_ctx.sql(query).await?; + self.execute_stream_dataframe(df).await + } + + pub async fn execute_stream_dataframe(&self, df: DataFrame) -> Result<()> { + let physical_plan = df.create_physical_plan().await?; // We use small batch size because web socket stream comes in small increments (each // message usually only has at most a few records). let stream_cfg = SessionConfig::default(); diff --git a/src/app/mod.rs b/src/app/mod.rs index d8ff8a6..3d0d844 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -25,6 +25,8 @@ use crate::{cli, ui}; use color_eyre::eyre::eyre; use color_eyre::Result; use crossterm::event as ct; +use datafusion::sql::parser::DFParser; +use datafusion::sql::sqlparser::dialect::GenericDialect; use futures::FutureExt; use log::{debug, error, info, trace}; use ratatui::backend::CrosstermBackend; @@ -325,7 +327,21 @@ pub async fn run_app(cli: cli::DftCli, state: state::AppState<'_>) -> Result<()> app.exit() } -pub async fn execute_files(files: Vec, state: &state::AppState<'_>) -> Result<()> { +pub async fn execute_files_or_commands( + files: Vec, + commands: Vec, + state: &state::AppState<'_>, +) -> Result<()> { + match (files.is_empty(), commands.is_empty()) { + (true, true) => Err(eyre!("No files or commands provided to execute")), + (false, true) => execute_files(files, state).await, + (true, false) => execute_commands(commands, state).await, + (false, false) => Err(eyre!( + "Cannot execute both files and commands at the same time" + )), + } +} +async fn execute_files(files: Vec, state: &state::AppState<'_>) -> Result<()> { info!("Executing files: {:?}", files); let execution = ExecutionContext::new(state.config.execution.clone()); @@ -335,6 +351,24 @@ pub async fn execute_files(files: Vec, state: &state::AppState<'_>) -> Ok(()) } +async fn execute_commands(commands: Vec, state: &state::AppState<'_>) -> Result<()> { + info!("Executing commands: {:?}", commands); + for command in commands { + exec_from_string(&command, state).await? + } + + Ok(()) +} + +async fn exec_from_string(sql: &str, state: &state::AppState<'_>) -> Result<()> { + let dialect = GenericDialect {}; + let execution = ExecutionContext::new(state.config.execution.clone()); + let statements = DFParser::parse_sql_with_dialect(sql, &dialect)?; + for statement in statements { + execution.execute_statement(statement).await?; + } + Ok(()) +} /// run and execute SQL statements and commands from a file, against a context /// with the given print options diff --git a/src/cli/mod.rs b/src/cli/mod.rs index ec4a532..e539196 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -34,7 +34,7 @@ Environment Variables RUST_LOG { trace | debug | info | error }: Standard rust logging level. Default is info. "; -#[derive(Clone, Debug, Parser)] +#[derive(Clone, Debug, Parser, Default)] #[command(author, version, about, long_about = LONG_ABOUT)] pub struct DftCli { #[clap( @@ -46,7 +46,16 @@ pub struct DftCli { )] pub files: Vec, - #[clap(short, long, help = "Path to the configuration file")] + #[clap( + short = 'c', + long, + num_args = 0.., + help = "Execute the given SQL string(s), then exit.", + value_parser(parse_command) + )] + pub commands: Vec, + + #[clap(long, help = "Path to the configuration file")] pub config: Option, } @@ -76,3 +85,11 @@ fn parse_valid_file(file: &str) -> Result { Ok(path) } } + +fn parse_command(command: &str) -> Result { + if !command.is_empty() { + Ok(command.to_string()) + } else { + Err("-c flag expects only non empty commands".to_string()) + } +} diff --git a/src/main.rs b/src/main.rs index 928ab21..d3813b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,21 +17,24 @@ use clap::Parser; use color_eyre::Result; -use dft::app::state; -use dft::app::{execute_files, run_app}; +use dft::app::{execute_files_or_commands, run_app, state}; use dft::cli; use dft::telemetry; #[tokio::main] async fn main() -> Result<()> { - telemetry::initialize_logs()?; let cli = cli::DftCli::parse(); - let state = state::initialize(cli.clone()); // If executing commands from files, do so and then exit - if !cli.files.is_empty() { - execute_files(cli.files.clone(), &state).await?; + if !cli.files.is_empty() || !cli.commands.is_empty() { + // use env_logger to setup logging for CLI + env_logger::init(); + let state = state::initialize(cli.clone()); + execute_files_or_commands(cli.files.clone(), cli.commands.clone(), &state).await?; } else { + // use alternate logging for TUI + telemetry::initialize_logs()?; + let state = state::initialize(cli.clone()); run_app(cli.clone(), state).await?; } diff --git a/tests/cli.rs b/tests/cli.rs index fdc8147..3d2c8e6 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -18,9 +18,9 @@ //! Tests for the CLI (e.g. run from files) use assert_cmd::Command; +use predicates::str::ContainsPredicate; use std::path::PathBuf; - -mod util; +use tempfile::NamedTempFile; #[test] fn test_help() { @@ -30,7 +30,7 @@ fn test_help() { .assert() .success(); - assert.stdout(util::contains_str("dft")); + assert.stdout(contains_str("dft")); } #[test] @@ -44,7 +44,7 @@ fn test_logging() { .assert() .success(); - assert.stdout(util::contains_str("INFO")); + assert.stdout(contains_str("INFO")); } #[test] @@ -57,12 +57,12 @@ fn test_command_in_file() { +---------------------+ "##; - let file = util::sql_in_file("SELECT 1 + 1"); - util::assert_output_contains(vec![file], expected); + let file = sql_in_file("SELECT 1 + 1"); + assert_output_contains(vec![file], expected); // same test but with a semicolon at the end - let file = util::sql_in_file("SELECT 1 + 1;"); - util::assert_output_contains(vec![file], expected); + let file = sql_in_file("SELECT 1 + 1;"); + assert_output_contains(vec![file], expected); } #[test] @@ -91,12 +91,12 @@ CREATE TABLE foo as values (42); SELECT column1 + 2 FROM foo "#; - let file = util::sql_in_file(sql); - util::assert_output_contains(vec![file], expected); + let file = sql_in_file(sql); + assert_output_contains(vec![file], expected); // same test but with a semicolon at the end of second command - let file = util::sql_in_file(format!("{sql};")); - util::assert_output_contains(vec![file], expected); + let file = sql_in_file(format!("{sql};")); + assert_output_contains(vec![file], expected); } #[test] @@ -119,14 +119,14 @@ fn test_multiple_commands_in_multiple_files() { +----------+ "##; - let file1 = util::sql_in_file("SELECT 1 + 2"); - let file2 = util::sql_in_file("SELECT 1;\nselect 2;"); - util::assert_output_contains(vec![file1, file2], expected); + let file1 = sql_in_file("SELECT 1 + 2"); + let file2 = sql_in_file("SELECT 1;\nselect 2;"); + assert_output_contains(vec![file1, file2], expected); } #[test] fn test_non_existent_file() { - let file = util::sql_in_file("SELECT 1 + 1"); + let file = sql_in_file("SELECT 1 + 1"); let p = PathBuf::from(file.path()); // dropping the file makes it non existent drop(file); @@ -139,13 +139,13 @@ fn test_non_existent_file() { .failure(); let expected = format!("File does not exist: '{}'", p.to_string_lossy()); - assert.code(2).stderr(util::contains_str(&expected)); + assert.code(2).stderr(contains_str(&expected)); } #[test] fn test_one_existent_and_one_non_existent_file() { - let file1 = util::sql_in_file("SELECT 1 + 1"); - let file2 = util::sql_in_file("SELECT 3 + 4"); + let file1 = sql_in_file("SELECT 1 + 1"); + let file2 = sql_in_file("SELECT 3 + 4"); let p1 = PathBuf::from(file1.path()); let p2 = PathBuf::from(file2.path()); // dropping the file makes it non existent @@ -161,12 +161,12 @@ fn test_one_existent_and_one_non_existent_file() { .failure(); let expected_err = format!("File does not exist: '{}'", p2.to_string_lossy()); - assert.code(2).stderr(util::contains_str(&expected_err)); + assert.code(2).stderr(contains_str(&expected_err)); } #[test] fn test_sql_err_in_file() { - let file = util::sql_in_file("SELECT this is not valid SQL"); + let file = sql_in_file("SELECT this is not valid SQL"); let assert = Command::cargo_bin("dft") .unwrap() @@ -177,12 +177,12 @@ fn test_sql_err_in_file() { let expected_err = "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; - assert.code(101).stderr(util::contains_str(expected_err)); + assert.code(1).stderr(contains_str(expected_err)); } #[test] fn test_sql_err_in_file_after_first() { - let file = util::sql_in_file( + let file = sql_in_file( r#" -- First line is valid SQL SELECT 1 + 1; @@ -200,5 +200,146 @@ SELECT this is not valid SQL let expected_err = "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; - assert.code(101).stderr(util::contains_str(expected_err)); + assert.code(1).stderr(contains_str(expected_err)); +} + +#[test] +fn test_sql_in_file_and_arg() { + let file = sql_in_file("SELECT 1 + 1"); + + let assert = Command::cargo_bin("dft") + .unwrap() + .arg("-f") + .arg(file.path()) + // also specify a query on the command line + .arg("-c") + .arg("SELECT 3 + 4") + .assert() + .failure(); + + assert.code(1).stderr(contains_str( + "Error: Cannot execute both files and commands at the same time", + )); +} + +#[test] +fn test_sql_in_arg() { + let expected = r##" ++---------------------+ +| Int64(1) + Int64(2) | ++---------------------+ +| 3 | ++---------------------+ + "##; + let assert = Command::cargo_bin("dft") + .unwrap() + .arg("-c") + .arg("SELECT 1 + 2;") + .assert() + .success(); + + assert.stdout(contains_str(expected)); +} + +#[test] +fn test_multiple_sql_in_arg() { + let expected = r##" ++------------+ +| sum(foo.x) | ++------------+ +| 3 | ++------------+ + "##; + let assert = Command::cargo_bin("dft") + .unwrap() + .arg("-c") + // use multiple SQL statements in one argument that need to run in the same + // context + .arg("CREATE TABLE foo(x int) as values (1), (2); SELECT sum(x) FROM foo") + .assert() + .success(); + + assert.stdout(contains_str(expected)); +} +#[test] +fn test_multiple_sql_in_multiple_args() { + let expected = r##" ++---------------------+ +| Int64(1) + Int64(2) | ++---------------------+ +| 3 | ++---------------------+ ++---------------------+ +| Int64(3) + Int64(5) | ++---------------------+ +| 8 | ++---------------------+ + "##; + let assert = Command::cargo_bin("dft") + .unwrap() + .arg("-c") + .arg("SELECT 1 + 2") + .arg("SELECT 3 + 5") + .assert() + .success(); + + assert.stdout(contains_str(expected)); +} + +#[test] +fn test_multiple_sql_in_multiple_args2() { + let expected = r##" ++---------------------+ +| Int64(1) + Int64(2) | ++---------------------+ +| 3 | ++---------------------+ ++---------------------+ +| Int64(3) + Int64(5) | ++---------------------+ +| 8 | ++---------------------+ + "##; + let assert = Command::cargo_bin("dft") + .unwrap() + .arg("-c") + .arg("SELECT 1 + 2") + .arg("-c") // add second -c + .arg("SELECT 3 + 5") + .assert() + .success(); + + assert.stdout(contains_str(expected)); +} + +/// Creates a temporary file with the given SQL content +pub fn sql_in_file(sql: impl AsRef) -> NamedTempFile { + let file = NamedTempFile::new().unwrap(); + std::fs::write(file.path(), sql.as_ref()).unwrap(); + file +} + +/// Returns a predicate that expects the given string to be contained in the +/// output +/// +/// Whitespace is trimmed from the start and end of the string +pub fn contains_str(s: &str) -> ContainsPredicate { + predicates::str::contains(s.trim()) +} + +/// Invokes `dft -f` with the given files and asserts that it exited +/// successfully and the output contains the given string +pub fn assert_output_contains(files: Vec, expected_output: &str) { + let mut cmd = Command::cargo_bin("dft").unwrap(); + for file in &files { + cmd.arg("-f").arg(file.path()); + } + + let assert = cmd.assert().success(); + + // Since temp files are deleted when they go out of scope ensure they are + // dropped only after the command is run + drop(files); + + assert.stdout(contains_str(expected_output)); } diff --git a/tests/common.rs b/tests/common.rs deleted file mode 100644 index 1d25be6..0000000 --- a/tests/common.rs +++ /dev/null @@ -1,236 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Tests for the CLI (e.g. run from files) - -use assert_cmd::Command; -use predicates::str::ContainsPredicate; -use std::path::PathBuf; -use tempfile::NamedTempFile; - -#[test] -fn test_help() { - let assert = Command::cargo_bin("dft") - .unwrap() - .arg("--help") - .assert() - .success(); - - assert.stdout(contains_str("dft")); -} - -#[test] -#[ignore] -fn test_logging() { - // currently fails with - // Error: Device not configured (os error 6) - let assert = Command::cargo_bin("dft") - .unwrap() - .env("RUST_LOG", "info") - .assert() - .success(); - - assert.stdout(contains_str("INFO")); -} - -#[test] -fn test_command_in_file() { - let expected = r##" -+---------------------+ -| Int64(1) + Int64(1) | -+---------------------+ -| 2 | -+---------------------+ - "##; - - let file = sql_in_file("SELECT 1 + 1"); - assert_output_contains(vec![file], expected); - - // same test but with a semicolon at the end - let file = sql_in_file("SELECT 1 + 1;"); - assert_output_contains(vec![file], expected); -} - -#[test] -fn test_multiple_commands_in_file() { - let expected = r##" -+---------+ -| column1 | -+---------+ -| 42 | -+---------+ -+------------------------+ -| foo.column1 + Int64(2) | -+------------------------+ -| 44 | -+------------------------+ - "##; - - let sql = r#" --- The first line is a comment -CREATE TABLE foo as values (42); --- lets ignore some whitespace - - SELECT column1 FROM foo; - --- Another comment -SELECT column1 + 2 FROM foo - "#; - - let file = sql_in_file(sql); - assert_output_contains(vec![file], expected); - - // same test but with a semicolon at the end of second command - let file = sql_in_file(format!("{sql};")); - assert_output_contains(vec![file], expected); -} - -#[test] -fn test_multiple_commands_in_multiple_files() { - let expected = r##" -+---------------------+ -| Int64(1) + Int64(2) | -+---------------------+ -| 3 | -+---------------------+ -+----------+ -| Int64(1) | -+----------+ -| 1 | -+----------+ -+----------+ -| Int64(2) | -+----------+ -| 2 | -+----------+ - "##; - - let file1 = sql_in_file("SELECT 1 + 2"); - let file2 = sql_in_file("SELECT 1;\nselect 2;"); - assert_output_contains(vec![file1, file2], expected); -} - -#[test] -fn test_non_existent_file() { - let file = sql_in_file("SELECT 1 + 1"); - let p = PathBuf::from(file.path()); - // dropping the file makes it non existent - drop(file); - - let assert = Command::cargo_bin("dft") - .unwrap() - .arg("-f") - .arg(&p) - .assert() - .failure(); - - let expected = format!("File does not exist: '{}'", p.to_string_lossy()); - assert.code(2).stderr(contains_str(&expected)); -} - -#[test] -fn test_one_existent_and_one_non_existent_file() { - let file1 = sql_in_file("SELECT 1 + 1"); - let file2 = sql_in_file("SELECT 3 + 4"); - let p1 = PathBuf::from(file1.path()); - let p2 = PathBuf::from(file2.path()); - // dropping the file makes it non existent - drop(file2); - - let assert = Command::cargo_bin("dft") - .unwrap() - .arg("-f") - .arg(p1) - .arg("-f") - .arg(&p2) - .assert() - .failure(); - - let expected_err = format!("File does not exist: '{}'", p2.to_string_lossy()); - assert.code(2).stderr(contains_str(&expected_err)); -} - -#[test] -fn test_sql_err_in_file() { - let file = sql_in_file("SELECT this is not valid SQL"); - - let assert = Command::cargo_bin("dft") - .unwrap() - .arg("-f") - .arg(file.path()) - .assert() - .failure(); - - let expected_err = - "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; - assert.code(101).stderr(contains_str(expected_err)); -} - -#[test] -fn test_sql_err_in_file_after_first() { - let file = sql_in_file( - r#" --- First line is valid SQL -SELECT 1 + 1; --- Second line is not -SELECT this is not valid SQL - "#, - ); - - let assert = Command::cargo_bin("dft") - .unwrap() - .arg("-f") - .arg(file.path()) - .assert() - .failure(); - - let expected_err = - "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; - assert.code(101).stderr(contains_str(expected_err)); -} - -/// Creates a temporary file with the given SQL content -fn sql_in_file(sql: impl AsRef) -> NamedTempFile { - let file = NamedTempFile::new().unwrap(); - std::fs::write(file.path(), sql.as_ref()).unwrap(); - file -} - -/// Returns a predicate that expects the given string to be contained in the -/// output -/// -/// Whitespace is trimmed from the start and end of the string -fn contains_str(s: &str) -> ContainsPredicate { - predicates::str::contains(s.trim()) -} - -/// Invokes `dft -f` with the given files and asserts that it exited -/// successfully and the output contains the given string -fn assert_output_contains(files: Vec, expected_output: &str) { - let mut cmd = Command::cargo_bin("dft").unwrap(); - for file in &files { - cmd.arg("-f").arg(file.path()); - } - - let assert = cmd.assert().success(); - - // Since temp files are deleted when they go out of scope ensure they are - // dropped only after the command is run - drop(files); - - assert.stdout(contains_str(expected_output)); -} diff --git a/tests/tui.rs b/tests/tui.rs index 8863d34..86a1570 100644 --- a/tests/tui.rs +++ b/tests/tui.rs @@ -22,10 +22,7 @@ use dft::app::App; use dft::cli::DftCli; fn setup_app() -> App<'static> { - let args = DftCli { - files: Vec::new(), - config: None, - }; + let args = DftCli::default(); let state = initialize(args.clone()); let app = App::new(state, args); app diff --git a/tests/util.rs b/tests/util.rs deleted file mode 100644 index 5c0eebd..0000000 --- a/tests/util.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Tests for the CLI (e.g. run from files) - -use assert_cmd::Command; -use predicates::str::ContainsPredicate; -use tempfile::NamedTempFile; - -/// Creates a temporary file with the given SQL content -pub fn sql_in_file(sql: impl AsRef) -> NamedTempFile { - let file = NamedTempFile::new().unwrap(); - std::fs::write(file.path(), sql.as_ref()).unwrap(); - file -} - -/// Returns a predicate that expects the given string to be contained in the -/// output -/// -/// Whitespace is trimmed from the start and end of the string -pub fn contains_str(s: &str) -> ContainsPredicate { - predicates::str::contains(s.trim()) -} - -/// Invokes `dft -f` with the given files and asserts that it exited -/// successfully and the output contains the given string -pub fn assert_output_contains(files: Vec, expected_output: &str) { - let mut cmd = Command::cargo_bin("dft").unwrap(); - for file in &files { - cmd.arg("-f").arg(file.path()); - } - - let assert = cmd.assert().success(); - - // Since temp files are deleted when they go out of scope ensure they are - // dropped only after the command is run - drop(files); - - assert.stdout(contains_str(expected_output)); -}