refactor preprocessors. refactor unpack-mode. rename selectors.

.

README.md

@@ -40,21 +40,23 @@ respectively is used in the documentation. If the context makes it clear
 whether it is a reference to a JSON data structure or the corresponding
 internal Lisp data structure.
 
-## Clauses
+## Operators
 
-The following clauses have special behaviour. You can also write generic CL code,
-including the functions further down.
+The following operators have special behaviour. You can also write generic CL code,
+anywhere you can use an operator. Including the functions further down.
 
   - ` (|| ...)` pipe the results from the first clause into the second etc.
     returns the result of the last clause.
-  - `#{s1 ...}` iterate vector of `kvs` and select keys a new vector of `kvs` using `kv` selectors.
-  - `#[s1 ...]` iterate `vector` of `kvs` and select keys into new `vector` using `kv` selectors.
+  - `#{s1 ...}` select from vector of `kvs` into new vector of `kvs` using `kv` selectors.
+  - `#[s1 ...]` select from `vector` of `kvs` into new `vector` using `kv` selectors.
   - ` {s1 ...}` select from `kv` into new `kv` using `kv` selectors.
-  - ` [s1 ...]` iterate `vector` and filter into new `vector` using vector vector filters
+  - ` [s1 ...]` select from `vector` into new `vector` using vector `vector` selectors.
   - ` (*map fx)` map `#'fx` current `(dat)`.
   - ` (*map (fx _))` map `(fx _)` across current `(dat)`.
   - ` (*map k (fx ... k))` map `(fx ... k)` across current `(dat)`.
-  - ` (*fld init fx)` TODO
+  - ` (*fld init fx)` fold `fx` with init as the first argument.
+  - ` (*fld init nxt (fx acc ... nxt))` fold `(fx ...)` where the accumulated value is implicit.
+  - ` (*fld init acc nxt (fx ... acc ... nxt))` both the accumulated value and the next is explicit.
 
 ## `kv` Selectors
 
@@ -98,9 +100,9 @@ To select everything, but replace some keys with new values or drop keys entirel
   -@key3}         ; drop key3
 ```
 
-## `vector` Filters
+## `vector` Selectors
 
-`vector` filters are similar to `kv` Selectors, but they are used with `[...]`
+`vector` selectors are similar to `kv` Selectors, but they are used with `[...]`
 
  - `[hello]` select all string items that contain `"hello"`.
  - `[hi hello]` select all string items that contain either `"hello"` or `"hi"`.
@@ -110,6 +112,7 @@ To select everything, but replace some keys with new values or drop keys entirel
  - `[(+@pref? _ "start") (+@post? _ "end")]` select all lines that start with
    `"start"` and end with `"end"`.
  - `[(> _ 3)] select all number items larger than `3`.
+ - `[_ -@hi] select all string items except those that contain `"hi"`.
 
 ## Query Utility Functions
 
@@ -194,3 +197,6 @@ Unfortunately this will tend to be quite slow. To get around this you can
 create an image that has `jqn` preloaded and dump it using
 `sb-ext:save-lisp-and-die`. Then use your image in the alias instead of SBCL.
 
+
+; TODO: warn if jqn reads only some of a file?
+

bin/ex.lisp

@@ -6,29 +6,29 @@
 (ql:quickload :jqn :silent t)
 ; (in-package :jqn)
 
+   ; (jqn:jsnout (jqn:jsnqryf "./sample.json" :db t (|| #{_ _id}) :db t) :indent t)
 
 (defun main ()
   (print
-    ; (jqn:qryd (jqn:jsnloads "{\"a\": {\"b\": 3,
-    ;                                               \"c\": { \"d\": 89 }}}")
-    ;           :q (*new
-    ;                (@_ :a/c/d)
-    ;                (@_ :a/b)
-    ;                )
-    ;           :db t)
-   (jqn:jsnout
-     (jqn:jsnqryf "./sample.json" :db t
-        :q (|| #{_ _id}
-               ; { (a 1) (b 2)}
+    ; (jqn::preproc/$$itr
+    ;     '(ccc :ddd "IIUJ" "%@UU" ?@aa ?@bb ("cc" (progn _))
+    ;       (:% "xBC" (print _)) (:% "ABC" _)))
+    (mapcar #'print
+     (jqn::preproc/**filter
+        '(ccc :ddd "IIUJ" "%@UU" ?@aa ?@bb ("cc" (progn _))
+          (% "ABC" (print _)) (:% "ABC" _))))
 
-               ; {a b}
-               ; (<> [things])
-               ; [id]
-               )
-        :db t
-        )
-        :indent t)
-   ))
+   ; (jqn:jsnout (jqn:jsnqryf "./sample.json" (||
+
+   ;                                            #{(:?@_id (print _))}
+   ;                                            ; #{(:? _id (print _))}
+   ;                                            ; #{(?@_id print _)}
+   ;                                            )
+   ;                          :db t
+   ;                          ) :indent t)
+
+   )
+  )
 
 (main)
 

bin/jqn-sh.lisp

@@ -40,8 +40,7 @@ examples:
     (error (e) (exit-with-msg 2 "jqn: failed to read json file: ~a~%~a" f e))))
 
 (defun jqn/parse-query (args)
-  (handler-case (let ((all (read-all-str args)))
-                  (if (= (length all) 1) (car all) `(|| ,@all)))
+  (handler-case `(|| ,@(read-all-str args))
     (error (e) (exit-with-msg 3 "jqn: failed to parse qry:~%~a" (mkstr e)))))
 
 (defun jqn/run-files (opts q files)

bin/tqn-sh.lisp

@@ -27,8 +27,7 @@ options can be write as -i -v or -iv
     (error (e) (exit-with-msg 2 "tqn: failed to read txt file: ~a~%~a" f e))))
 
 (defun tqn/parse-query (args)
-  (handler-case (let ((all (read-all-str args)))
-                  (if (= (length all) 1) (car all) `(|| ,@all)))
+  (handler-case `(|| ,@(read-all-str args))
     (error (e) (exit-with-msg 3 "tqn: failed to parse qry:~%~a" (mkstr e)))))
 
 (defun tqn/run-files (opts q files)

docs/jqn.md

@@ -235,14 +235,10 @@
  ; 
  ; >< names a compiled function:
  ;   Lambda-list: (O)
- ;   Derived type: (FUNCTION (T)
- ;                  (VALUES
- ;                   (OR LIST HASH-TABLE (SIMPLE-ARRAY * (*))
- ;                       SB-KERNEL:EXTENDED-SEQUENCE)
- ;                   &OPTIONAL))
+ ;   Derived type: (FUNCTION (T) (VALUES T &OPTIONAL))
  ;   Documentation:
  ;     remove none/nil, emtpy arrays, empty objects, empty keys and empty lists from `a`.
- ;   Source file: /data/x/jqn/src/qry.lisp
+ ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 
 #### JQN:??
@@ -402,24 +398,52 @@
  ;   [symbol]
  ; 
  ; IPREF? names a compiled function:
- ;   Lambda-list: (S SUF)
- ;   Derived type: (FUNCTION (T T) *)
+ ;   Lambda-list: (S SUF &OPTIONAL D)
+ ;   Derived type: (FUNCTION (T T &OPTIONAL T) *)
  ;   Documentation:
  ;     case insensitive pref?
  ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 
+#### JQN:IS?
+
+```
+ ; JQN:IS?
+ ;   [symbol]
+ ; 
+ ; IS? names a compiled function:
+ ;   Lambda-list: (K &OPTIONAL D)
+ ;   Derived type: (FUNCTION (T &OPTIONAL T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     k if k is not nil ,not an empty sequence, and not an empty hash-table; or d
+ ;   Source file: /data/x/jqn/src/utils.lisp
+```
+
 #### JQN:ISUB?
 
 ```
  ; JQN:ISUB?
  ;   [symbol]
  ; 
  ; ISUB? names a compiled function:
+ ;   Lambda-list: (S SUB &OPTIONAL D)
+ ;   Derived type: (FUNCTION (T T &OPTIONAL T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     case insensitive sub?
+ ;   Source file: /data/x/jqn/src/utils.lisp
+```
+
+#### JQN:ISUBX?
+
+```
+ ; JQN:ISUBX?
+ ;   [symbol]
+ ; 
+ ; ISUBX? names a compiled function:
  ;   Lambda-list: (S SUB)
  ;   Derived type: (FUNCTION (T T) *)
  ;   Documentation:
- ;     case insensitive check is sub is substring of s.
+ ;     case insensitive subx?
  ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 
@@ -430,8 +454,8 @@
  ;   [symbol]
  ; 
  ; ISUF? names a compiled function:
- ;   Lambda-list: (S SUF)
- ;   Derived type: (FUNCTION (T T) *)
+ ;   Lambda-list: (S SUF &OPTIONAL D)
+ ;   Derived type: (FUNCTION (T T &OPTIONAL T) *)
  ;   Documentation:
  ;     case insensitive suf?
  ;   Source file: /data/x/jqn/src/utils.lisp
@@ -628,10 +652,11 @@
  ;   [symbol]
  ; 
  ; PREF? names a compiled function:
- ;   Lambda-list: (S PREF &AUX (S (MKSTR S)))
- ;   Derived type: (FUNCTION (STRING STRING) (VALUES BOOLEAN &OPTIONAL))
+ ;   Lambda-list: (S PREF &OPTIONAL D &AUX (S (MKSTR S)))
+ ;   Derived type: (FUNCTION (STRING STRING &OPTIONAL T)
+ ;                  (VALUES T &OPTIONAL))
  ;   Documentation:
- ;     t if s starts with pref
+ ;     s if s starts with pref; or d
  ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 
@@ -719,6 +744,21 @@
  ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 
+#### JQN:SIZE
+
+```
+ ; JQN:SIZE
+ ;   [symbol]
+ ; 
+ ; SIZE names a compiled function:
+ ;   Lambda-list: (L)
+ ;   Derived type: (FUNCTION (T)
+ ;                  (VALUES (MOD 4611686018427387901) &OPTIONAL))
+ ;   Documentation:
+ ;     length of sequence l or number of keys in kv l
+ ;   Source file: /data/x/jqn/src/utils.lisp
+```
+
 #### JQN:SPLIT
 
 ```
@@ -769,6 +809,20 @@
  ;   [symbol]
  ; 
  ; SUB? names a compiled function:
+ ;   Lambda-list: (S SUB &OPTIONAL D)
+ ;   Derived type: (FUNCTION (T T &OPTIONAL T) (VALUES T &OPTIONAL))
+ ;   Documentation:
+ ;     s if sub is substring of s; ord
+ ;   Source file: /data/x/jqn/src/utils.lisp
+```
+
+#### JQN:SUBX?
+
+```
+ ; JQN:SUBX?
+ ;   [symbol]
+ ; 
+ ; SUBX? names a compiled function:
  ;   Lambda-list: (S SUB)
  ;   Derived type: (FUNCTION (STRING STRING)
  ;                  (VALUES (OR NULL (MOD 4611686018427387901)) &OPTIONAL))
@@ -784,10 +838,10 @@
  ;   [symbol]
  ; 
  ; SUF? names a compiled function:
- ;   Lambda-list: (S SUF)
- ;   Derived type: (FUNCTION (STRING STRING) *)
+ ;   Lambda-list: (S SUF &OPTIONAL D)
+ ;   Derived type: (FUNCTION (STRING STRING &OPTIONAL T) *)
  ;   Documentation:
- ;     t if s ends with suf
+ ;     s if s ends with suf; or d
  ;   Source file: /data/x/jqn/src/utils.lisp
 ```
 

src/init.lisp

@@ -1,11 +1,11 @@
 (in-package :jqn)
 
-(defvar *qmodes* '(:+@ :?@ :-@ :%@))
+(defvar *qmodes* '(:+ :? :- :%))
 (defvar *fxns* '(:fn :fi :ctx :num :cnt :par :$ :$_ :>< :??
-                 :*cat :$cat *ind *sel *seq :head :tail
+                 :*cat :$cat *ind *sel *seq :head :tail :size
                  :sup :sdwn :mkstr :repl :strcat
-                 :pref? :suf? :sub? :ipref? :isuf? :isub?
-                 :flt? :int? :kv? :lst? :num? :seq? :str? :vec?
+                 :pref? :suf? :sub? :subx? :ipref? :isuf? :isub? :isubx?
+                 :flt? :int? :kv? :lst? :num? :seq? :str? :vec? :is?
                  :int!? :flt!? :num!?
                  :fmt :out))
 

src/packages.lisp

@@ -7,12 +7,12 @@
     #:qryd #:jsnqryf #:qryl
     #:cnt #:ctx #:fi #:fn #:num #:par
     #:|| #:?? #:>< #:<> #:$ :$_ #:noop
-    #:*new #:$new #:$cat #:*cat
-    #:flt? #:int? #:kv? #:lst? #:num? #:seq? #:str? #:vec?
+    #:*new #:$new #:$cat #:*cat #:size
+    #:flt? #:int? #:kv? #:lst? #:num? #:seq? #:str? #:vec? #:is?
     #:int!? #:flt!? #:num!?
     #:*ind #:*sel #:*seq
     #:$add+ #:$add? #:$add% #:$del #:*add+ #:*add? #:*add%
     #:sup #:sdwn #:strcat #:repl #:mkstr #:split
-    #:pref? #:suf? #:sub? #:ipref? #:isuf? #:isub?
+    #:pref? #:suf? #:sub? #:subx? #:ipref? #:isuf? #:isub? #:isubx?
     #:head #:tail #:fmt #:out))