From 12d58f1073e83301b5349c9a2015ae038eef1cf4 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 00:46:16 +0200 Subject: [PATCH 01/25] pasteguard feature added --- clicol/clicol.py | 75 ++++++++++++++++++++++++++++++++++++---- clicol/ini/cm_cisco.ini | 5 +++ clicol/ini/cm_common.ini | 18 ++++++++++ doc/clicol.cfg | 5 +++ 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 95cd463..9d926bb 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -56,6 +56,8 @@ pause = 0 # if true, then coloring is paused pastepause_needed = False # switch for turning off coloring while pasting multiple lines pastepause = False # while pasting multiple lines, turn off coloring +pasteguard = False # While pasting, catch errors and stop pasting +pastebuffer = list() # Buffer containing remaining lines to paste debug = 0 # global debug (D: hidden command) timeout = 0 # counts timeout timeoutact = True # act on timeout warning @@ -66,6 +68,7 @@ RUNNING = True # signal to timeoutcheck WORKING = True # signal to timeoutcheck bufferlock = threading.Lock() +pastelock = threading.Lock() plugins = None # all active plugins # Interactive regex matches for # - prompt (asd# ) (all) @@ -113,6 +116,9 @@ def timeoutcheck(maxwait=1.0): """ global bufferlock, debug, timeout, maxtimeout, charbuffer global RUNNING, WORKING + global conn + global effects + global pasteguard, pastebuffer, pastelock timeout = time.time() while RUNNING: @@ -138,6 +144,35 @@ def timeoutcheck(maxwait=1.0): except threading.ThreadError as e: if debug >= 1: print("\r\n\033[38;5;208mTERR-%s\033[0m\r\n" % e) # DEBUG pass + try: + pastelock.acquire() + if pasteguard and len(pastebuffer) > 0: + lineno = 1 + line = pastebuffer.pop(0) + while len(pastebuffer) > 0: + if 'paste_error' in effects: + print("\r", colorize("Paste error at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_error') + effects.discard('paste_abort') + conn.send("\r") + break + if 'paste_abort' in effects: + print("\r", colorize("Paste aborted at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_abort') + effects.discard('paste_error') + conn.send("\r") + break + line = pastebuffer.pop(0) + lineno += 1 + conn.send(line) + if debug >= 1: print("\r\n\033[38;5;208meffects:%s-PASTE-%s\033[0m\r\n" % (effects, line)) # DEBUG + time.sleep(maxwait) + finally: + pastelock.release() def sigwinch_passthrough(sig, data): @@ -184,9 +219,10 @@ def printhelp(shortcuts): print("%s: \"%s\"" % (key.upper(), value.strip(r'"'))) -def colorize(text, only_effect=None): +def colorize(text, only_effect=None, matchers_only=False): """ This function is manipulating input text + :param matchers_only: use for setting effects and do not modify text :param text: input string to colorize :param only_effect: select specific regex group(with specified effect) to work with :return: manipulated text @@ -202,8 +238,7 @@ def colorize(text, only_effect=None): text = plugins.preprocess(text, effects) except UnicodeDecodeError: pass - except: - raise + for line in text.splitlines(True): cmap_counter = 0 if debug >= 2: print("\r\n\033[38;5;208mC-", repr(line), "\033[0m\r\n") # DEBUG @@ -235,6 +270,8 @@ def colorize(text, only_effect=None): effects.remove('timeoutwarn') preventtimeout() continue + elif matchers_only: + continue origline = line if option == 2: # need to cleanup existing coloring (CLEAR) @@ -247,7 +284,8 @@ def colorize(text, only_effect=None): if len(effect) > 0: # we have an effect effects.add(effect) if 'prompt' in effects: # prompt eliminates all other effects - effects = {'prompt'} + # remove effects other than prompt and pasting + effects.intersection_update({'prompt', 'paste_error', 'paste_abort'}) if option > 0: # non-zero means non-final match break elif option == 2: # need to restore existing coloring as there was no match (by CLEAR) @@ -267,7 +305,8 @@ def ifilter(inputtext): :return: byte array of manipulated input. Type is expected by pexpect! """ global is_break, timeout, prevents, interactive, effects - global pastepause_needed, pastepause + global pastepause_needed, pastepause, pasteguard + global pastebuffer, pastelock is_break = inputtext == b'\x1c' if not is_break: @@ -294,6 +333,23 @@ def ifilter(inputtext): pastepause = True else: pastepause = False + if pasteguard: + if pastelock.locked() and inputtext == b'\x03': + effects.add('paste_abort') + inputtext = b'' + elif pastelock.locked() or len(pastebuffer) > 0: + # do not let user input while pasting + print("\rPasting in progress... Input ignored! CTRL-C to abort pasting!\r") + inputtext = b'' + elif inputtext.count(b'\r') > 1: + pastelock.acquire() + effects.discard('paste_error') + effects.discard('paste_abort') + pastebuffer = inputtext.splitlines(True) + # send only first line + # all other lines will be sent by timeoutchecker + inputtext = pastebuffer[0] + pastelock.release() return inputtext @@ -316,7 +372,7 @@ def ofilter(inputtext): global effects # Coloring is paused by escape character or pasting - if pause or pastepause: + if pause: return inputtext # Normalize input. py2_py3 @@ -324,6 +380,10 @@ def ofilter(inputtext): inputtext = inputtext.decode('utf-8', errors='ignore') except AttributeError: pass + + if pastepause: + return colorize(inputtext, matchers_only=True).encode('utf-8') + bufferlock.acquire() # we got input, have to access buffer exclusively WORKING = True try: @@ -413,6 +473,7 @@ def main(argv=None): global is_break global maxtimeout, maxprevents global pastepause_needed + global pasteguard global RUNNING global plugins highlight = "" @@ -455,6 +516,7 @@ def main(argv=None): 'maxprevents': r'0', 'maxwait': r'1.0', 'pastepause': r'false', + 'pasteguard': r'false', 'update_caption': r'false', 'default_caption': r'%(hostname)s', 'F1': r'show ip interface brief | e unassign\r', @@ -509,6 +571,7 @@ def main(argv=None): maxtimeout = config.getint('clicol', 'maxtimeout') maxprevents = config.getint('clicol', 'maxprevents') pastepause_needed = config.getboolean('clicol', 'pastepause') + pasteguard = config.getboolean('clicol', 'pasteguard') debug = config.getint('clicol', 'debug') colors = ConfigParser.SafeConfigParser() diff --git a/clicol/ini/cm_cisco.ini b/clicol/ini/cm_cisco.ini index aafe8a0..e6f8889 100644 --- a/clicol/ini/cm_cisco.ini +++ b/clicol/ini/cm_cisco.ini @@ -319,3 +319,8 @@ example= ip vrf forwarding SomeVRF tunnel vrf SomeVRF +[cisco_paste_invalid] +matcher=1 +priority=20 +regex=%(BOL)s%% Invalid +effect=paste_error \ No newline at end of file diff --git a/clicol/ini/cm_common.ini b/clicol/ini/cm_common.ini index 270988f..032a1dd 100644 --- a/clicol/ini/cm_common.ini +++ b/clicol/ini/cm_common.ini @@ -153,3 +153,21 @@ example= ******************* * BANNER * ******************* + +[common_paste_error] +priority=100 +regex=%(BOL)s(Paste error) +replacement=\1%(alert)s\2%(default)s +options=%(CLEAR)s +dependency=paste_error +example= + Paste error at line 2: b' ip adress 1.1.1.2 255.255.255.0\r' + +[common_paste_abort] +priority=100 +regex=%(BOL)s(Paste aborted) +replacement=\1%(alert)s\2%(default)s +options=%(CLEAR)s +dependency=paste_abort +example= + Paste error at line 2: b' ip adress 1.1.1.2 255.255.255.0\r' \ No newline at end of file diff --git a/doc/clicol.cfg b/doc/clicol.cfg index 87d571f..177dd03 100644 --- a/doc/clicol.cfg +++ b/doc/clicol.cfg @@ -56,6 +56,11 @@ # default value: false #pastepause=true # +# Paste guarding. If device reports error, we stop pasting. Pasting can be stopped by CTRL-C +# valid options: true|false +# default value: false +#pasteguard=true +# # Update terminal caption for telnet/ssh # valid options: true|false # default value: false From 032180db0e9b3257c27579d707502323581e1e2e Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 09:29:15 +0200 Subject: [PATCH 02/25] code cleanup --- clicol/clicol.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 9d926bb..e723dea 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -469,14 +469,15 @@ def merge_dicts(x, y): def main(argv=None): - global conn, ct, cmap, pause, timeoutact, terminal, charbuffer, lastline, debug + global conn, ct, cmap, pause, timeoutact, charbuffer, lastline, debug global is_break global maxtimeout, maxprevents global pastepause_needed global pasteguard global RUNNING global plugins - highlight = "" + global highlight + regex = "" cfgdir = "~/.clicol" tc = None From c2b77aa2d9769ecd9bd3ec7fa673627d6a088e10 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 09:45:37 +0200 Subject: [PATCH 03/25] help enhancement --- README.md | 15 +++++++++++++++ clicol/clicol.py | 14 ++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fa3c264..79fd352 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,21 @@ Consider using aliases. A basic template can be found in *doc* folder. Your terminal software should support ANSI colors. [Putty](https://www.chiark.greenend.org.uk/~sgtatham/putty/)/[SecureCRT](https://www.vandyke.com/products/securecrt) are tested. I am developing with default colorsets. If you are using other software, colors can differ somewhat. +FEATURES +-------- +A bit more detail on the builtin features. All features are available in the session when pressing CTRL-\ break keysequence. + +1. Pause coloring
+This feature turns off all modifications and features on the output. The only available feature which continues working is the break key CTRL-\ . +You can toggle it by hitting the command. +2. Pasteguard
+This feature tries to prevent you from pasting configuration with errors. It will stop pasting when device return errors. +You can also stop pasting by hitting CTRL-C.
+*Note: error detection is through matchers which provides paste_error effect. You can add your own!* +3. Highlight
+This feature allows you to search for a given text snippet by specifying a regex. The specified text will be highlighted +in the output. Useful if you are searching for something in a large output. It will catch your eyes! + TESTING ------- diff --git a/clicol/clicol.py b/clicol/clicol.py index e723dea..26a6e71 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -70,6 +70,7 @@ bufferlock = threading.Lock() pastelock = threading.Lock() plugins = None # all active plugins +highlight = re.compile("") # highlight pattern # Interactive regex matches for # - prompt (asd# ) (all) # - question ([yes]) (cisco) @@ -205,13 +206,18 @@ def printhelp(shortcuts): :param shortcuts: shortcut key list """ global plugins + global pasteguard + global pause + global highlight print( """ commands in BREAK mode: q: quit program -p: pause coloring -T: highlight regex (empty turns off)""") +p: [%s] toggle pause coloring +g: [%s] toggle pasteguard +T: [%s] highlight regex (empty turns off)""" % ("On" if pause else "Off", "On" if pasteguard else "Off", + highlight.pattern)) for key in plugins.keybinds.keys(): print("%s:%s" % (key, plugins.keybinds[key].plugin_help(key))) print("Shortcuts") @@ -738,8 +744,8 @@ def main(argv=None): conn.interact(escape_character='\x1c' if PY3 else b'\x1c', output_filter=ofilter, input_filter=ifilter) if is_break: is_break = False - print("\r" + " " * 100 + "\rCLICOL: q:quit,p:pause,T:highlight,F1-12,SF1-8:shortcuts,h-help", - end='') + print("\r" + " " * 100 + + "\rCLICOL: q:quit,p:pause,g:pasteguard,T:highlight,F1-12,SF1-8:shortcuts,h-help", end='') command = getcommand() if command == "D": debug += 1 From 1fb8f0a807962ebc3553342315ed396ecce917d5 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 09:46:03 +0200 Subject: [PATCH 04/25] help enhancement --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 79fd352..5f4eaa9 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Default break key is CTRL-\ After hitting the break key you have some options:
p - pausing coloring
+g - toggle pasteguard
q - quit from session
h - print help
T - highlight regex (set regex in runtime to highligh something important) From 5cc0a0a75c81030bb25757423efae2a601aba760 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 09:47:34 +0200 Subject: [PATCH 05/25] pasteguard feature fixes maxwait defaulted to a smaller value to support pasteguard more. This value may be even lower. Worth to tune in case of high response time devices. --- clicol/clicol.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 26a6e71..68786ff 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -106,7 +106,7 @@ def sigint_handler(sig, data): signal.signal(signal.SIGINT, sigint_handler) -def timeoutcheck(maxwait=1.0): +def timeoutcheck(maxwait=0.3): """ This thread is responsible for outputting the buffer if the predefined timeout is overlapped. pexpect.interact does lock the main thread, this thread will dump the buffer if we experience unexpected input @@ -367,7 +367,9 @@ def ofilter(inputtext): """ global charbuffer global pause # coloring must be paused + global pastelock global pastepause + global pastebuffer global lastline global debug global bufferlock @@ -392,6 +394,7 @@ def ofilter(inputtext): bufferlock.acquire() # we got input, have to access buffer exclusively WORKING = True + pastingcolors = ['prompt', 'paste_error'] if pastelock.locked() or pastebuffer else None try: # If not ending with linefeed we are interacting or buffering if not (inputtext[-1] == "\r" or inputtext[-1] == "\n"): @@ -413,7 +416,7 @@ def ofilter(inputtext): if debug: print("\r\n\033[38;5;208mINTERACT/effects:", repr(effects), "\033[0m\r\n") # DEBUG bufout = charbuffer charbuffer = "" - bufout = colorize(bufout).encode('utf-8') + bufout = colorize(bufout, pastingcolors).encode('utf-8') if 'prompt' in effects: effects.discard('prompt') interactive = True @@ -425,7 +428,7 @@ def ofilter(inputtext): if "\r" in inputtext or "\n" in inputtext: # multiline input, not interactive bufout = "".join(charbuffer.splitlines(True)[:-1]) # all buffer except last line charbuffer = lastline # delete printed text. last line remains in buffer - bufout = colorize(bufout).encode('utf-8') + bufout = colorize(bufout, pastingcolors).encode('utf-8') # Ignore prompt in input effects.discard('prompt') interactive = False @@ -442,7 +445,7 @@ def ofilter(inputtext): return b"" else: charbuffer = lastline # delete printed text. last line remains in buffer - bufout = colorize(bufout).encode('utf-8') + bufout = colorize(bufout, pastingcolors).encode('utf-8') # Ignore prompt in input effects.discard('prompt') return bufout @@ -451,7 +454,7 @@ def ofilter(inputtext): # Got linefeed, dump buffer bufout = charbuffer + inputtext charbuffer = "" - bufout = colorize(bufout).encode('utf-8') + bufout = colorize(bufout, pastingcolors).encode('utf-8') # Ignore prompt in input effects.discard('prompt') interactive = False @@ -770,6 +773,8 @@ def main(argv=None): cmap[0] = cmap_highlight else: cmap.insert(0, cmap_highlight) + elif command == "g": + pasteguard = not pasteguard elif command == "h": printhelp(shortcuts) elif command in plugins.keybinds.keys(): From 15b7362b2ede76953bd28c60a26cd74b02fd04f7 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 12:33:18 +0200 Subject: [PATCH 06/25] pasteguard feature fixes - bulk paste fix --- clicol/clicol.py | 66 +++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 68786ff..79593b7 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -145,35 +145,36 @@ def timeoutcheck(maxwait=0.3): except threading.ThreadError as e: if debug >= 1: print("\r\n\033[38;5;208mTERR-%s\033[0m\r\n" % e) # DEBUG pass - try: + + if pasteguard and len(pastebuffer) > 0: + lineno = 1 pastelock.acquire() - if pasteguard and len(pastebuffer) > 0: - lineno = 1 - line = pastebuffer.pop(0) - while len(pastebuffer) > 0: - if 'paste_error' in effects: - print("\r", colorize("Paste error at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() - effects.discard('paste_error') - effects.discard('paste_abort') - conn.send("\r") - break - if 'paste_abort' in effects: - print("\r", colorize("Paste aborted at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() - effects.discard('paste_abort') - effects.discard('paste_error') - conn.send("\r") - break - line = pastebuffer.pop(0) - lineno += 1 - conn.send(line) - if debug >= 1: print("\r\n\033[38;5;208meffects:%s-PASTE-%s\033[0m\r\n" % (effects, line)) # DEBUG - time.sleep(maxwait) - finally: + line = pastebuffer.pop(0) pastelock.release() + while len(pastebuffer) > 0: + if 'paste_error' in effects: + print("\r", colorize("Paste error at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_error') + effects.discard('paste_abort') + conn.send("\r") + break + if 'paste_abort' in effects: + print("\r", colorize("Paste aborted at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_abort') + effects.discard('paste_error') + conn.send("\r") + break + pastelock.acquire() + line = pastebuffer.pop(0) + pastelock.release() + lineno += 1 + conn.send(line) + if debug >= 1: print("\r\n\033[38;5;208meffects:%s-PASTE-%s\033[0m\r\n" % (effects, line)) # DEBUG + time.sleep(maxwait) def sigwinch_passthrough(sig, data): @@ -340,13 +341,16 @@ def ifilter(inputtext): else: pastepause = False if pasteguard: - if pastelock.locked() and inputtext == b'\x03': + if (pastelock.locked() or len(pastebuffer) > 0) and inputtext == b'\x03': effects.add('paste_abort') inputtext = b'' - elif pastelock.locked() or len(pastebuffer) > 0: - # do not let user input while pasting - print("\rPasting in progress... Input ignored! CTRL-C to abort pasting!\r") + elif (pastelock.locked() or len(pastebuffer) > 0) and \ + not (effects.intersection({'paste_error', 'paste_abort'})): + # If we got bulk data or user did input, add it to the buffer + pastelock.acquire() + pastebuffer += inputtext.splitlines(True) inputtext = b'' + pastelock.release() elif inputtext.count(b'\r') > 1: pastelock.acquire() effects.discard('paste_error') From bb92375bdc9ae1873d4418f184f7e69074e280d8 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 14:53:23 +0200 Subject: [PATCH 07/25] thread safety fix --- clicol/clicol.py | 78 ++++++++++++++++++++++------------------- clicol/ini/cm_cisco.ini | 12 +++++-- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 79593b7..728dcfa 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -36,6 +36,7 @@ import threading import time import signal +from heapq import heappush, heappop from socket import gethostname from pkg_resources import resource_filename from .command import getcommand @@ -67,6 +68,7 @@ maxprevents = 0 # maximum number of timeout prevention (0 turns this off) RUNNING = True # signal to timeoutcheck WORKING = True # signal to timeoutcheck +PASTING = False # signal that pasting is in progress bufferlock = threading.Lock() pastelock = threading.Lock() plugins = None # all active plugins @@ -116,7 +118,7 @@ def timeoutcheck(maxwait=0.3): :param maxwait: float timeout value in seconds what we wait for end of output from device. """ global bufferlock, debug, timeout, maxtimeout, charbuffer - global RUNNING, WORKING + global RUNNING, WORKING, PASTING global conn global effects global pasteguard, pastebuffer, pastelock @@ -147,34 +149,35 @@ def timeoutcheck(maxwait=0.3): pass if pasteguard and len(pastebuffer) > 0: - lineno = 1 - pastelock.acquire() - line = pastebuffer.pop(0) - pastelock.release() + PASTING = True while len(pastebuffer) > 0: - if 'paste_error' in effects: - print("\r", colorize("Paste error at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() - effects.discard('paste_error') - effects.discard('paste_abort') - conn.send("\r") - break - if 'paste_abort' in effects: - print("\r", colorize("Paste aborted at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() - effects.discard('paste_abort') - effects.discard('paste_error') - conn.send("\r") - break pastelock.acquire() - line = pastebuffer.pop(0) + lines = heappop(pastebuffer)[1] pastelock.release() - lineno += 1 - conn.send(line) - if debug >= 1: print("\r\n\033[38;5;208meffects:%s-PASTE-%s\033[0m\r\n" % (effects, line)) # DEBUG - time.sleep(maxwait) + lineno = 0 + while len(lines) > 0: + if 'paste_error' in effects: + print("\r", colorize("Paste error at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_error') + effects.discard('paste_abort') + conn.send("\r") + break + if 'paste_abort' in effects: + print("\r", colorize("Paste aborted at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") + pastebuffer = list() + effects.discard('paste_abort') + effects.discard('paste_error') + conn.send("\r") + break + line = lines.pop(0) + lineno += 1 + conn.send(line) + if debug >= 1: print("\r\n\033[38;5;208meffects:%s-PASTE-%s\033[0m\r\n" % (effects, line)) # DEBUG + time.sleep(maxwait) + PASTING = False def sigwinch_passthrough(sig, data): @@ -314,6 +317,7 @@ def ifilter(inputtext): global is_break, timeout, prevents, interactive, effects global pastepause_needed, pastepause, pasteguard global pastebuffer, pastelock + global PASTING is_break = inputtext == b'\x1c' if not is_break: @@ -341,24 +345,24 @@ def ifilter(inputtext): else: pastepause = False if pasteguard: - if (pastelock.locked() or len(pastebuffer) > 0) and inputtext == b'\x03': + if PASTING and inputtext == b'\x03': effects.add('paste_abort') inputtext = b'' - elif (pastelock.locked() or len(pastebuffer) > 0) and \ - not (effects.intersection({'paste_error', 'paste_abort'})): + elif PASTING and not (effects.intersection({'paste_error', 'paste_abort'})): # If we got bulk data or user did input, add it to the buffer pastelock.acquire() - pastebuffer += inputtext.splitlines(True) + heappush(pastebuffer, (time.time(), inputtext.splitlines(True))) inputtext = b'' pastelock.release() - elif inputtext.count(b'\r') > 1: + elif len(inputtext) > inputtext.count(b'\r') > 1: + # start pasting. Ignore ENTERs only. That's when user lye on the button. pastelock.acquire() + PASTING = True effects.discard('paste_error') effects.discard('paste_abort') - pastebuffer = inputtext.splitlines(True) - # send only first line - # all other lines will be sent by timeoutchecker - inputtext = pastebuffer[0] + heappush(pastebuffer, (time.time(), inputtext.splitlines(True))) + # all lines will be sent by timeoutchecker + inputtext = b'' pastelock.release() return inputtext @@ -377,7 +381,7 @@ def ofilter(inputtext): global lastline global debug global bufferlock - global WORKING + global WORKING, PASTING global plugins global interactive global timeout @@ -398,7 +402,7 @@ def ofilter(inputtext): bufferlock.acquire() # we got input, have to access buffer exclusively WORKING = True - pastingcolors = ['prompt', 'paste_error'] if pastelock.locked() or pastebuffer else None + pastingcolors = ['prompt', 'paste_error'] if PASTING else None try: # If not ending with linefeed we are interacting or buffering if not (inputtext[-1] == "\r" or inputtext[-1] == "\n"): diff --git a/clicol/ini/cm_cisco.ini b/clicol/ini/cm_cisco.ini index e6f8889..f862f99 100644 --- a/clicol/ini/cm_cisco.ini +++ b/clicol/ini/cm_cisco.ini @@ -319,8 +319,14 @@ example= ip vrf forwarding SomeVRF tunnel vrf SomeVRF -[cisco_paste_invalid] +[cisco_ios_paste_error] matcher=1 priority=20 -regex=%(BOL)s%% Invalid -effect=paste_error \ No newline at end of file +regex=%(BOL)s%% (Invalid|Ambiguous) +effect=paste_error + +[cisco_asa_paste_error] +matcher=1 +priority=20 +regex=%(BOL)s(ERROR|WARNING): +effect=paste_error From 3e347be9feed8b00255d03b71ac93b9741121c9f Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 15:43:14 +0200 Subject: [PATCH 08/25] small fixes --- clicol/clicol.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 728dcfa..584c53c 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -150,27 +150,27 @@ def timeoutcheck(maxwait=0.3): if pasteguard and len(pastebuffer) > 0: PASTING = True + lineno = 0 while len(pastebuffer) > 0: pastelock.acquire() lines = heappop(pastebuffer)[1] pastelock.release() - lineno = 0 while len(lines) > 0: if 'paste_error' in effects: - print("\r", colorize("Paste error at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") + conn.send("\r") + print("\r", " "*100, "\r", colorize("Paste error at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") pastebuffer = list() effects.discard('paste_error') effects.discard('paste_abort') - conn.send("\r") break if 'paste_abort' in effects: - print("\r", colorize("Paste aborted at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") + conn.send("\r") + print("\r", " "*100, "\r", colorize("Paste aborted at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") pastebuffer = list() effects.discard('paste_abort') effects.discard('paste_error') - conn.send("\r") break line = lines.pop(0) lineno += 1 From c85377bd7de93efa42e167a867cbfad910a31ecb Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Wed, 8 Apr 2020 15:46:16 +0200 Subject: [PATCH 09/25] small fixes --- clicol/clicol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clicol/clicol.py b/clicol/clicol.py index 584c53c..10a0b8a 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -151,6 +151,7 @@ def timeoutcheck(maxwait=0.3): if pasteguard and len(pastebuffer) > 0: PASTING = True lineno = 0 + line = "" while len(pastebuffer) > 0: pastelock.acquire() lines = heappop(pastebuffer)[1] From 03dcdc3a206000b220d953aa299917d3dc54e408 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Thu, 9 Apr 2020 08:26:21 +0200 Subject: [PATCH 10/25] maxwait default changed --- doc/clicol.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/clicol.cfg b/doc/clicol.cfg index 177dd03..937d72c 100644 --- a/doc/clicol.cfg +++ b/doc/clicol.cfg @@ -44,12 +44,12 @@ # default value: 0 #debug=0 # -# Wait timer for timeoutcheck in seconds -# might need to increase in case of slow connections/devices -# or decrease in case of local connection +# Wait timer for timeoutcheck in seconds might need to increase in case of slow connections/devices +# or decrease in case of local connection. If you see glitches by pasting with pasteguard, +# you might need increase this value # valid options: any float value > 0.0 -# default value: 1.0 -#maxwait=2.0 +# default value: 0.3 +#maxwait=0.5 # # Pause coloring while pasting multiple lines # valid options: true|false From 738a6248f4ac52045531064bcc3634027d49cfc1 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 11:20:16 +0200 Subject: [PATCH 11/25] bugfix for set --- clicol/clicol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 10a0b8a..ac28711 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -403,7 +403,7 @@ def ofilter(inputtext): bufferlock.acquire() # we got input, have to access buffer exclusively WORKING = True - pastingcolors = ['prompt', 'paste_error'] if PASTING else None + pastingcolors = {'prompt', 'paste_error'} if PASTING else None try: # If not ending with linefeed we are interacting or buffering if not (inputtext[-1] == "\r" or inputtext[-1] == "\n"): @@ -420,7 +420,7 @@ def ofilter(inputtext): lastline[0] != "\a" and lastline[0] != "\b") and inputtext == lastline: bufout = charbuffer charbuffer = "" - return colorize(bufout, ["prompt"]).encode('utf-8') + return colorize(bufout, {"prompt"}).encode('utf-8') if INTERACT.search(lastline): # prompt or question at the end if debug: print("\r\n\033[38;5;208mINTERACT/effects:", repr(effects), "\033[0m\r\n") # DEBUG bufout = charbuffer @@ -445,7 +445,7 @@ def ofilter(inputtext): elif interactive or 'prompt' in effects or 'ping' in effects: charbuffer = "" # colorize only short stuff (up key,ping) - return colorize(bufout, ["prompt", "ping"]).encode('utf-8') + return colorize(bufout, {"prompt", "ping"}).encode('utf-8') else: # need to collect more output return b"" else: # large data. we need to print until last line which goes into buffer From ea118f750a02354d16d34c5ed0b7d1f0312e3a96 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 11:22:42 +0200 Subject: [PATCH 12/25] documentation update --- clicol/clicol.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clicol/clicol.py b/clicol/clicol.py index ac28711..ed1a253 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -233,6 +233,9 @@ def printhelp(shortcuts): def colorize(text, only_effect=None, matchers_only=False): """ This function is manipulating input text + :type only_effect: set + :type matchers_only: bool + :type text: str :param matchers_only: use for setting effects and do not modify text :param text: input string to colorize :param only_effect: select specific regex group(with specified effect) to work with From 1ae10f93a0786bab548a5d4c79b5602cbadcdc1d Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 11:23:17 +0200 Subject: [PATCH 13/25] small refactoring --- clicol/clicol.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index ed1a253..74a8b8b 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -157,22 +157,16 @@ def timeoutcheck(maxwait=0.3): lines = heappop(pastebuffer)[1] pastelock.release() while len(lines) > 0: - if 'paste_error' in effects: + if effects.intersection({'paste_error', 'paste_abort'}): conn.send("\r") - print("\r", " "*100, "\r", colorize("Paste error at line %s: %s" % + print("\r", " "*100, "\r", colorize("Paste " + + ("error" if 'paste_error' in effects else "aborted") + + " at line %s: %s" % (lineno, line.decode('utf-8', errors='ignore'))), sep="") pastebuffer = list() effects.discard('paste_error') effects.discard('paste_abort') break - if 'paste_abort' in effects: - conn.send("\r") - print("\r", " "*100, "\r", colorize("Paste aborted at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() - effects.discard('paste_abort') - effects.discard('paste_error') - break line = lines.pop(0) lineno += 1 conn.send(line) @@ -270,7 +264,7 @@ def colorize(text, only_effect=None, matchers_only=False): cdebug = i[7] # debug name = i[8] # regex name - if only_effect != [] and effect not in only_effect: # check if only specified regexes should be used + if only_effect and effect not in only_effect: # check if only specified regexes should be used continue # move on to the next regex if len(dep) > 0 and dep not in effects: # we don't meet our dependency continue # move on to the next regex From 1ed806fd5bc48b5c8883f178a76ac9ecd0d42a78 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:38:01 +0200 Subject: [PATCH 14/25] argument handling fix --- clicol/clicol.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 74a8b8b..bdc5fb5 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -497,6 +497,7 @@ def main(argv=None): cfgdir = "~/.clicol" tc = None caption = "" + cmd = None try: if not argv: argv = sys.argv @@ -513,7 +514,8 @@ def main(argv=None): del argv[1] # remove --caption from args del argv[1] # remove caption string from args except IndexError: # index error, wrong call - cmd = 'error' + print("Wrong arguments!\n") + cmd = "error" hostname = gethostname() default_config = { @@ -654,7 +656,8 @@ def main(argv=None): # Check how we were called # valid options: clicol-telnet, clicol-ssh, clicol-test - cmd = str(os.path.basename(argv[0])).replace('clicol-', '') + if cmd != "error": + cmd = str(os.path.basename(argv[0])).replace('clicol-', '') if cmd == 'test' and len(argv) > 1: # Print starttime: From 3f6434584739088529889b6e7836f92f49129a3a Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:38:53 +0200 Subject: [PATCH 15/25] cleanup --- clicol/clicol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index bdc5fb5..620f7e8 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -691,14 +691,14 @@ def main(argv=None): print(repr(test_d['regex'])) print(repr(test_d['replacement'])) - except: + except KeyError: pass print("%s" % plugins.runtests()) elif cmd == 'file' and len(argv) > 1: try: f = open(argv[1], 'r') - except: + except IOError: print("Error opening " + argv[1]) raise for line in f: @@ -789,7 +789,7 @@ def main(argv=None): elif command in plugins.keybinds.keys(): plugins.keybinds[command].plugin_command(command) - print("\r" + " " * 100 + "\r" + colorize(lastline, ["prompt"]), end='') # restore last line/prompt + print("\r" + " " * 100 + "\r" + colorize(lastline, {"prompt"}), end='') # restore last line/prompt if command is not None: for (key, value) in shortcuts: From aecbbb408c934a9da99c7d650f70e576ceb7e7d4 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:43:25 +0200 Subject: [PATCH 16/25] docs update --- README.md | 9 ++++++-- clicol/clicol.py | 57 +++++++++++++++++++++++++----------------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5f4eaa9..77516f8 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,13 @@ Available command line option for clicol:
`clicol-{telnet|ssh} [--c {colormap}] [--cfgdir {dir}] [--caption {caption}] [args]`
`clicol-file [--c {colormap}] [--cfgdir {dir}] {inputfile}`
`clicol-cmd [--c {colormap}] [--cfgdir {dir}] {command} [args]`
-`clicol-test [--c {colormap}] [--cfgdir {dir}] {colormap regex name (e.g.: '.*' or 'cisco_if|juniper_if')}` +`clicol-test [--c {colormap}] [--cfgdir {dir}] [--plugins] {colormap regex (.*, common.*, etc)}` Explanation for arguments:
`--c {colormap}` : use only specified colormap (`all`, `common`, `cisco`, `juniper`) Defaulted to `all`
`--cfgdir {dir}` : use specified config directory. Defaulted to `~/.clicol`
-`--caption {caption}`: use this caption template (you can use `%(host)s` for connected device and `%(hostname)s` for actual host name) Defaulted to `%(host)s` +`--caption {caption}`: use this caption template (you can use `%(host)s` for connected device and `%(hostname)s` for actual host name) Defaulted to `%(host)s`
+`--plugins`: run also plugin tests clicol can be run on Windows in [cygwin](https://www.cygwin.com). If you want to use [SecureCRT](https://www.vandyke.com/products/securecrt), you must enable sshd in [cygwin](https://www.cygwin.com) and connect to localhost. It is not necessary to be administrator on the desktop for this to work. You must bind to localhost and use port number >1024. @@ -106,6 +107,10 @@ Then the desired regex can be specified in the clicol.cfg in your $HOME and only Output can be tested by running `clicol-file {filename}` script. This will colorize the textfile and dump it. Good for testing. +Plugins can be tested by calling with `--plugins` argument: + +`clicol-test --plugins` + CUSTOMIZING ----------- diff --git a/clicol/clicol.py b/clicol/clicol.py index 620f7e8..c36c6a1 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . If you need to contact the author, you can do so by emailing: - vkertesz2 [~at~] gmail [/dot\] com + vkertesz2 [~at~] gmail [dot] com """ from __future__ import absolute_import from __future__ import division @@ -47,32 +47,32 @@ # Global variables PY3 = sys.version_info.major == 3 -conn = None # connection handler -charbuffer = u'' # input buffer -lastline = u'' # input buffer's last line -is_break = False # is break key pressed? -effects = set() # state effects set -ct = dict() # color table (contains colors) -cmap = list() # color map (contains coloring rules) -pause = 0 # if true, then coloring is paused +conn = None # connection handler +charbuffer = u'' # input buffer +lastline = u'' # input buffer's last line +is_break = False # is break key pressed? +effects = set() # state effects set +ct = dict() # color table (contains colors) +cmap = list() # color map (contains coloring rules) +pause = 0 # if true, then coloring is paused pastepause_needed = False # switch for turning off coloring while pasting multiple lines -pastepause = False # while pasting multiple lines, turn off coloring -pasteguard = False # While pasting, catch errors and stop pasting -pastebuffer = list() # Buffer containing remaining lines to paste -debug = 0 # global debug (D: hidden command) -timeout = 0 # counts timeout -timeoutact = True # act on timeout warning -maxtimeout = 0 # maximum timeout (0 turns off this feature) -interactive = False # signal to buffering -prevents = 0 # counts timeout prevention -maxprevents = 0 # maximum number of timeout prevention (0 turns this off) -RUNNING = True # signal to timeoutcheck -WORKING = True # signal to timeoutcheck -PASTING = False # signal that pasting is in progress -bufferlock = threading.Lock() -pastelock = threading.Lock() -plugins = None # all active plugins -highlight = re.compile("") # highlight pattern +pastepause = False # while pasting multiple lines, turn off coloring +pasteguard = False # While pasting, catch errors and stop pasting +pastebuffer = list() # Buffer containing remaining lines to paste +debug = 0 # global debug (D: hidden command) +timeout = 0 # counts timeout +timeoutact = True # act on timeout warning +maxtimeout = 0 # maximum timeout (0 turns off this feature) +interactive = False # signal to buffering +prevents = 0 # counts timeout prevention +maxprevents = 0 # maximum number of timeout prevention (0 turns this off) +RUNNING = True # signal to timeoutcheck +WORKING = True # signal to timeoutcheck +PASTING = False # signal that pasting is in progress +bufferlock = threading.Lock() # lock charbuffer for read/write +pastelock = threading.Lock() # lock pastebuffer for read/write +plugins = None # all active plugins +highlight = re.compile("") # highlight pattern # Interactive regex matches for # - prompt (asd# ) (all) # - question ([yes]) (cisco) @@ -368,7 +368,10 @@ def ifilter(inputtext): def ofilter(inputtext): """ This function manipulate output text. + :type inputtext: bytes + :type testrun: bool :param inputtext: UTF-8 encoded text to manipulate + :param testrun: upon testrun we don't care about dependencies :return: byte array of manipulated input. Type is expected by pexpect! """ global charbuffer @@ -820,7 +823,7 @@ def main(argv=None): Usage: clicol-{telnet|ssh} [--c {colormap}] [--cfgdir {dir}] [--caption {caption}] [args] Usage: clicol-file [--c {colormap}] [--cfgdir {dir}] {inputfile} Usage: clicol-cmd [--c {colormap}] [--cfgdir {dir}] {command} [args] -Usage: clicol-test [--c {colormap}] [--cfgdir {dir}] {colormap regex name (e.g.: '.*' or 'cisco_if|juniper_if')} +Usage: clicol-test [--c {colormap}] [--cfgdir {dir}] [--plugins] {colormap regex (.*, common.*, etc)} Usage while in session Press break key CTRL-\\""") From 9478969ccdd8848667cab3536146196851a089a3 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:44:20 +0200 Subject: [PATCH 17/25] fix test handling Added `all` effect which means all dependency should met. Good for testrun. --- clicol/clicol.py | 18 +++++++++++++----- clicol/ini/cm_common.ini | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index c36c6a1..91fd849 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -264,9 +264,10 @@ def colorize(text, only_effect=None, matchers_only=False): cdebug = i[7] # debug name = i[8] # regex name - if only_effect and effect not in only_effect: # check if only specified regexes should be used + # check if only specified regexes should be used + if only_effect and (effect not in only_effect) and ('all' not in only_effect): continue # move on to the next regex - if len(dep) > 0 and dep not in effects: # we don't meet our dependency + if len(dep) > 0 and (dep not in effects) and ('all' not in only_effect): # we don't meet our dependency continue # move on to the next regex if cdebug > 0: print("\r\n\033[38;5;208mD-", name, repr(line), repr(effects), "\033[0m\r\n") # debug @@ -365,7 +366,7 @@ def ifilter(inputtext): return inputtext -def ofilter(inputtext): +def ofilter(inputtext, testrun=False): """ This function manipulate output text. :type inputtext: bytes @@ -463,6 +464,8 @@ def ofilter(inputtext): # Got linefeed, dump buffer bufout = charbuffer + inputtext charbuffer = "" + if testrun: + pastingcolors = {'all'} bufout = colorize(bufout, pastingcolors).encode('utf-8') # Ignore prompt in input effects.discard('prompt') @@ -497,6 +500,7 @@ def main(argv=None): global highlight regex = "" + plugintestrun = False cfgdir = "~/.clicol" tc = None caption = "" @@ -516,6 +520,9 @@ def main(argv=None): caption = argv[2] del argv[1] # remove --caption from args del argv[1] # remove caption string from args + if len(argv) > 1 and argv[1] == '--plugins': # run plugin tests + plugintestrun = True + del argv[1] except IndexError: # index error, wrong call print("Wrong arguments!\n") cmd = "error" @@ -662,7 +669,7 @@ def main(argv=None): if cmd != "error": cmd = str(os.path.basename(argv[0])).replace('clicol-', '') - if cmd == 'test' and len(argv) > 1: + if cmd == 'test' and (len(argv) > 1 or plugintestrun): # Print starttime: print("Starttime: %s s" % (round(time.time() - starttime, 3))) # Sanity check on colormaps @@ -682,7 +689,7 @@ def main(argv=None): match_in_regex = re.findall(r'(? 1: try: diff --git a/clicol/ini/cm_common.ini b/clicol/ini/cm_common.ini index 032a1dd..20788ca 100644 --- a/clicol/ini/cm_common.ini +++ b/clicol/ini/cm_common.ini @@ -161,7 +161,7 @@ replacement=\1%(alert)s\2%(default)s options=%(CLEAR)s dependency=paste_error example= - Paste error at line 2: b' ip adress 1.1.1.2 255.255.255.0\r' + Paste error at line 2: ip adress 1.1.1.2 255.255.255.0 [common_paste_abort] priority=100 @@ -170,4 +170,4 @@ replacement=\1%(alert)s\2%(default)s options=%(CLEAR)s dependency=paste_abort example= - Paste error at line 2: b' ip adress 1.1.1.2 255.255.255.0\r' \ No newline at end of file + Paste aborted at line 2: ip adress 1.1.1.2 255.255.255.0 \ No newline at end of file From c7edeabac8ac32ecaa18118d6f5d215172fe2b3d Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:44:38 +0200 Subject: [PATCH 18/25] some formatting fix --- clicol/clicol.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 91fd849..e410eec 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -159,10 +159,10 @@ def timeoutcheck(maxwait=0.3): while len(lines) > 0: if effects.intersection({'paste_error', 'paste_abort'}): conn.send("\r") - print("\r", " "*100, "\r", colorize("Paste " + - ("error" if 'paste_error' in effects else "aborted") + - " at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") + print("\r", " " * 100, "\r", colorize("Paste " + + ("error" if 'paste_error' in effects else "aborted") + + " at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="") pastebuffer = list() effects.discard('paste_error') effects.discard('paste_abort') From d41db2fdebb45fc855112a410f5052bf2bb1343e Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Sun, 26 Apr 2020 13:45:12 +0200 Subject: [PATCH 19/25] py2/py3 compatibility fixes --- clicol/clicol.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index e410eec..2ee2a42 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -22,11 +22,11 @@ from __future__ import unicode_literals try: - # python2 - import ConfigParser -except ImportError: # python3 - import configparser as ConfigParser + import configparser +except ImportError: + # python2 + import ConfigParser as configparser import os import sys @@ -568,9 +568,9 @@ def main(argv=None): 'SF7': r'', 'SF8': r'', } try: - config = ConfigParser.SafeConfigParser(default_config, allow_no_value=True) + config = configparser.SafeConfigParser(default_config, allow_no_value=True) except TypeError: - config = ConfigParser.SafeConfigParser(default_config) # keep compatibility with pre2.7 + config = configparser.SafeConfigParser(default_config) # keep compatibility with pre2.7 starttime = time.time() config.add_section('clicol') # Read config in this order (last are the lastly read, therefore it overrides everything set before) @@ -578,7 +578,7 @@ def main(argv=None): os.path.expanduser('~/clicol.cfg')]) terminal = config.get('clicol', 'terminal') plugincfgfile = config.get('clicol', 'plugincfg') - plugincfg = ConfigParser.SafeConfigParser() + plugincfg = configparser.SafeConfigParser() plugincfg.read([os.path.expanduser(plugincfgfile)]) shortcuts = [o_v for o_v in config.items('clicol') if @@ -602,12 +602,12 @@ def main(argv=None): pasteguard = config.getboolean('clicol', 'pasteguard') debug = config.getint('clicol', 'debug') - colors = ConfigParser.SafeConfigParser() + colors = configparser.SafeConfigParser() colors.read([resource_filename(__name__, 'ini/colors_' + terminal + '.ini'), os.path.expanduser(cfgdir + '/clicol_customcolors.ini'), os.path.expanduser('~/clicol_customcolors.ini')]) - ctfile = ConfigParser.SafeConfigParser(dict(colors.items('colors'))) + ctfile = configparser.SafeConfigParser(dict(colors.items('colors'))) del colors if cct == "dbg_net" or cct == "lbg_net": ctfile.read( @@ -639,7 +639,7 @@ def main(argv=None): ct[key] = value.decode('unicode_escape') except AttributeError: pass - cmaps = ConfigParser.SafeConfigParser(merge_dicts(ct, default_cmap)) + cmaps = configparser.SafeConfigParser(merge_dicts(ct, default_cmap)) if len(regex) == 0: regex = config.get('clicol', 'regex') if regex == "all": @@ -714,7 +714,8 @@ def main(argv=None): raise for line in f: # convert to CRLF to support files created in linux - print(ofilter(line).decode() if PY3 else ofilter(line), end='') + print(ofilter(line.encode('utf-8', 'ignore')).decode() if PY3 + else ofilter(line), end='') f.close() elif cmd == 'telnet' or cmd == 'ssh' or (cmd == 'cmd' and len(argv) > 1): try: From bdbf0857666b22ed453f9c50ed039a56b8d8c6e7 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 4 May 2020 22:58:52 +0200 Subject: [PATCH 20/25] use builtins instead of function --- clicol/clicol.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 2ee2a42..5a7111e 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -52,13 +52,13 @@ lastline = u'' # input buffer's last line is_break = False # is break key pressed? effects = set() # state effects set -ct = dict() # color table (contains colors) -cmap = list() # color map (contains coloring rules) +ct = {} # color table (contains colors) +cmap = [] # color map (contains coloring rules) pause = 0 # if true, then coloring is paused pastepause_needed = False # switch for turning off coloring while pasting multiple lines pastepause = False # while pasting multiple lines, turn off coloring pasteguard = False # While pasting, catch errors and stop pasting -pastebuffer = list() # Buffer containing remaining lines to paste +pastebuffer = [] # Buffer containing remaining lines to paste debug = 0 # global debug (D: hidden command) timeout = 0 # counts timeout timeoutact = True # act on timeout warning @@ -673,7 +673,7 @@ def main(argv=None): # Print starttime: print("Starttime: %s s" % (round(time.time() - starttime, 3))) # Sanity check on colormaps - cmbuf = list() + cmbuf = [] for cm in cmap: # search for duplicate patterns if cm[4] in cmbuf: From 9feee028273e9aa62e11a8e2441ba4f9b836bc76 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 4 May 2020 22:59:09 +0200 Subject: [PATCH 21/25] fix window resize --- clicol/clicol.py | 54 +++++++++++++++++++++++++++-------------------- clicol/command.py | 7 +++--- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 5a7111e..3cca82b 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -73,6 +73,8 @@ pastelock = threading.Lock() # lock pastebuffer for read/write plugins = None # all active plugins highlight = re.compile("") # highlight pattern +rows = 0 +cols = 0 # Interactive regex matches for # - prompt (asd# ) (all) # - question ([yes]) (cisco) @@ -105,7 +107,25 @@ def sigint_handler(sig, data): sys.exit(0) +# noinspection PyUnusedLocal +def sigwinch_passthrough(sig, data): + """ + Update known window size for proper text wrapping + parameters are not used but signal handler passes them! + :param sig: getting this signal from terminal + :param data: getting this data from signal handling + """ + global conn, rows, cols + + rows, cols = getterminalsize() + if conn: + conn.setwinsize(rows, cols) + + +# Set signal handler for CTRL-C signal.signal(signal.SIGINT, sigint_handler) +# Set signal handler for window resizing +signal.signal(signal.SIGWINCH, sigwinch_passthrough) def timeoutcheck(maxwait=0.3): @@ -122,6 +142,7 @@ def timeoutcheck(maxwait=0.3): global conn global effects global pasteguard, pastebuffer, pastelock + global rows, cols timeout = time.time() while RUNNING: @@ -159,11 +180,12 @@ def timeoutcheck(maxwait=0.3): while len(lines) > 0: if effects.intersection({'paste_error', 'paste_abort'}): conn.send("\r") - print("\r", " " * 100, "\r", colorize("Paste " + - ("error" if 'paste_error' in effects else "aborted") + - " at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="") - pastebuffer = list() + print("\r", " " * cols, "\r", colorize("!CLICOL - Paste " + + ("error" if 'paste_error' in effects else "aborted") + + " at line %s: %s" % + (lineno, line.decode('utf-8', errors='ignore'))), sep="", + end='') + pastebuffer = [] effects.discard('paste_error') effects.discard('paste_abort') break @@ -175,19 +197,6 @@ def timeoutcheck(maxwait=0.3): PASTING = False -def sigwinch_passthrough(sig, data): - """ - Update known window size for proper text wrapping - parameters are not used but signal handler passes them! - :param sig: getting this signal from terminal - :param data: getting this data from signal handling - """ - global conn - - rows, cols = getterminalsize() - conn.setwinsize(rows, cols) - - def preventtimeout(): """ Action taken in case we need to prevent device timeout @@ -498,6 +507,7 @@ def main(argv=None): global RUNNING global plugins global highlight + global rows, cols regex = "" plugintestrun = False @@ -741,8 +751,6 @@ def main(argv=None): print("\033]2;%s\007" % caption, end='') break conn = pexpect.spawn(cmd, args, timeout=1) - # Set signal handler for window resizing - signal.signal(signal.SIGWINCH, sigwinch_passthrough) # Set initial terminal size rows, cols = getterminalsize() conn.setwinsize(rows, cols) @@ -768,7 +776,7 @@ def main(argv=None): conn.interact(escape_character='\x1c' if PY3 else b'\x1c', output_filter=ofilter, input_filter=ifilter) if is_break: is_break = False - print("\r" + " " * 100 + + print("\r" + " " * cols + "\rCLICOL: q:quit,p:pause,g:pasteguard,T:highlight,F1-12,SF1-8:shortcuts,h-help", end='') command = getcommand() if command == "D": @@ -782,7 +790,7 @@ def main(argv=None): conn.close() break elif command == "T": - highlight = getregex() + highlight = getregex(cols) cmap_highlight = [False, 0, "", "", highlight, dict(ctfile.items('colortable'))['highlight'] + r"\1" + dict(ctfile.items('colortable'))['default'], 0, 0, 'user_highlight'] @@ -801,7 +809,7 @@ def main(argv=None): elif command in plugins.keybinds.keys(): plugins.keybinds[command].plugin_command(command) - print("\r" + " " * 100 + "\r" + colorize(lastline, {"prompt"}), end='') # restore last line/prompt + print("\r" + " " * cols + "\r" + colorize(lastline, {"prompt"}), end='') # restore last line/prompt if command is not None: for (key, value) in shortcuts: diff --git a/clicol/command.py b/clicol/command.py index 7b11d90..d4d66d4 100755 --- a/clicol/command.py +++ b/clicol/command.py @@ -131,8 +131,8 @@ def getcommand(): return cmd -def getregex(): - regexstr = input("\r" + " " * 100 + "\rHighlight regex: ") +def getregex(cols=80): + regexstr = input("\r" + " " * cols + "\rHighlight regex: ") if not PY3: regexstr = regexstr.decode('utf-8') try: @@ -160,8 +160,7 @@ def getterminalsize(): def ioctl_GWINSZ(fd): try: import fcntl, termios, struct, os - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, - '1234')) + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) except: return return cr From d355f86155ee42c347c2d43b08b3095e150f2f70 Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 4 May 2020 22:59:39 +0200 Subject: [PATCH 22/25] IDE warning suppression --- clicol/clicol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 3cca82b..60b061a 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -93,6 +93,7 @@ flags=re.S) +# noinspection PyUnusedLocal def sigint_handler(sig, data): """ Handle SIGINT (CTRL-C) inside clicol. If spawned connection is alive, do not exit. @@ -723,7 +724,7 @@ def main(argv=None): print("Error opening " + argv[1]) raise for line in f: - # convert to CRLF to support files created in linux + # noinspection PyTypeChecker print(ofilter(line.encode('utf-8', 'ignore')).decode() if PY3 else ofilter(line), end='') f.close() From 834d06340ada23890d009417b88dbf1c00eafd2e Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 4 May 2020 23:15:33 +0200 Subject: [PATCH 23/25] add clicol prompt for paste interaction --- clicol/clicol.py | 5 ++--- clicol/ini/cm_common.ini | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/clicol/clicol.py b/clicol/clicol.py index 60b061a..5df4c44 100755 --- a/clicol/clicol.py +++ b/clicol/clicol.py @@ -181,11 +181,10 @@ def timeoutcheck(maxwait=0.3): while len(lines) > 0: if effects.intersection({'paste_error', 'paste_abort'}): conn.send("\r") - print("\r", " " * cols, "\r", colorize("!CLICOL - Paste " + + print("\r", " " * cols, "\r", colorize("#CLICOL - Paste " + ("error" if 'paste_error' in effects else "aborted") + " at line %s: %s" % - (lineno, line.decode('utf-8', errors='ignore'))), sep="", - end='') + (lineno, line.decode('utf-8', errors='ignore'))), sep="") pastebuffer = [] effects.discard('paste_error') effects.discard('paste_abort') diff --git a/clicol/ini/cm_common.ini b/clicol/ini/cm_common.ini index 20788ca..1b2a3e9 100644 --- a/clicol/ini/cm_common.ini +++ b/clicol/ini/cm_common.ini @@ -156,8 +156,8 @@ example= [common_paste_error] priority=100 -regex=%(BOL)s(Paste error) -replacement=\1%(alert)s\2%(default)s +regex=%(BOL)s(#CLICOL - )(Paste error) +replacement=\1%(pager)s\2%(alert)s\3%(default)s options=%(CLEAR)s dependency=paste_error example= @@ -165,8 +165,8 @@ example= [common_paste_abort] priority=100 -regex=%(BOL)s(Paste aborted) -replacement=\1%(alert)s\2%(default)s +regex=%(BOL)s(#CLICOL - )(Paste aborted) +replacement=\1%(pager)s\2%(alert)s\3%(default)s options=%(CLEAR)s dependency=paste_abort example= From 018fc17e02d56044729bb4952fb8fd8cc320ebea Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 4 May 2020 23:25:55 +0200 Subject: [PATCH 24/25] version bump --- clicol/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clicol/__init__.py b/clicol/__init__.py index 4ade127..d21a3b5 100755 --- a/clicol/__init__.py +++ b/clicol/__init__.py @@ -1 +1 @@ -__version__ = "1.1.4.post1" +__version__ = "1.1.5rc0" From 251d85ed2a4bac79882eb0142413797e79c88efa Mon Sep 17 00:00:00 2001 From: Viktor Kertesz Date: Mon, 18 May 2020 20:03:43 +0200 Subject: [PATCH 25/25] version bump --- clicol/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clicol/__init__.py b/clicol/__init__.py index d21a3b5..9b102be 100755 --- a/clicol/__init__.py +++ b/clicol/__init__.py @@ -1 +1 @@ -__version__ = "1.1.5rc0" +__version__ = "1.1.5"