From bc4193a25a4cbaedc0fcf302f53b0ba34b37fd8d Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Fri, 7 Jul 2023 19:26:03 -0500 Subject: [PATCH 01/30] Implement logic to allow connecting via SSH by name #119 User would just `ssh -t -p 2202 ` The `-t` is key or picocom launches without a tty and tab completion and some other things break. `` is the name of the adapter. Which should be an alias like `r1-6300M-sw` but could also be `ttyUSB0` currently extract the baud and connection settings from ser2net config. --- src/remote_launcher.py | 80 +++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/src/remote_launcher.py b/src/remote_launcher.py index 3e47e4d0..45996c07 100755 --- a/src/remote_launcher.py +++ b/src/remote_launcher.py @@ -9,7 +9,7 @@ import psutil import sys import subprocess - from time import sleep + import time sys.path.insert(0, '/etc/ConsolePi/src/pypkg') from consolepi import config, utils # type: ignore # NoQA from consolepi.consolepi import ConsolePi # type: ignore # NoQA @@ -40,33 +40,57 @@ def terminate_process(pid): x += 1 +def check_hung_process(cmd: str, device: str) -> int: + ppid = find_procs_by_name(cmd, device) + retry = 0 + msg = f"\n{_device.replace('/dev/', '')} appears to be in use (may be a previous hung session)." + msg += '\nDo you want to Terminate the existing session' + if ppid is not None and utils.user_input_bool(msg): + while ppid is not None and retry < 3: + print('Terminating Existing Session...') + try: + terminate_process(ppid) + time.sleep(3) + ppid = find_procs_by_name(cmd, device) + except PermissionError: + print('This appears to be a locally established session, if you want to kill that do it yourself') + break + except psutil.AccessDenied: + print('This appears to be a locally established session, session will not be terminated') + break + except psutil.NoSuchProcess: + ppid = find_procs_by_name(cmd, device) + retry += 1 + + return ppid + + if __name__ == '__main__': - if len(sys.argv) >= 3: + # Allow user to ssh to configured port which using ForceCommand and specifying only the device they want to connect to + if len(sys.argv) == 2 and "picocom" not in sys.argv[1]: + _device = f'/dev/{sys.argv[1].replace("/dev/", "")}' + adapter_data = cpi.local.adapters.get(_device) + if not adapter_data: + print(f'{_device.replace("/dev/", "")} Not found on system... Refreshing local adapters.') + cpi.local.build_adapter_dict(refresh=True) + adapter_data = cpi.local.adapters.get(_device) + + if adapter_data: + print(f'Establishing Connection to {_device.replace("/dev/", "")}...') + _cmd = adapter_data["config"]["cmd"].replace("{{timestamp}}", time.strftime("%F_%H.%M")).split() + else: + print(f'{_device.replace("/dev/", "")} Not found on system... Exiting.') + sys.exit(1) + # Sent from menu on remote full command is sent + elif len(sys.argv) >= 3: _cmd = sys.argv[1:] - ppid = find_procs_by_name(sys.argv[1], sys.argv[2]) - retry = 0 - msg = f"\n{sys.argv[2].replace('/dev/', '')} appears to be in use (may be a previous hung session)." - msg += '\nDo you want to Terminate the existing session' - if ppid is not None and utils.user_input_bool(msg): - while ppid is not None and retry < 3: - print('Terminating Existing Session...') - try: - terminate_process(ppid) - sleep(3) - ppid = find_procs_by_name(sys.argv[1], sys.argv[2]) - except PermissionError: - print('This appears to be a locally established session, if you want to kill that do it yourself') - break - except psutil.AccessDenied: - print('This appears to be a locally established session, session will not be terminated') - break - except psutil.NoSuchProcess: - ppid = find_procs_by_name(sys.argv[1], sys.argv[2]) - retry += 1 + _device = f'/dev/{sys.argv[2].replace("/dev/", "")}' + + ppid = check_hung_process(_cmd[0], _device) - if ppid is None: - # if power feature enabled and adapter linked - ensure outlet is on - # TODO try/except block here - if config.power: - cpi.cpiexec.exec_auto_pwron(sys.argv[2]) - subprocess.run(_cmd) + if ppid is None: + # if power feature enabled and adapter linked - ensure outlet is on + # TODO try/except block here + if config.power: + cpi.cpiexec.exec_auto_pwron(_device) + subprocess.run(_cmd) From d3c5495d984d033054541f2e1038241a02a6131d Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 03:11:46 -0500 Subject: [PATCH 02/30] update requirements --- installer/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer/requirements.txt b/installer/requirements.txt index 50e98c81..897e9527 100755 --- a/installer/requirements.txt +++ b/installer/requirements.txt @@ -52,3 +52,6 @@ Werkzeug>=0.15.4 # wrapt>=1.11.2 zeroconf>=0.23.0 rich +setproctitle +aiohttp +asyncio From 82ad02a95e28d024d0fa6e4892ab9da3fb5fd3ce Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 03:12:44 -0500 Subject: [PATCH 03/30] improve logging, setproctitle --- src/mdns_browser.py | 13 ++++++++++--- src/mdns_register.py | 23 ++++++++++++++--------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/mdns_browser.py b/src/mdns_browser.py index aec785ff..92a344eb 100755 --- a/src/mdns_browser.py +++ b/src/mdns_browser.py @@ -7,6 +7,7 @@ import time import sys from zeroconf import ServiceBrowser, ServiceStateChange, Zeroconf +import setproctitle from rich.traceback import install install(show_locals=True) @@ -18,12 +19,14 @@ RESTART_INTERVAL = 300 # time in seconds browser service will restart +setproctitle.setproctitle("consolepi-mdnsbrowser") + class MDNS_Browser: def __init__(self, show=False): config.cloud = False # mdns doesn't need to sync w cloud - self.cpi = ConsolePi(bypass_outlets=True) + self.cpi = ConsolePi(bypass_outlets=True, bypass_cloud=True) self.debug = config.cfg.get('debug', False) self.show = show self.stop = False @@ -167,6 +170,7 @@ def run(self): if __name__ == '__main__': + program_start = int(time.time()) if len(sys.argv) > 1: mdns = MDNS_Browser(show=True) RESTART_INTERVAL = 30 # when running in interactive mode reduce restart interval @@ -188,12 +192,15 @@ def run(self): log.warning(f'[MDNS BROWSE] caught {e.__class__.__name__} retrying in 5 sec.\nException:\n{e}') time.sleep(5) continue - start = time.time() # re-init zeroconf browser every RESTART_INTERVAL seconds + start = time.time() while time.time() < start + RESTART_INTERVAL: - time.sleep(0.1) + time.sleep(start + RESTART_INTERVAL - time.time()) if mdns.zc is not None: mdns.zc.close() + + duration = f"{RESTART_INTERVAL / 60}m" if RESTART_INTERVAL > 60 else f"{RESTART_INTERVAL}s" + log.info(f'[MDNS DSCVRY] Discovered {len(mdns.discovered)} remote ConsolePis via mdns in last {duration}') mdns.discovered = [] except KeyboardInterrupt: diff --git a/src/mdns_register.py b/src/mdns_register.py index fc9de332..5ca1d09d 100755 --- a/src/mdns_register.py +++ b/src/mdns_register.py @@ -8,19 +8,21 @@ import threading import struct import sys -try: - import better_exceptions # NoQA pylint: disable=import-error -except ImportError: - pass +import setproctitle + +from rich.traceback import install +install(show_locals=True) sys.path.insert(0, '/etc/ConsolePi/src/pypkg') -from consolepi import log, config # NoQA -from consolepi.consolepi import ConsolePi # NoQA -from consolepi.gdrive import GoogleDrive # NoQA +from consolepi import log, config # type: ignore # NoQA +from consolepi.consolepi import ConsolePi # type: ignore # NoQA +from consolepi.gdrive import GoogleDrive # type: ignore # NoQA UPDATE_DELAY = 30 +setproctitle.setproctitle("consolepi-mdnsreg") + class MDNS_Register: @@ -129,8 +131,11 @@ def try_build_info(self): info = self.build_info(squash='interfaces') except (struct.error, ValueError): log.critical('[MDNS REG] data is still too big for mdns') - log.debug('[MDNS REG] offending interface data \n {}'.format( - json.dumps(local.interfaces, indent=4, sort_keys=True))) + log.debug( + '[MDNS REG] offending interface data \n {}'.format( + json.dumps(local.interfaces, indent=4, sort_keys=True) + ) + ) return info From 67187af3ccd68b304757323f92dfca790bd971a6 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 03:14:05 -0500 Subject: [PATCH 04/30] add bypass_cloud bool for Remotes Used for mdns_browse which never updates cloud anyway --- src/pypkg/consolepi/consolepi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pypkg/consolepi/consolepi.py b/src/pypkg/consolepi/consolepi.py index d513a30d..417ba604 100755 --- a/src/pypkg/consolepi/consolepi.py +++ b/src/pypkg/consolepi/consolepi.py @@ -11,7 +11,7 @@ class ConsolePi(): - def __init__(self, bypass_remotes: bool = False, bypass_outlets: bool = False): + def __init__(self, bypass_remotes: bool = False, bypass_outlets: bool = False, bypass_cloud: bool = False): self.menu = Menu("main_menu") self.local = Local() if not bypass_outlets and config.cfg.get('power'): @@ -22,7 +22,7 @@ def __init__(self, bypass_remotes: bool = False, bypass_outlets: bool = False): self.pwr = None self.cpiexec = ConsolePiExec(config, self.pwr, self.local, self.menu) if not bypass_remotes: - self.remotes = Remotes(self.local, self.cpiexec) + self.remotes = Remotes(self.local, self.cpiexec, bypass_cloud=bypass_cloud) # TODO Move to menu launch and prompt user # verify TELNET is installed and install if not if hosts of type TELNET are defined. From 1566dca07cea1a86d508e4589873786b444d201b Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 03:14:46 -0500 Subject: [PATCH 05/30] Major change: Covert from threading to asyncio --- src/pypkg/consolepi/remotes.py | 191 +++++++++++++++------------------ 1 file changed, 86 insertions(+), 105 deletions(-) diff --git a/src/pypkg/consolepi/remotes.py b/src/pypkg/consolepi/remotes.py index 8bee546e..c4190c28 100644 --- a/src/pypkg/consolepi/remotes.py +++ b/src/pypkg/consolepi/remotes.py @@ -1,13 +1,15 @@ #!/etc/ConsolePi/venv/bin/python3 import time -import threading import socket from typing import Any, Dict, List, Union from halo import Halo from sys import stdin from log_symbols import LogSymbols as log_sym # Enum -from consolepi import utils, log, config, json, requests # type: ignore +from consolepi import utils, log, config, json # type: ignore +from aiohttp import ClientSession +import asyncio +from aiohttp.client_exceptions import ContentTypeError # from pydantic import BaseModel # from consolepi.gdrive import GoogleDrive !!--> Import burried in refresh method to speed menu load times on older platforms @@ -17,9 +19,12 @@ class Remotes: - """Remotes Object Contains attributes for discovered remote ConsolePis""" + """Remotes Object Contains attributes for discovered remote ConsolePis - def __init__(self, local, cpiexec): + bypass_cloud overrides the config value for cloud (gdrive sync) + Used by mdns services which don't need cloud updates. + """ + def __init__(self, local, cpiexec, bypass_cloud: bool = False): self.cpiexec = cpiexec self.pop_list = [] self.old_api_log_sent = False @@ -29,8 +34,9 @@ def __init__(self, local, cpiexec): self.connected: bool = False self.cache_update_pending = False self.spin = Halo(spinner="dots") + self.running_spinners = [] self.cloud = None # Set in refresh method if reachable - self.do_cloud = config.cfg.get("cloud", False) + self.do_cloud = False if bypass_cloud is True else config.cfg.get("cloud", False) CLOUD_CREDS_FILE = config.static.get("CLOUD_CREDS_FILE") if not CLOUD_CREDS_FILE: self.no_creds_error() @@ -45,9 +51,11 @@ def __init__(self, local, cpiexec): show=True, ) self.local_only = True - self.data = self.get_remote( - data=config.remote_update() - ) # re-get cloud.json to capture any updates via mdns + self.data = asyncio.run( + self.get_remote( + data=config.remote_update() + ) # re-get cloud.json to capture any updates via mdns + ) def no_creds_error(self): cloud_svc = config.cfg.get("cloud_svc", "UNDEFINED!") @@ -59,18 +67,41 @@ def no_creds_error(self): self.do_cloud = config.cfg["do_cloud"] = False # get remote consoles from local cache refresh function will check/update cloud file and update local cache - def get_remote(self, data: dict = None, rename: bool = False) -> Dict[str, Any]: + async def get_remote(self, data: dict = None, rename: bool = False) -> Dict[str, Any]: spin = self.spin + _get_remote_start = time.perf_counter() - def verify_remote_thread(remotepi: str, data: dict, rename: bool) -> None: + async def verify_remote(remotepi: str, data: dict, rename: bool) -> None: """sub to verify reachability and api data for remotes params: remotepi: The hostname currently being processed data: dict remote ConsolePi dict with hostname as key + rename: bool set True to perform rename request (TODO not sure this is used.) """ this = data[remotepi] - res = self.api_reachable(remotepi, this, rename=rename) + if stdin.isatty(): + self.spin.stop() + self.spin.start(f"verifying {remotepi}") + self.running_spinners += [remotepi] + res = await self.api_reachable(remotepi, this, rename=rename) + _ = self.running_spinners.pop(self.running_spinners.index(remotepi)) + if stdin.isatty(): # restore spin text to any spinners that are still runnning + self.spin.stop() if res.reachable else self.spin.fail() + if self.running_spinners: + self.spin.start(f"verifying {self.running_spinners[-1]}") + else: + if config.remotes: + self.spin.succeed( + "[GET REM] Querying Remotes via API to verify reachability and adapter data\n\t" + f"Found {len(config.remotes)} Remote ConsolePis in {time.perf_counter() - _get_remote_start:.2f}" + ) + else: + self.spin.warn( + "[GET REM] Querying Remotes via API to verify reachability and adapter data\n\t" + "No Reachable Remote ConsolePis Discovered" + ) + this = res.data if res.update: self.cache_update_pending = True @@ -97,7 +128,6 @@ def verify_remote_thread(remotepi: str, data: dict, rename: bool) -> None: data = config.remotes # remotes from local cloud cache if not data: - # print(self.log_sym_warn + " No Remotes in Local Cache") log.info("No Remotes found in Local Cache") data = {} # convert None type to empy dict else: @@ -113,38 +143,14 @@ def verify_remote_thread(remotepi: str, data: dict, rename: bool) -> None: spin.start( "Querying Remotes via API to verify reachability and adapter data" ) - for remotepi in data: - # -- // Launch Threads to verify all remotes in parallel \\ -- - threading.Thread( - target=verify_remote_thread, - args=(remotepi, data, rename), - name=f"vrfy_{remotepi}", - ).start() - # verify_remote_thread(remotepi, data) # Non-Threading DEBUG - - # -- wait for threads to complete -- - if not self.cpiexec.wait_for_threads(name="vrfy_", thread_type="remote"): - if config.remotes: - if stdin.isatty(): - spin.succeed( - "[GET REM] Querying Remotes via API to verify reachability and adapter data\n\t" - f"Found {len(config.remotes)} Remote ConsolePis" - ) - else: - if stdin.isatty(): - spin.warn( - "[GET REM] Querying Remotes via API to verify reachability and adapter data\n\t" - "No Reachable Remote ConsolePis Discovered" - ) - else: - log.error( - "[GET REM] Remote verify threads Still running / exceeded timeout" - ) - if stdin.isatty(): - spin.stop() + + _ = await asyncio.gather(*[verify_remote(remotepi, data, rename) for remotepi in data]) # update local cache if any ConsolePis found UnReachable if self.cache_update_pending: + _start = time.perf_counter() + if stdin.isatty(): + spin.start("Updating local cloud cache...") if self.pop_list: for remotepi in self.pop_list: if ( @@ -167,6 +173,8 @@ def verify_remote_thread(remotepi: str, data: dict, rename: bool) -> None: data = self.update_local_cloud_file(data) self.pop_list = [] self.cache_update_pending = False + if stdin.isatty(): + spin.succeed(f"Updating local cloud cache... Completed in {time.perf_counter() - _start:.2f}s") # TODO this is working, prob change adapters to a list of models # remotes = [Remote(**{"name": k, **data[k]}) for k in data] @@ -257,7 +265,7 @@ def refresh(self, bypass_cloud: bool = False): ) # Update Remote data with data from local_cloud cache / cloud - self.data = self.get_remote(data=remote_consoles) + self.data = asyncio.run(self.get_remote(data=remote_consoles)) def update_local_cloud_file( self, remote_consoles=None, current_remotes=None, local_cloud_file=None @@ -422,46 +430,7 @@ def update_local_cloud_file( return remote_consoles - # Currently not Used - def do_api_request(self, ip: str, path: str, *args, **kwargs): - """Send RestFul GET request to Remote ConsolePi to collect data - - params: - ip(str): ip address or FQDN of remote ConsolePi - path(str): path beyond /api/v1.0/ - - returns: - response object - """ - url = f"http://{ip}:5000/api/v1.0/{path}" - log.debug(f'[do_api_request] URL: {url}') - - headers = { - "Accept": "*/*", - "Cache-Control": "no-cache", - "Host": f"{ip}:5000", - "accept-encoding": "gzip, deflate", - "Connection": "keep-alive", - "cache-control": "no-cache", - } - - try: - response = requests.request( - "GET", url, headers=headers, timeout=config.remote_timeout - ) - except (OSError, TimeoutError): - log.warning(f"[API RQST OUT] Remote ConsolePi @ {ip} TimeOut when querying via API - Unreachable.") - return False - - if response.ok: - log.info(f"[API RQST OUT] {url} Response: OK") - log.debugv(f"[API RQST OUT] Response: \n{json.dumps(response.json(), indent=4, sort_keys=True)}") - else: - log.error(f"[API RQST OUT] API Request Failed {url}") - - return response - - def get_adapters_via_api(self, ip: str, port: int = 5000, rename: bool = False, log_host: str = None): + async def get_adapters_via_api(self, ip: str, port: int = 5000, rename: bool = False, log_host: str = None): """Send RestFul GET request to Remote ConsolePi to collect adapter info params: @@ -490,30 +459,40 @@ def get_adapters_via_api(self, ip: str, port: int = 5000, rename: bool = False, "cache-control": "no-cache", } + ret = None try: - response = requests.request("GET", url, headers=headers, timeout=config.remote_timeout) - except (OSError, TimeoutError): - log.warning(f"[API RQST OUT] Remote ConsolePi: {log_host} TimeOut when querying via API - Unreachable.") - return False - - if response.ok: - ret = response.json() - ret = ret["adapters"] if ret["adapters"] else response.status_code - _msg = f"Adapters Successfully retrieved via API for Remote ConsolePi: {log_host}" - log.info("[API RQST OUT] {}".format(_msg)) - log.debugv( - "[API RQST OUT] Response: \n{}".format( - json.dumps(ret, indent=4, sort_keys=True) + _start = time.perf_counter() + async with ClientSession() as client: + resp = await client.request( + method="GET", + url=url, + headers=headers, + timeout=config.remote_timeout, ) - ) - else: - ret = response.status_code - log.error( - f"[API RQST OUT] Failed to retrieve adapters via API for Remote ConsolePi: {log_host}\n{ret}:{response.text}" - ) + _elapsed = time.perf_counter() - _start + if resp.ok: + try: + ret = await resp.json() + ret = ret["adapters"] if ret["adapters"] else resp.status + _msg = f"Adapters Successfully retrieved via API for Remote ConsolePi: {log_host}, elapsed {_elapsed:.2f}s" + log.info("[API RQST OUT] {}".format(_msg)) + log.debugv( + "[API RQST OUT] Response: \n{}".format( + json.dumps(ret, indent=4, sort_keys=True) + ) + ) + except (json.decoder.JSONDecodeError, ContentTypeError): + log.error(f'[API RQST OUT] Puked on payload from {log_host} \n{await resp.text()}') + ret = resp.status + except asyncio.TimeoutError: + log.warning(f"[API RQST OUT] Remote ConsolePi: {log_host} TimeOut when querying via API - Unreachable.") + except Exception as e: + log.show(f'Exception: {e.__class__.__name__}, in remotes.get_adapters_via_api() check logs') + log.exception(e) + return ret - def api_reachable(self, remote_host: str, cache_data: dict, rename: bool = False): + async def api_reachable(self, remote_host: str, cache_data: dict, rename: bool = False): """Check Rechability & Fetch adapter data via API for remote ConsolePi params: @@ -524,8 +503,10 @@ def api_reachable(self, remote_host: str, cache_data: dict, rename: bool = False changed as a result of remote rename operation. returns: - tuple [0]: Bool, indicating if data is different than cache - [1]: dict, Updated ConsolePi dictionary for the remote + APIReachableResponse object with the following attributes + update: bool flag indicating if data was updated (I think) + data: dict remote consolepi dict + rechable: bool if the consolepi is reachable """ class ApiReachableResponse: @@ -554,7 +535,7 @@ def __init__(self, update, data, reachable): rem_ip = _adapters = None for _ip in rem_ip_list: log.debug(f"[API_REACHABLE] verifying {remote_host}") - _adapters = self.get_adapters_via_api(_ip, port=int(cache_data.get("api_port", 5000)), rename=rename, log_host=f"{remote_host}({_ip})") + _adapters = await self.get_adapters_via_api(_ip, port=int(cache_data.get("api_port", 5000)), rename=rename, log_host=f"{remote_host}({_ip})") if _adapters: rem_ip = _ip # Remote is reachable if not isinstance(_adapters, int): # indicates status_code returned (error or no adapters found) From a9a49a885dc787f1f6aab6c34a8afaae6b168793 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 03:15:01 -0500 Subject: [PATCH 06/30] bump ver 2023-6.0 --- .static.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.static.yaml b/.static.yaml index 98962a16..2febc8d9 100644 --- a/.static.yaml +++ b/.static.yaml @@ -2,7 +2,7 @@ # This file should not be modified. Static variables/paths used throughout ConsolePi # Versioning = YYYY.Major.Patch/Minor --- -CONSOLEPI_VER: 2023-5.2 +CONSOLEPI_VER: 2023-6.0 INSTALLER_VER: 67 CFG_FILE_VER: 10 CONFIG_FILE_YAML: /etc/ConsolePi/ConsolePi.yaml From cd691bc5c5f11ea176aaecd3dedff1ba2c4ec248 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 19:00:06 -0500 Subject: [PATCH 07/30] fix referenced to `Remotes` methods, make async Previous push to dev converted the remote verification from threading to async. Some references needed to be adjusted to call a couroutine vs a method. (await / asyncio.run()) --- src/consolepi-menu.py | 12 ++++-------- src/mdns_browser.py | 3 ++- src/pypkg/consolepi/exec.py | 4 +++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/consolepi-menu.py b/src/consolepi-menu.py index f211517e..a07c843f 100755 --- a/src/consolepi-menu.py +++ b/src/consolepi-menu.py @@ -11,6 +11,7 @@ from collections import OrderedDict as od from typing import Union from halo import Halo +import asyncio # from rich.console import Console # --// ConsolePi imports \\-- @@ -891,7 +892,7 @@ def rename_menu(self, direct_launch: bool = False, from_name: str = None): if choice == 'r': local.adapters = local.build_adapter_dict(refresh=True) if not direct_launch: - remotes.data = remotes.get_remote(data=config.remote_update()) + remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update())) loc = local.adapters rem = remotes.data if not direct_launch else [] @@ -956,7 +957,7 @@ def rename_menu(self, direct_launch: bool = False, from_name: str = None): if choice.isdigit() and int(choice) >= rem_item: print('Triggering Refresh due to Remote Name Change') # remotes.refresh(bypass_cloud=True) # NoQA TODO would be more ideal just to query the remote involved in the rename and update the dict - remotes.data = remotes.get_remote(data=config.remote_update(), rename=True) + remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update(), rename=True)) # TODO Temp need more elegant way to handle back to main_menu elif menu_actions.get(choice, {}) is None and choice == "b": @@ -973,12 +974,7 @@ def refresh_local(self): cpi = self.cpi remotes = cpi.remotes cpi.local.adapters = cpi.local.build_adapter_dict(refresh=True) - remotes.data = remotes.get_remote(data=config.remote_update()) - # loc = cpi.local.adapters - # rem = remotes.data - - # def toggle_show_legend(self): - # self.menu.show_legend = not self.menu.show_legend + remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update())) # ------ // MAIN MENU \\ ------ # def main_menu(self): diff --git a/src/mdns_browser.py b/src/mdns_browser.py index 92a344eb..505c2be8 100755 --- a/src/mdns_browser.py +++ b/src/mdns_browser.py @@ -8,6 +8,7 @@ import sys from zeroconf import ServiceBrowser, ServiceStateChange, Zeroconf import setproctitle +import asyncio from rich.traceback import install install(show_locals=True) @@ -108,7 +109,7 @@ def on_service_state_change(self, # TODO check this don't think needed had a hung process on one of my Pis added it to be safe try: # TODO we are setting update time here so always result in a cache update with the restart timer - res = cpi.remotes.api_reachable(hostname, mdns_data[hostname]) + res = asyncio.run(cpi.remotes.api_reachable(hostname, mdns_data[hostname])) update_cache = res.update if not res.data.get('adapters'): self.no_adapters.append(hostname) diff --git a/src/pypkg/consolepi/exec.py b/src/pypkg/consolepi/exec.py index 8dcd4ea1..86df1549 100644 --- a/src/pypkg/consolepi/exec.py +++ b/src/pypkg/consolepi/exec.py @@ -341,6 +341,7 @@ def menu_exec(self, choice, menu_actions, calling_menu="main_menu"): os.system("clear") if not choice.lower or choice.lower in menu_actions and menu_actions[choice.lower] is None: + # They just hit return with no input ( self.menu.rows, self.menu.cols, @@ -630,7 +631,8 @@ def wait_for_boot(): log.show("Operation Aborted by User") elif menu_actions[ch].__name__ in ["power_menu", "dli_menu"]: menu_actions[ch](calling_menu=calling_menu) - else: + else: # the selection is a function / coroutine + # TODO may need to add inspect.iscoroutinefunction(func) if any of the options are async menu_actions[ch]() except KeyError as e: if len(choice.orig) <= 2 or not self.exec_shell_cmd(choice.orig): From 72e17de55974870ccc6dfcd4de7ab1be1ef4b48c Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sat, 8 Jul 2023 19:25:24 -0500 Subject: [PATCH 08/30] comments --- src/pypkg/consolepi/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pypkg/consolepi/config.py b/src/pypkg/consolepi/config.py index 521d737e..1995b196 100644 --- a/src/pypkg/consolepi/config.py +++ b/src/pypkg/consolepi/config.py @@ -122,6 +122,8 @@ def do_overrides(self): self.cycle_time = int(ovrd.get('cycle_time', DEFAULT_CYCLE_TIME)) self.api_port = int(ovrd.get("api_port", DEFAULT_API_PORT)) self.hide_legend = ovrd.get("hide_legend", False) + # Additional override settings not needed by the python files + # ovpn_share: Share VPN connection when wired_dhcp enabled with hotspot connected devices def get_outlets_from_file(self): '''Get outlets defined in config From 5d8d4ee54a60965606e1db0e8b42b5ddd290830c Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Sun, 9 Jul 2023 12:59:12 -0500 Subject: [PATCH 09/30] tweak v4 ser2net config stanza (created when adding alias) --- src/pypkg/consolepi/config.py | 2 +- src/pypkg/consolepi/udevrename.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pypkg/consolepi/config.py b/src/pypkg/consolepi/config.py index 1995b196..8c6938ca 100644 --- a/src/pypkg/consolepi/config.py +++ b/src/pypkg/consolepi/config.py @@ -429,7 +429,7 @@ def get_ser2netv4(self): ######################################################## # --- ser2net (4.x) config lines look like this --- # connection: &con0096 - # accepter: tcp,2000 + # accepter: telnet(rfc2217),tcp,2000 # enable: on # options: # banner: *banner diff --git a/src/pypkg/consolepi/udevrename.py b/src/pypkg/consolepi/udevrename.py index 5cf6a87c..b42523bd 100644 --- a/src/pypkg/consolepi/udevrename.py +++ b/src/pypkg/consolepi/udevrename.py @@ -366,9 +366,10 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int log.error(f'Rename Menu Error while attempting to cp ser2net.conf from src {error}', show=True) return error # error added to display in calling method + # FIXME we use the precense of v3 file if it exists vs the ser2net ver if config.ser2net_ver.startswith("4"): ser2net_line = f"""connection: &{to_name} - accepter: tcp,{next_port} + accepter: telnet(rfc2217),tcp,{next_port} enable: on options: banner: *banner From d5daa5a8f3a79f2ba129c98e019599eea9a26f5e Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 10 Jul 2023 00:43:15 -0500 Subject: [PATCH 10/30] Update spinner fail with device Prior to this change the spin text was unpredictable given it's set by multiple async tasks at once --- src/pypkg/consolepi/remotes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypkg/consolepi/remotes.py b/src/pypkg/consolepi/remotes.py index c4190c28..33a2e672 100644 --- a/src/pypkg/consolepi/remotes.py +++ b/src/pypkg/consolepi/remotes.py @@ -87,7 +87,7 @@ async def verify_remote(remotepi: str, data: dict, rename: bool) -> None: res = await self.api_reachable(remotepi, this, rename=rename) _ = self.running_spinners.pop(self.running_spinners.index(remotepi)) if stdin.isatty(): # restore spin text to any spinners that are still runnning - self.spin.stop() if res.reachable else self.spin.fail() + self.spin.stop() if res.reachable else self.spin.fail(f'verifying {remotepi}') if self.running_spinners: self.spin.start(f"verifying {self.running_spinners[-1]}") else: From 31d9a27c2eb8e1e9c2377b5314a7d3a3f25b5291 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 10 Jul 2023 00:48:23 -0500 Subject: [PATCH 11/30] Update help text --- src/dhcpcd.exit-hook | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dhcpcd.exit-hook b/src/dhcpcd.exit-hook index 317b46ea..c68c6910 100755 --- a/src/dhcpcd.exit-hook +++ b/src/dhcpcd.exit-hook @@ -136,6 +136,7 @@ check_test_mode() { echo -e "Run via consolepi-pbtest [OPTIONS]\n" echo -e "Options:" echo -e " 'static': Test wired fallback to static flow (wired-dhcp, configure nat if wlan has valid connection)" + echo -e " 'eth0|wlan0': Simulate new IP from specified interface" echo -e " --no-cloud: bypass cloud update" echo -e " --no-push: bypass PushBullet notification" echo -e " -L|--logs: Forces output of all logs for this test session after test is ran." From 44e8d6ef480610d3b3a55908a9feb2e76794d880 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 10 Jul 2023 13:49:00 -0500 Subject: [PATCH 12/30] add to list of exceptions that indicate remote cpi unreachable --- src/pypkg/consolepi/remotes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pypkg/consolepi/remotes.py b/src/pypkg/consolepi/remotes.py index 33a2e672..d962b0cd 100644 --- a/src/pypkg/consolepi/remotes.py +++ b/src/pypkg/consolepi/remotes.py @@ -9,7 +9,7 @@ from consolepi import utils, log, config, json # type: ignore from aiohttp import ClientSession import asyncio -from aiohttp.client_exceptions import ContentTypeError +from aiohttp.client_exceptions import ContentTypeError, ClientConnectorError # from pydantic import BaseModel # from consolepi.gdrive import GoogleDrive !!--> Import burried in refresh method to speed menu load times on older platforms @@ -484,7 +484,7 @@ async def get_adapters_via_api(self, ip: str, port: int = 5000, rename: bool = F except (json.decoder.JSONDecodeError, ContentTypeError): log.error(f'[API RQST OUT] Puked on payload from {log_host} \n{await resp.text()}') ret = resp.status - except asyncio.TimeoutError: + except (asyncio.TimeoutError, ClientConnectorError): log.warning(f"[API RQST OUT] Remote ConsolePi: {log_host} TimeOut when querying via API - Unreachable.") except Exception as e: log.show(f'Exception: {e.__class__.__name__}, in remotes.get_adapters_via_api() check logs') From 7456aba9ef1d9bea36cb2e8719c77acf1daef7be Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 10 Jul 2023 14:03:18 -0500 Subject: [PATCH 13/30] minor tweaks to config import logic --- src/dhcpcd.exit-hook | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dhcpcd.exit-hook b/src/dhcpcd.exit-hook index c68c6910..3affe53a 100755 --- a/src/dhcpcd.exit-hook +++ b/src/dhcpcd.exit-hook @@ -66,10 +66,10 @@ fi # >> Get Configuration from config file << -if [[ -f "${config_builder}" ]] && "${config_builder}" > /tmp/ConsolePi.conf && . /tmp/ConsolePi.conf ; then +if [ -f "$config_builder" ] && "${config_builder}" > /tmp/ConsolePi.conf && . /tmp/ConsolePi.conf ; then rm /tmp/ConsolePi.conf # Disable OpenVPN if ovpn config is not found - $ovpn_enable && [[ ! -f "${ovpn_config}" ]] && ovpn_enable=false && logit -L -t "${log_process}-ovpn" "OpenVPN is enabled but ConsolePi.ovpn not found - disabling" "ERROR" + $ovpn_enable && [ ! -f "${ovpn_config}" ] && ovpn_enable=false && logit -L -t "${log_process}-ovpn" "OpenVPN is enabled but ConsolePi.ovpn not found - disabling" "ERROR" else # Log and exit if config not found logit -L -t "${log_process}" "Unable to find or Parse Configuration... disabling hooks" "ERROR" # logit overrides exit 1 default when called from hook file From c0e13bad326fc5af5825cf1124d606dd46d9aef9 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 16:11:57 -0500 Subject: [PATCH 14/30] improve exception handling during gdrive update --- src/pypkg/consolepi/gdrive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pypkg/consolepi/gdrive.py b/src/pypkg/consolepi/gdrive.py index 7dd1cdb7..255f24ca 100755 --- a/src/pypkg/consolepi/gdrive.py +++ b/src/pypkg/consolepi/gdrive.py @@ -39,7 +39,8 @@ def exec_request(self, _request): result = _request.execute() break except Exception as e: - log.error(('[GDRIVE]: Exception while communicating with Gdrive\n {}'.format(e))) + log.show(f'Exception while communicating with Gdrive: {e.__class__.__name__}', show=True) + log.exception(f'[GDRIVE]: Exception while communicating with Gdrive\n{e}') if attempt > 2: log.error(('[GDRIVE]: Giving up after {} attempts'.format(attempt))) break @@ -172,7 +173,7 @@ def update_files(self, data): spreadsheetId=spreadsheet_id, range='A:B') result = self.exec_request(request) log.info('[GDRIVE]: Reading from Cloud Config') - if result.get('values') is not None: + if result and result.get('values') is not None: x = 1 for row in result.get('values'): if k == row[0]: # k is hostname row[0] is column A of current row From 74f4e6ef0adf107c6a9b3c761d03d8a3b18ecb12 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 22:39:32 -0500 Subject: [PATCH 15/30] Support ser2netv4 config Will still default to ser2netv3 file if it exists regardless of ser2net ver --- src/consolepi-commands/consolepi-editudev | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/consolepi-commands/consolepi-editudev b/src/consolepi-commands/consolepi-editudev index 46615d3d..532573ff 100755 --- a/src/consolepi-commands/consolepi-editudev +++ b/src/consolepi-commands/consolepi-editudev @@ -1,5 +1,15 @@ #!/usr/bin/env bash -sudo nano /etc/udev/rules.d/10-ConsolePi.rules /etc/ser2net.conf +[ -f /etc/ser2net.yaml ] && ser2net_file=/etc/ser2net.yaml +[ -f /etc/ser2net.yml ] && ser2net_file=/etc/ser2net.yml +[ -f /etc/ser2net.conf ] && ser2net_file=/etc/ser2net.conf + +if [ -n "$ser2net_file" ]; then + sudo nano /etc/udev/rules.d/10-ConsolePi.rules "$ser2net_file" +else + sudo nano /etc/udev/rules.d/10-ConsolePi.rules + echo '!!! No ser2net file found to edit. Run consolepi-upgrade to init ser2net config.' +fi + echo use consolepi-showaliases to validate udev and ser2net, check for orphans \ No newline at end of file From 60e4c3d288183030a69123e7a400d6c712016def Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 22:40:15 -0500 Subject: [PATCH 16/30] Clean Up connection menus (formatting) --- src/consolepi-menu.py | 30 ++++++++++++++++++------------ src/pypkg/consolepi/menu.py | 15 +++++++++++---- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/consolepi-menu.py b/src/consolepi-menu.py index a07c843f..758f2ca7 100755 --- a/src/consolepi-menu.py +++ b/src/consolepi-menu.py @@ -218,7 +218,7 @@ def power_menu(self, calling_menu: str = 'main_menu'): cpi = self.cpi pwr = cpi.pwr # menu = self.menu - menu = Menu() + menu = Menu("power_menu") states = self.states menu_actions = { # 'b': self.main_menu, @@ -1164,10 +1164,10 @@ def rshell_menu(self): def con_menu(self, rename: bool = False, con_dict: dict = None): # menu = self.cpi.menu menu = Menu("con_menu") - menu.legend_options = { - 'back': ['b', 'Back'], - 'x': ['x', 'Exit'] - } + # menu.legend_options = { + # 'back': ['b', 'Back'], + # 'x': ['x', 'Exit'] + # } menu_actions = { '1': self.baud_menu, '2': self.data_bits_menu, @@ -1190,7 +1190,7 @@ def con_menu(self, rename: bool = False, con_dict: dict = None): self.flow = con_dict['flow'] self.sbits = con_dict['sbits'] while True: - header = ' Connection Settings Menu ' + header = 'Connection Settings Menu' mlines = [] mlines.append('Baud [{}]'.format(self.baud)) mlines.append('Data Bits [{}]'.format(self.data_bits)) @@ -1199,7 +1199,7 @@ def con_menu(self, rename: bool = False, con_dict: dict = None): legend = {'opts': ['back', 'x'], 'overrides': {'back': ['b', 'Back {}'.format(' (Apply Changes to Files)' if rename else '')]} } - menu.print_menu(mlines, header=header, legend=legend, menu_actions=menu_actions) + menu.print_menu(mlines, header=header, legend=legend, menu_actions=menu_actions, hide_legend=False) ch = self.wait_for_input(locs=locals()).lower try: if ch == 'b': @@ -1208,13 +1208,14 @@ def con_menu(self, rename: bool = False, con_dict: dict = None): except KeyError as e: if ch: - log.show('Invalid selection {}, please try again.'.format(e)) + log.show(f'Invalid selection {e}, please try again.') log.clear() # -- // BAUD MENU \\ -- def baud_menu(self): # config = self.cpi.config - menu = self.cpi.menu + # menu = self.cpi.menu + menu = Menu("baud_menu") menu_actions = od([ ('1', 300), ('2', 1200), @@ -1239,6 +1240,7 @@ def baud_menu(self): print(' {0}. {1}'.format(key, _cur_baud if _cur_baud != self.baud else '[{}]'.format(_cur_baud))) legend = menu.format_legend(legend={"opts": 'back'}) + menu.page.legend.hide = False # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu" footer = menu.format_footer() print(legend) print(footer) @@ -1273,7 +1275,8 @@ def baud_menu(self): # -- // DATA BITS MENU \\ -- def data_bits_menu(self): - menu = self.cpi.menu + # menu = self.cpi.menu + menu = Menu("data_bits") valid = False while not valid: if not config.debug: @@ -1283,6 +1286,7 @@ def data_bits_menu(self): print('\n Default 8, Current [{}], Valid range 5-8'.format(self.data_bits)) legend = menu.format_legend(legend={"opts": 'back'}) + menu.page.legend.hide = False # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu" footer = menu.format_footer() print(legend) print(footer) @@ -1306,7 +1310,7 @@ def data_bits_menu(self): # -- // PARITY MENU \\ -- def parity_menu(self): - menu = self.cpi.menu + menu = Menu("parity_menu") def print_menu(): if not config.debug: @@ -1319,6 +1323,7 @@ def print_menu(): print(f" 3. {'[Even]' if self.parity == 'e' else 'Even'}") legend = menu.format_legend(legend={"opts": 'back'}) + menu.page.legend.hide = False # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu" footer = menu.format_footer() print(legend) print(footer) @@ -1346,7 +1351,7 @@ def print_menu(): # -- // FLOW MENU \\ -- def flow_menu(self): - menu = self.cpi.menu + menu = Menu("flow_menu") def print_menu(): if not config.debug: @@ -1359,6 +1364,7 @@ def print_menu(): print(f" 2. {'[Xon/Xoff]' if self.flow == 'x' else 'Xon/Xoff'} (software)") print(f" 3. {'[RTS/CTS]' if self.flow == 'h' else 'RTS/CTS'} (hardware)") legend = menu.format_legend(legend={"opts": 'back'}) + menu.page.legend.hide = False # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu" footer = menu.format_footer() print(legend) print(footer) diff --git a/src/pypkg/consolepi/menu.py b/src/pypkg/consolepi/menu.py index 5bdf3e9f..79b0b555 100644 --- a/src/pypkg/consolepi/menu.py +++ b/src/pypkg/consolepi/menu.py @@ -382,7 +382,8 @@ def print_menu( legend: Union[dict, list] = None, format_subs: bool = False, by_tens: bool = False, - menu_actions: dict = {} + menu_actions: dict = {}, + hide_legend: bool = None, ) -> dict: """Format and print current menu, sized to fit terminal. Pager implemented if required. @@ -415,6 +416,7 @@ def print_menu( Defaults to False. menu_actions (dict, optional): The Actions dict (TODO make object). Determines what function is called when a menu item is selected Defaults to {}. + menu_actions (bool, optional): Override config option for menus where you always want legend printed. Returns: [dict]: Returns the menu_actions dict which may include paging actions if warranted. @@ -463,10 +465,12 @@ def print_menu( elif isinstance(legend, list): self.legend_in = {"opts": legend} - self.legend_options = {**DEF_LEGEND_OPTIONS, **self.legend_options} + self.legend_options = DEF_LEGEND_OPTIONS if not self.legend_options else {**DEF_LEGEND_OPTIONS, **self.legend_options} header = self.format_header(header) subhead = self.format_subhead(subhead) self.format_legend(self.legend_in) + if hide_legend is not None: + self.page.legend.hide = hide_legend self.format_footer() if not subs: @@ -700,6 +704,9 @@ def print_menu( # update slices (first/last items) in the event tty was resized and no page change was selected self.update_slices() + # FIXME this was issue with con menu where legend was widest section + self.cur_page_width = max([self.cur_page_width, self.page.legend.cols, self.page.footer.cols, self.page.header.cols]) + for menu_part in [self.page.legend, self.page.footer, self.page.header]: menu_part.update(width=self.cur_page_width) # menu_part.update(width=self.page.cols) @@ -1278,7 +1285,7 @@ def format_header(self, text: Union[str, List[str]], width: int = MIN_WIDTH) -> name="header", update_method=self.format_header, update_args=(text, ), - update_kwargs={"width": width}, + update_kwargs={"width": max(width_list)}, ) # _header.update_method = self.format_header # _header.update_args = (text) @@ -1389,7 +1396,7 @@ def format_legend(self, legend: dict = {}, left_offset: int = None, **kwargs) -> pre_text, post_text, legend_text = [], [], [] # init # replace any pre-defined options with those passed in as overrides - legend_options = self.legend_options + legend_options = self.legend_options or DEF_LEGEND_OPTIONS if legend.get("overrides") and isinstance(legend["overrides"], dict): legend_options = {**legend_options, **legend["overrides"]} no_match_overrides = [ From 854c842a380dddf70163f5cd6b81d61aa459ff08 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 22:41:14 -0500 Subject: [PATCH 17/30] Change how `line` key (ser2net line) is formatted --- src/pypkg/consolepi/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pypkg/consolepi/config.py b/src/pypkg/consolepi/config.py index 8c6938ca..87cbdb19 100644 --- a/src/pypkg/consolepi/config.py +++ b/src/pypkg/consolepi/config.py @@ -429,19 +429,19 @@ def get_ser2netv4(self): ######################################################## # --- ser2net (4.x) config lines look like this --- # connection: &con0096 - # accepter: telnet(rfc2217),tcp,2000 + # accepter: telnet(rfc2217),tcp,2000 + # connector: serialdev,/dev/ttyS0,9600n81,local # enable: on # options: # banner: *banner # kickolduser: true # telnet-brk-on-sync: true - # connector: serialdev, - # /dev/ttyS0, - # 9600n81,local ######################################################## ser2net_conf = {} raw = self.ser2net_file.read_text() raw_mod = "\n".join([line if not line.startswith("connection") else f"{line.split('&')[-1]}:" for line in raw.splitlines()]) + banner = [line for line in raw.splitlines() if line.startswith("define: &banner")] + banner = None if not banner else banner[-1].replace("define: &banner", "").lstrip() ser_dict = yaml.safe_load(raw_mod) for k, v in ser_dict.items(): @@ -528,7 +528,7 @@ def get_ser2netv4(self): 'logfile': logfile, 'log_ptr': log_ptr, 'cmd': cmd, - 'line': json.dumps(con_dict, indent=4) + 'line': f'{list(con_dict.keys())[0]}\n{"".join(line if "banner:" not in line else line.replace(f"banner: {banner}", "banner: *banner") for idx, line in enumerate(yaml.safe_dump(con_dict, indent=2).splitlines(keepends=True)) if idx > 0)}' } return ser2net_conf From e46569a5c40b91aac5f57af54b224dc3dc5dc91d Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 22:41:48 -0500 Subject: [PATCH 18/30] remove spinner on cache update In testing was always a fraction of a second. --- src/pypkg/consolepi/remotes.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pypkg/consolepi/remotes.py b/src/pypkg/consolepi/remotes.py index d962b0cd..1aab8747 100644 --- a/src/pypkg/consolepi/remotes.py +++ b/src/pypkg/consolepi/remotes.py @@ -148,9 +148,6 @@ async def verify_remote(remotepi: str, data: dict, rename: bool) -> None: # update local cache if any ConsolePis found UnReachable if self.cache_update_pending: - _start = time.perf_counter() - if stdin.isatty(): - spin.start("Updating local cloud cache...") if self.pop_list: for remotepi in self.pop_list: if ( @@ -173,8 +170,6 @@ async def verify_remote(remotepi: str, data: dict, rename: bool) -> None: data = self.update_local_cloud_file(data) self.pop_list = [] self.cache_update_pending = False - if stdin.isatty(): - spin.succeed(f"Updating local cloud cache... Completed in {time.perf_counter() - _start:.2f}s") # TODO this is working, prob change adapters to a list of models # remotes = [Remote(**{"name": k, **data[k]}) for k in data] From 2c31ab2b0bd27a8bb1d749e49d740784b574cd61 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Mon, 17 Jul 2023 22:42:50 -0500 Subject: [PATCH 19/30] =?UTF-8?q?=E2=9C=A8implement=20rename=20support=20f?= =?UTF-8?q?or=20ser2netv4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pypkg/consolepi/udevrename.py | 49 +++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/pypkg/consolepi/udevrename.py b/src/pypkg/consolepi/udevrename.py index b42523bd..13fb1b10 100644 --- a/src/pypkg/consolepi/udevrename.py +++ b/src/pypkg/consolepi/udevrename.py @@ -1,6 +1,7 @@ #!/etc/ConsolePi/venv/bin/python3 import os +import yaml from consolepi import log, config, utils # type: ignore @@ -350,7 +351,10 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int cur_line = config.ser2net_conf.get(f'/dev/{from_name}', {}).get('line') if cur_line and '/dev/ttyUSB' not in cur_line and '/dev/ttyACM' not in cur_line: new_entry = False - next_port = next_port = cur_line.split(':')[0] # Renaming existing + if config.ser2net_file.suffix in [".yaml", ".yml"]: + next_port = int(yaml.safe_load(cur_line.replace(": *", ": REF_"))["connection"].get("accepter").split(",")[-1]) + else: + next_port = cur_line.split(':')[0] # Renaming existing log_ptr = config.ser2net_conf[f'/dev/{from_name}'].get('log_ptr') if not log_ptr: log_ptr = '' @@ -366,23 +370,19 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int log.error(f'Rename Menu Error while attempting to cp ser2net.conf from src {error}', show=True) return error # error added to display in calling method - # FIXME we use the precense of v3 file if it exists vs the ser2net ver - if config.ser2net_ver.startswith("4"): + if config.ser2net_file.suffix in [".yaml", ".yml"]: ser2net_line = f"""connection: &{to_name} accepter: telnet(rfc2217),tcp,{next_port} + connector: serialdev,/dev/{to_name},{baud}{parity}{dbits}{sbits},local{'' if flow == 'n' else ',' + ser2net_flow[flow].lower()} enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev, - /dev/{to_name}, - {baud}{parity}{dbits}{sbits}, - local """ - if flow != "n": - ser2net_line = ser2net_line.rstrip("\n") - ser2net_line = f'{ser2net_line},\n {ser2net_flow[flow].lower()}\n' + # if flow != "n": + # ser2net_line = ser2net_line.rstrip("\n") + # ser2net_line = f'{ser2net_line},\n {ser2net_flow[flow].lower()}\n' else: ser2net_line = ( '{telnet_port}:telnet:0:/dev/{alias}:{baud} {dbits}DATABITS {parity} {sbits}STOPBIT {flow} banner {log_ptr}'.format( @@ -393,14 +393,39 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int sbits=sbits, parity=ser2net_parity[parity], flow=ser2net_flow[flow], - log_ptr=log_ptr) + log_ptr=log_ptr + ) ) - # -- // Append to ser2net.conf \\ -- + # -- // Append to ser2net config \\ -- if new_entry: error = utils.append_to_file(config.ser2net_file, ser2net_line) # -- // Rename Existing Definition in ser2net.conf \\ -- # -- for devices with existing definitions cur_line is the existing line + elif config.ser2net_file.suffix in [".yaml", ".yml"]: + # sudo sed -i 's|^connection: &delme1$|connection: \&test1|g' /etc/ser2net.yaml + # sudo sed -i 's|^ connector: serialdev,/dev/orange1| connector: serialdev,/dev/delme1|g' /etc/ser2net.yaml + if not cur_line: + return f'cur_line has no value. Leaving {config.ser2net_file} unchanged.' + connection_line = cur_line.splitlines()[0] + connector_line = [line for line in cur_line.splitlines() if "connector:" in line and f'/dev/{from_name},' in line.replace("\\/", "/")] + if connector_line: + if len(connector_line) > 1: + error = f'Found {len(connector_line)} lines in {config.ser2net_file} with /dev/{from_name} defined. Expected 1. Aborting' + elif "connection:" not in connection_line: + error = f'Unexpected connection line {connection_line} extracted for /dev/{from_name} from {config.ser2net_file}' + else: + new_connection_line = connection_line.replace(f'&{from_name}', f'\&{to_name}') # NoQA + connector_line = connector_line[0] + new_connector_line = connector_line.replace(f"/dev/{from_name},", f"/dev/{to_name},") + cmds = [ + f"sudo sed -i 's|^{connection_line}$|{new_connection_line}|g' {config.ser2net_file}", + f"sudo sed -i 's|^{connector_line}$|{new_connector_line}|g' {config.ser2net_file}" + ] + for cmd in cmds: + error = utils.do_shell_cmd(cmd, shell=True) + if error: + break else: ser2net_line = ser2net_line.strip().replace('/', r'\/') cur_line = cur_line.replace('/', r'\/') From 597c2090777846fd30b0b4a27933683795ab1c5f Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 00:06:02 -0500 Subject: [PATCH 20/30] tweak formatting key order No impactful change --- src/ser2net.yaml | 168 +++++++++++++++++++++++------------------------ 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/src/ser2net.yaml b/src/ser2net.yaml index 4509207d..9f8136fe 100644 --- a/src/ser2net.yaml +++ b/src/ser2net.yaml @@ -15,340 +15,340 @@ define: &banner \r\nser2net port \p device \d [\B] (Debian GNU/Linux)\r\n\r\n # unknown devices (known devices will also use these ports first come first serve in order) # -- Typical USB to serial Adapters -- connection: &ttyUSB0 - accepter: tcp,8000 + accepter: telnet(rfc2217),tcp,8000 + connector: serialdev,/dev/ttyUSB0,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB0,9600n81,local connection: &ttyUSB1 - accepter: tcp,8001 + accepter: telnet(rfc2217),tcp,8001 + connector: serialdev,/dev/ttyUSB1,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB1,9600n81,local connection: &ttyUSB2 - accepter: tcp,8002 + accepter: telnet(rfc2217),tcp,8002 + connector: serialdev,/dev/ttyUSB2,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB2,9600n81,local connection: &ttyUSB3 - accepter: tcp,8003 + accepter: telnet(rfc2217),tcp,8003 + connector: serialdev,/dev/ttyUSB3,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB3,9600n81,local connection: &ttyUSB4 - accepter: tcp,8004 + accepter: telnet(rfc2217),tcp,8004 + connector: serialdev,/dev/ttyUSB4,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB4,9600n81,local connection: &ttyUSB5 - accepter: tcp,8005 + accepter: telnet(rfc2217),tcp,8005 + connector: serialdev,/dev/ttyUSB5,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB5,9600n81,local connection: &ttyUSB6 - accepter: tcp,8006 + accepter: telnet(rfc2217),tcp,8006 + connector: serialdev,/dev/ttyUSB6,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB6,9600n81,local connection: &ttyUSB7 - accepter: tcp,8007 + accepter: telnet(rfc2217),tcp,8007 + connector: serialdev,/dev/ttyUSB7,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB7,9600n81,local connection: &ttyUSB8 - accepter: tcp,8008 + accepter: telnet(rfc2217),tcp,8008 + connector: serialdev,/dev/ttyUSB8,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB8,9600n81,local connection: &ttyUSB9 - accepter: tcp,8009 + accepter: telnet(rfc2217),tcp,8009 + connector: serialdev,/dev/ttyUSB9,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB9,9600n81,local connection: &ttyUSB10 - accepter: tcp,8010 + accepter: telnet(rfc2217),tcp,8010 + connector: serialdev,/dev/ttyUSB10,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB10,9600n81,local connection: &ttyUSB11 - accepter: tcp,8011 + accepter: telnet(rfc2217),tcp,8011 + connector: serialdev,/dev/ttyUSB11,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB11,9600n81,local connection: &ttyUSB12 - accepter: tcp,8012 + accepter: telnet(rfc2217),tcp,8012 + connector: serialdev,/dev/ttyUSB12,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB12,9600n81,local connection: &ttyUSB13 - accepter: tcp,8013 + accepter: telnet(rfc2217),tcp,8013 + connector: serialdev,/dev/ttyUSB13,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB13,9600n81,local connection: &ttyUSB14 - accepter: tcp,8014 + accepter: telnet(rfc2217),tcp,8014 + connector: serialdev,/dev/ttyUSB14,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB14,9600n81,local connection: &ttyUSB15 - accepter: tcp,8015 + accepter: telnet(rfc2217),tcp,8015 + connector: serialdev,/dev/ttyUSB15,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB15,9600n81,local connection: &ttyUSB16 - accepter: tcp,8016 + accepter: telnet(rfc2217),tcp,8016 + connector: serialdev,/dev/ttyUSB16,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB16,9600n81,local connection: &ttyUSB17 - accepter: tcp,8017 + accepter: telnet(rfc2217),tcp,8017 + connector: serialdev,/dev/ttyUSB17,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB17,9600n81,local connection: &ttyUSB18 - accepter: tcp,8018 + accepter: telnet(rfc2217),tcp,8018 + connector: serialdev,/dev/ttyUSB18,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB18,9600n81,local connection: &ttyUSB19 - accepter: tcp,8019 + accepter: telnet(rfc2217),tcp,8019 + connector: serialdev,/dev/ttyUSB19,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB19,9600n81,local connection: &ttyUSB20 - accepter: tcp,8020 + accepter: telnet(rfc2217),tcp,8020 + connector: serialdev,/dev/ttyUSB20,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB20,9600n81,local # -- Some Networking Devices with built-in USB Console Ports show up as ttyACM -- connection: &ttyACM0 - accepter: tcp,9000 + accepter: telnet(rfc2217),tcp,9000 + connector: serialdev,/dev/ttyACM0,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM0,9600n81,local connection: &ttyACM1 - accepter: tcp,9001 + accepter: telnet(rfc2217),tcp,9001 + connector: serialdev,/dev/ttyACM1,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM1,9600n81,local connection: &ttyACM2 - accepter: tcp,9002 + accepter: telnet(rfc2217),tcp,9002 + connector: serialdev,/dev/ttyACM2,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM2,9600n81,local connection: &ttyACM3 - accepter: tcp,9003 + accepter: telnet(rfc2217),tcp,9003 + connector: serialdev,/dev/ttyACM3,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM3,9600n81,local connection: &ttyACM4 - accepter: tcp,9004 + accepter: telnet(rfc2217),tcp,9004 + connector: serialdev,/dev/ttyACM4,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM4,9600n81,local connection: &ttyACM5 - accepter: tcp,9005 + accepter: telnet(rfc2217),tcp,9005 + connector: serialdev,/dev/ttyACM5,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM5,9600n81,local connection: &ttyACM6 - accepter: tcp,9006 + accepter: telnet(rfc2217),tcp,9006 + connector: serialdev,/dev/ttyACM6,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM6,9600n81,local connection: &ttyACM7 - accepter: tcp,9007 + accepter: telnet(rfc2217),tcp,9007 + connector: serialdev,/dev/ttyACM7,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM7,9600n81,local connection: &ttyACM8 - accepter: tcp,9008 + accepter: telnet(rfc2217),tcp,9008 + connector: serialdev,/dev/ttyACM8,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM8,9600n81,local connection: &ttyACM9 - accepter: tcp,9009 + accepter: telnet(rfc2217),tcp,9009 + connector: serialdev,/dev/ttyACM9,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM9,9600n81,local connection: &ttyACM10 - accepter: tcp,9010 + accepter: telnet(rfc2217),tcp,9010 + connector: serialdev,/dev/ttyACM10,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM10,9600n81,local connection: &ttyACM11 - accepter: tcp,9011 + accepter: telnet(rfc2217),tcp,9011 + connector: serialdev,/dev/ttyACM11,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM11,9600n81,local connection: &ttyACM12 - accepter: tcp,9012 + accepter: telnet(rfc2217),tcp,9012 + connector: serialdev,/dev/ttyACM12,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM12,9600n81,local connection: &ttyACM13 - accepter: tcp,9013 + accepter: telnet(rfc2217),tcp,9013 + connector: serialdev,/dev/ttyACM13,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM13,9600n81,local connection: &ttyACM14 - accepter: tcp,9014 + accepter: telnet(rfc2217),tcp,9014 + connector: serialdev,/dev/ttyACM14,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM14,9600n81,local connection: &ttyACM15 - accepter: tcp,9015 + accepter: telnet(rfc2217),tcp,9015 + connector: serialdev,/dev/ttyACM15,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM15,9600n81,local connection: &ttyACM16 - accepter: tcp,9016 + accepter: telnet(rfc2217),tcp,9016 + connector: serialdev,/dev/ttyACM16,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM16,9600n81,local connection: &ttyACM17 - accepter: tcp,9017 + accepter: telnet(rfc2217),tcp,9017 + connector: serialdev,/dev/ttyACM17,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM17,9600n81,local connection: &ttyACM18 - accepter: tcp,9018 + accepter: telnet(rfc2217),tcp,9018 + connector: serialdev,/dev/ttyACM18,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM18,9600n81,local connection: &ttyACM19 - accepter: tcp,9019 + accepter: telnet(rfc2217),tcp,9019 + connector: serialdev,/dev/ttyACM19,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM19,9600n81,local connection: &ttyACM20 - accepter: tcp,9020 + accepter: telnet(rfc2217),tcp,9020 + connector: serialdev,/dev/ttyACM20,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM20,9600n81,local # -- predictable ports based on udev rules (use consolepi-addconsole or rename option in consolepi-menu) From 0584530aa6bce90122b7b1811b4152faaf97ebe0 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 00:06:57 -0500 Subject: [PATCH 21/30] change accepter (TELNET) tweak format/key order --- src/ser2net.yaml | 168 +++++++++++++++++++++++------------------------ 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/src/ser2net.yaml b/src/ser2net.yaml index 4509207d..9f8136fe 100644 --- a/src/ser2net.yaml +++ b/src/ser2net.yaml @@ -15,340 +15,340 @@ define: &banner \r\nser2net port \p device \d [\B] (Debian GNU/Linux)\r\n\r\n # unknown devices (known devices will also use these ports first come first serve in order) # -- Typical USB to serial Adapters -- connection: &ttyUSB0 - accepter: tcp,8000 + accepter: telnet(rfc2217),tcp,8000 + connector: serialdev,/dev/ttyUSB0,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB0,9600n81,local connection: &ttyUSB1 - accepter: tcp,8001 + accepter: telnet(rfc2217),tcp,8001 + connector: serialdev,/dev/ttyUSB1,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB1,9600n81,local connection: &ttyUSB2 - accepter: tcp,8002 + accepter: telnet(rfc2217),tcp,8002 + connector: serialdev,/dev/ttyUSB2,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB2,9600n81,local connection: &ttyUSB3 - accepter: tcp,8003 + accepter: telnet(rfc2217),tcp,8003 + connector: serialdev,/dev/ttyUSB3,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB3,9600n81,local connection: &ttyUSB4 - accepter: tcp,8004 + accepter: telnet(rfc2217),tcp,8004 + connector: serialdev,/dev/ttyUSB4,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB4,9600n81,local connection: &ttyUSB5 - accepter: tcp,8005 + accepter: telnet(rfc2217),tcp,8005 + connector: serialdev,/dev/ttyUSB5,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB5,9600n81,local connection: &ttyUSB6 - accepter: tcp,8006 + accepter: telnet(rfc2217),tcp,8006 + connector: serialdev,/dev/ttyUSB6,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB6,9600n81,local connection: &ttyUSB7 - accepter: tcp,8007 + accepter: telnet(rfc2217),tcp,8007 + connector: serialdev,/dev/ttyUSB7,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB7,9600n81,local connection: &ttyUSB8 - accepter: tcp,8008 + accepter: telnet(rfc2217),tcp,8008 + connector: serialdev,/dev/ttyUSB8,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB8,9600n81,local connection: &ttyUSB9 - accepter: tcp,8009 + accepter: telnet(rfc2217),tcp,8009 + connector: serialdev,/dev/ttyUSB9,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB9,9600n81,local connection: &ttyUSB10 - accepter: tcp,8010 + accepter: telnet(rfc2217),tcp,8010 + connector: serialdev,/dev/ttyUSB10,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB10,9600n81,local connection: &ttyUSB11 - accepter: tcp,8011 + accepter: telnet(rfc2217),tcp,8011 + connector: serialdev,/dev/ttyUSB11,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB11,9600n81,local connection: &ttyUSB12 - accepter: tcp,8012 + accepter: telnet(rfc2217),tcp,8012 + connector: serialdev,/dev/ttyUSB12,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB12,9600n81,local connection: &ttyUSB13 - accepter: tcp,8013 + accepter: telnet(rfc2217),tcp,8013 + connector: serialdev,/dev/ttyUSB13,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB13,9600n81,local connection: &ttyUSB14 - accepter: tcp,8014 + accepter: telnet(rfc2217),tcp,8014 + connector: serialdev,/dev/ttyUSB14,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB14,9600n81,local connection: &ttyUSB15 - accepter: tcp,8015 + accepter: telnet(rfc2217),tcp,8015 + connector: serialdev,/dev/ttyUSB15,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB15,9600n81,local connection: &ttyUSB16 - accepter: tcp,8016 + accepter: telnet(rfc2217),tcp,8016 + connector: serialdev,/dev/ttyUSB16,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB16,9600n81,local connection: &ttyUSB17 - accepter: tcp,8017 + accepter: telnet(rfc2217),tcp,8017 + connector: serialdev,/dev/ttyUSB17,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB17,9600n81,local connection: &ttyUSB18 - accepter: tcp,8018 + accepter: telnet(rfc2217),tcp,8018 + connector: serialdev,/dev/ttyUSB18,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB18,9600n81,local connection: &ttyUSB19 - accepter: tcp,8019 + accepter: telnet(rfc2217),tcp,8019 + connector: serialdev,/dev/ttyUSB19,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB19,9600n81,local connection: &ttyUSB20 - accepter: tcp,8020 + accepter: telnet(rfc2217),tcp,8020 + connector: serialdev,/dev/ttyUSB20,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyUSB20,9600n81,local # -- Some Networking Devices with built-in USB Console Ports show up as ttyACM -- connection: &ttyACM0 - accepter: tcp,9000 + accepter: telnet(rfc2217),tcp,9000 + connector: serialdev,/dev/ttyACM0,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM0,9600n81,local connection: &ttyACM1 - accepter: tcp,9001 + accepter: telnet(rfc2217),tcp,9001 + connector: serialdev,/dev/ttyACM1,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM1,9600n81,local connection: &ttyACM2 - accepter: tcp,9002 + accepter: telnet(rfc2217),tcp,9002 + connector: serialdev,/dev/ttyACM2,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM2,9600n81,local connection: &ttyACM3 - accepter: tcp,9003 + accepter: telnet(rfc2217),tcp,9003 + connector: serialdev,/dev/ttyACM3,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM3,9600n81,local connection: &ttyACM4 - accepter: tcp,9004 + accepter: telnet(rfc2217),tcp,9004 + connector: serialdev,/dev/ttyACM4,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM4,9600n81,local connection: &ttyACM5 - accepter: tcp,9005 + accepter: telnet(rfc2217),tcp,9005 + connector: serialdev,/dev/ttyACM5,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM5,9600n81,local connection: &ttyACM6 - accepter: tcp,9006 + accepter: telnet(rfc2217),tcp,9006 + connector: serialdev,/dev/ttyACM6,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM6,9600n81,local connection: &ttyACM7 - accepter: tcp,9007 + accepter: telnet(rfc2217),tcp,9007 + connector: serialdev,/dev/ttyACM7,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM7,9600n81,local connection: &ttyACM8 - accepter: tcp,9008 + accepter: telnet(rfc2217),tcp,9008 + connector: serialdev,/dev/ttyACM8,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM8,9600n81,local connection: &ttyACM9 - accepter: tcp,9009 + accepter: telnet(rfc2217),tcp,9009 + connector: serialdev,/dev/ttyACM9,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM9,9600n81,local connection: &ttyACM10 - accepter: tcp,9010 + accepter: telnet(rfc2217),tcp,9010 + connector: serialdev,/dev/ttyACM10,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM10,9600n81,local connection: &ttyACM11 - accepter: tcp,9011 + accepter: telnet(rfc2217),tcp,9011 + connector: serialdev,/dev/ttyACM11,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM11,9600n81,local connection: &ttyACM12 - accepter: tcp,9012 + accepter: telnet(rfc2217),tcp,9012 + connector: serialdev,/dev/ttyACM12,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM12,9600n81,local connection: &ttyACM13 - accepter: tcp,9013 + accepter: telnet(rfc2217),tcp,9013 + connector: serialdev,/dev/ttyACM13,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM13,9600n81,local connection: &ttyACM14 - accepter: tcp,9014 + accepter: telnet(rfc2217),tcp,9014 + connector: serialdev,/dev/ttyACM14,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM14,9600n81,local connection: &ttyACM15 - accepter: tcp,9015 + accepter: telnet(rfc2217),tcp,9015 + connector: serialdev,/dev/ttyACM15,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM15,9600n81,local connection: &ttyACM16 - accepter: tcp,9016 + accepter: telnet(rfc2217),tcp,9016 + connector: serialdev,/dev/ttyACM16,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM16,9600n81,local connection: &ttyACM17 - accepter: tcp,9017 + accepter: telnet(rfc2217),tcp,9017 + connector: serialdev,/dev/ttyACM17,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM17,9600n81,local connection: &ttyACM18 - accepter: tcp,9018 + accepter: telnet(rfc2217),tcp,9018 + connector: serialdev,/dev/ttyACM18,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM18,9600n81,local connection: &ttyACM19 - accepter: tcp,9019 + accepter: telnet(rfc2217),tcp,9019 + connector: serialdev,/dev/ttyACM19,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM19,9600n81,local connection: &ttyACM20 - accepter: tcp,9020 + accepter: telnet(rfc2217),tcp,9020 + connector: serialdev,/dev/ttyACM20,9600n81,local enable: on options: banner: *banner kickolduser: true telnet-brk-on-sync: true - connector: serialdev,/dev/ttyACM20,9600n81,local # -- predictable ports based on udev rules (use consolepi-addconsole or rename option in consolepi-menu) From eed5051c84b9a90c8fc7610be21547880230a07c Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 00:27:30 -0500 Subject: [PATCH 22/30] =?UTF-8?q?=E2=9C=A8Add=20connection=20settings=20ch?= =?UTF-8?q?ange=20spt=20for=20ser2netv4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pypkg/consolepi/udevrename.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pypkg/consolepi/udevrename.py b/src/pypkg/consolepi/udevrename.py index 13fb1b10..dce1f032 100644 --- a/src/pypkg/consolepi/udevrename.py +++ b/src/pypkg/consolepi/udevrename.py @@ -418,6 +418,12 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int new_connection_line = connection_line.replace(f'&{from_name}', f'\&{to_name}') # NoQA connector_line = connector_line[0] new_connector_line = connector_line.replace(f"/dev/{from_name},", f"/dev/{to_name},") + expected_complete_connector_line = f" connector: serialdev,/dev/{to_name},{baud}{parity}{dbits}{sbits},local{'' if flow == 'n' else ',' + ser2net_flow[flow].lower()}" + if new_connector_line != expected_complete_connector_line: + if "local" in new_connector_line: + new_connector_line = expected_complete_connector_line + else: + return f'rename requires all serial settings be on the same line i.e. {expected_complete_connector_line}.' cmds = [ f"sudo sed -i 's|^{connection_line}$|{new_connection_line}|g' {config.ser2net_file}", f"sudo sed -i 's|^{connector_line}$|{new_connector_line}|g' {config.ser2net_file}" From fdbe32c3221658fc6b41d2907429f7ef9f8ffe10 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 01:47:41 -0500 Subject: [PATCH 23/30] =?UTF-8?q?=E2=9C=A8v3=20to=20v4=20conversion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pypkg/consolepi/config.py | 43 +++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/pypkg/consolepi/config.py b/src/pypkg/consolepi/config.py index 87cbdb19..81aecb7f 100644 --- a/src/pypkg/consolepi/config.py +++ b/src/pypkg/consolepi/config.py @@ -1,4 +1,5 @@ #!/etc/ConsolePi/venv/bin/python3 +from __future__ import annotations import os from typing import Any, Dict import yaml @@ -288,6 +289,33 @@ def get_hosts(self): def get_ser2net(self): return self.get_ser2netv4() if self.ser2net_file.suffix in [".yaml", ".yml"] else self.get_ser2netv3() + @staticmethod + def get_v4_line(tty_dev: str, *, port: int, baud: int, parity: str, dbits: int, sbits: int, flow: str, logfile: str | Path, log_ptr: str, **kwargs) -> str: + ser2net_flow = { + 'n': '', + 'x': ' XONXOFF', + 'h': ' RTSCTS' + } + ser2netv4_line = f"""connection: &{tty_dev.replace("/dev/", "")} + accepter: telnet(rfc2217),tcp,{port} + connector: serialdev,{tty_dev},{baud}{parity}{dbits}{sbits},local{'' if flow == 'n' else ',' + ser2net_flow[flow].lower()} + enable: on + options: + banner: *banner + kickolduser: true + telnet-brk-on-sync: true +""" + if logfile and log_ptr: + log_type = log_ptr.split("=")[0] + log_alias = "".join(log_ptr.split("=")[1:]) + pointers = { + "tr": "trace-read", + "tw": "trace-write", + "tb": "trace-both", + } + ser2netv4_line = f'{ser2netv4_line} {pointers[log_type]}: *{log_alias}' + return ser2netv4_line + def get_ser2netv3(self): '''Parse ser2net.conf to extract connection info for serial adapters @@ -326,7 +354,7 @@ def get_ser2netv3(self): continue elif 'TRACEFILE:' in line: line = line.split(':') - trace_files[line[1]] = line[2] + trace_files[line[1]] = "".join(line[2:]) continue elif not line[0].isdigit(): continue @@ -403,13 +431,20 @@ def get_ser2netv3(self): 'logfile': logfile, 'log_ptr': log_ptr, 'cmd': cmd, - 'line': _line + 'line': _line, } + ser2net_conf[tty_dev]['v4line'] = self.get_v4_line(tty_dev, **ser2net_conf[tty_dev]) + if trace_files: + _v4_tracefiles = [f'define: &{k} {v}' for k, v in trace_files.items()] + ser2net_conf["_v4_tracefiles"] = "# TRACEFILE DEFINITIONS\n" + "\n".join(_v4_tracefiles) return ser2net_conf def get_ser2netv4(self): - '''Parse ser2net.yaml (ser2net 4.x) to extract connection info for serial adapters + """Parse ser2net.yaml (ser2net 4.x) to extract connection info for serial adapters + + Args: + ser_dict (dict, optional): Used to convert ser2netv3 to v4. Defaults to None. retruns 2 level dict (empty dict if ser2net.yaml not found or empty): { @@ -425,7 +460,7 @@ def get_ser2netv4(self): "line": The config lines from ser2net.yaml } } - ''' + """ ######################################################## # --- ser2net (4.x) config lines look like this --- # connection: &con0096 From 9e0b2d98066af3ba2ff428bebcb1d5576bfa5e6c Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 01:47:59 -0500 Subject: [PATCH 24/30] default tracefiles / mdns (off) --- src/ser2net.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/ser2net.yaml b/src/ser2net.yaml index 9f8136fe..38f95cdd 100644 --- a/src/ser2net.yaml +++ b/src/ser2net.yaml @@ -12,6 +12,20 @@ define: &banner \r\nser2net port \p device \d [\B] (Debian GNU/Linux)\r\n\r\n +# TRACEFILE DEFINITIONS +define: &trace0 /var/log/ser2net/trace0-\p-\d-\s\p-\M-\D-\Y_\H\i\s.\U +define: &trace1 /var/log/ser2net/trace1\p-\M-\D-\Y_\H\i\s.\U +define: &trace2 /var/log/ser2net/trace2\p-\M-\D-\Y_\H\i\s.\U +define: &trace3 /var/log/ser2net/trace3\p-\M-\D-\Y_\H\i\s.\U + +default: + name: mdns + value: false + +default: + name: mdns-sysattrs + value: false + # unknown devices (known devices will also use these ports first come first serve in order) # -- Typical USB to serial Adapters -- connection: &ttyUSB0 From 477fb56720ca737c33a1c250483bb94a04721bee Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 22:18:30 -0500 Subject: [PATCH 25/30] Make v3 config parser generates equiv v4 line. Also parse any `TRACEFILE:` definitions and convert to v4 compatible `define` line. These are stored in the dict key `_v4_tracefiles`. --- src/pypkg/consolepi/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pypkg/consolepi/config.py b/src/pypkg/consolepi/config.py index 81aecb7f..edb5164c 100644 --- a/src/pypkg/consolepi/config.py +++ b/src/pypkg/consolepi/config.py @@ -313,7 +313,7 @@ def get_v4_line(tty_dev: str, *, port: int, baud: int, parity: str, dbits: int, "tw": "trace-write", "tb": "trace-both", } - ser2netv4_line = f'{ser2netv4_line} {pointers[log_type]}: *{log_alias}' + ser2netv4_line = f'{ser2netv4_line} {pointers[log_type]}: *{log_alias}\n' return ser2netv4_line def get_ser2netv3(self): @@ -440,7 +440,7 @@ def get_ser2netv3(self): return ser2net_conf - def get_ser2netv4(self): + def get_ser2netv4(self, file: Path = None) -> Dict: """Parse ser2net.yaml (ser2net 4.x) to extract connection info for serial adapters Args: @@ -473,7 +473,10 @@ def get_ser2netv4(self): # telnet-brk-on-sync: true ######################################################## ser2net_conf = {} - raw = self.ser2net_file.read_text() + if file is None and self.ser2net_file is None: + return () + + raw = self.ser2net_file.read_text() if not file else file.read_text() raw_mod = "\n".join([line if not line.startswith("connection") else f"{line.split('&')[-1]}:" for line in raw.splitlines()]) banner = [line for line in raw.splitlines() if line.startswith("define: &banner")] banner = None if not banner else banner[-1].replace("define: &banner", "").lstrip() From a4aa61aaf4e3333a2dabd1ff4de95ccd6959e6b9 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 22:20:08 -0500 Subject: [PATCH 26/30] Minor update based on change to config_dict generated in config.Config --- src/pypkg/consolepi/udevrename.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pypkg/consolepi/udevrename.py b/src/pypkg/consolepi/udevrename.py index dce1f032..2e9e0002 100644 --- a/src/pypkg/consolepi/udevrename.py +++ b/src/pypkg/consolepi/udevrename.py @@ -361,7 +361,7 @@ def do_ser2net_line(self, from_name: str = None, to_name: str = None, baud: int else: new_entry = True if utils.valid_file(config.ser2net_file): - ports = [a['port'] for a in config.ser2net_conf.values() if 7000 < a.get('port', 0) <= 7999] + ports = [v['port'] for k, v in config.ser2net_conf.items() if not k.startswith("_") and 7000 < v.get('port', 0) <= 7999] next_port = 7001 if not ports else int(max(ports)) + 1 else: next_port = 7001 @@ -452,6 +452,7 @@ def add_to_udev(self, udev_line: str, section_marker: str, label: str = None): udev_line {str} -- The properly formatted udev line being added to the file section_marker {str} -- Match text used to determine where to place the line + Keyword Arguments: label {str} -- The rules file GOTO label used in some scenarios (i.e. multi-port 1 serial) (default: {None}) @@ -459,7 +460,8 @@ def add_to_udev(self, udev_line: str, section_marker: str, label: str = None): Returns: {str|None} -- Returns error string if an error occurs ''' - found = ser_label_exists = get_next = update_file = False # init + found = ser_label_exists = get_next = update_file = alias_found = False # NoQA + # alias_name = udev_line.split("SYMLINK+=")[1].split(",")[0].replace('"', "") # TODO WIP add log when alias is already associated with diff serial goto = line = cmd = '' # init rules_file = self.rules_file # if 'ttyAMA' not in udev_line else self.ttyama_rules_file Testing 1 rules file if utils.valid_file(rules_file): @@ -475,6 +477,9 @@ def add_to_udev(self, udev_line: str, section_marker: str, label: str = None): # No longer including SUBSYSTEM in formatted udev line, redundant given logic @ top of rules file if line.replace('SUBSYSTEM=="tty", ', '').strip() == udev_line.strip(): return # Line is already in file Nothing to do. + # elif f'SYMLINK+="{alias_name}",' in line: + # alias_found = True # NoQA + # log.warning(f'{rules_file.split("/")[-1]} already contains an {alias_name} alias\nExisting alias {udev_line}', show=True) if get_next: goto = line get_next = False From 4aaca533f81ea5e7080ee17383b90539fc6eb810 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 22:22:05 -0500 Subject: [PATCH 27/30] :sparkles: Add `consolepi-convert` command for ser2net v3 to v4 config migration. --- src/consolepi-commands/consolepi-convert | 3 + src/convert-ser2net.py | 150 +++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100755 src/consolepi-commands/consolepi-convert create mode 100644 src/convert-ser2net.py diff --git a/src/consolepi-commands/consolepi-convert b/src/consolepi-commands/consolepi-convert new file mode 100755 index 00000000..667987f1 --- /dev/null +++ b/src/consolepi-commands/consolepi-convert @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +sudo /etc/ConsolePi/venv/bin/python3 /etc/ConsolePi/src/convert-ser2net.py "$@" \ No newline at end of file diff --git a/src/convert-ser2net.py b/src/convert-ser2net.py new file mode 100644 index 00000000..ad1af84f --- /dev/null +++ b/src/convert-ser2net.py @@ -0,0 +1,150 @@ +#!/etc/ConsolePi/venv/bin/python3 + +import sys +from pathlib import Path +import shutil +from rich import print +from rich.prompt import Confirm +from datetime import datetime +from typing import List + +sys.path.insert(0, '/etc/ConsolePi/src/pypkg') +from consolepi import config # type: ignore # NoQA + +DEF_V4_FILE = Path("/etc/ConsolePi/src/ser2net.yaml") +V3_FILE = Path("/etc/ser2net.conf") +BAK_DIR = Path("/etc/ConsolePi/bak") +DEF_PROMPT = f""" +[green]--------------------[/] [cyan]ser2net[/] [bright_red]v3[/] to [green3]v4[/] conversion utility [green]--------------------[/] +This utility will parse the [cyan]ser2net[/] v3 configuration file [cyan italic]{V3_FILE.name}[/] +and create a ConsolePi compatible ser2net v4 config [cyan italic]ser2net.yaml[/]. + +If ser2net v4 is installed the v3 config will be moved to {BAK_DIR}. +""" + + +class Convert: + def __init__(self): + self.v4_file: Path = self.get_v4_config_name() + + def __call__(self): + if "-y" in " ".join(sys.argv[1:]).lower() or self.confirm(): + _ = self.check_v4_config() + v4_lines = self.gather_config_data() + if self.v4_file.exists() and self.v4_file.read_text() == v4_lines: + print(f' [magenta]-[/] Process resulted in no change in {self.v4_file.name} content. Existing v4 config left as is.') + else: + print(f' [magenta]-[/] Writing converted config to {self.v4_file}') + self.v4_file.write_text(v4_lines) + self.backup_v3_config() + print() + else: + print("[bright_red]Aborted.") + + def get_v4_config_name(self) -> Path: + new_file = Path("/etc/ser2net.yaml") + if not new_file.exists() and Path("/etc/ser2net.yml").exists(): + new_file = Path("/etc/ser2net.yml") + + return new_file + + def check_v4_config(self) -> bool: + if self.v4_file.exists(): + if "ConsolePi" not in self.v4_file.read_text().splitlines()[2]: + now = datetime.now() + _bak_file = BAK_DIR / f'{self.v4_file.name}.{now.strftime("%F_%H%M")}' + print(f' [magenta]-[/] Backing up existing non ConsolePi version of [cyan]{self.v4_file}[/] to [cyan]{_bak_file}[/].') + shutil.move(self.v4_file, _bak_file) + do_ser2net = True + print(" [magenta]-[/] Preparing default ConsolePi ser2net v4 base config.") + else: + print(f' [magenta]-[/] Found existing ConsolePi compatible version of [cyan]{self.v4_file}[reset].') + do_ser2net = False + else: + do_ser2net = True + print(" [magenta]-[/] Preparing default ConsolePi ser2net v4 base config.") + + return do_ser2net + + def backup_v3_config(self): + if int("".join(config.ser2net_ver.split(".")[0])) > 3: + if not self.v4_file.exists: + print(f' [dark_orange3][blink]!![/blink] WARNING[/]: ser2net v4 Configuration ({self.v4_file}) not found. Keeping ser2net v3 file ({V3_FILE}) in place') + else: + now = datetime.now() + _bak_file = BAK_DIR / f'{V3_FILE.name}.{now.strftime("%F_%H%M")}' + print(f" [magenta]-[/] ser2net [cyan]v{config.ser2net_ver}[/] installed. Backing up v3 config {V3_FILE} to {_bak_file}") + shutil.move(V3_FILE, _bak_file) + else: + print(f" [magenta]-[/] ser2net [cyan]v{config.ser2net_ver}[/] installed. Keeping v3 config {V3_FILE} in place.") + + def gather_config_data(self) -> str: + v3_aliases = {k: v for k, v in config.ser2net_conf.items() if not k.startswith("_") and 7000 < v.get('port', 0) <= 7999} + if self.v4_file.exists(): + v4_config: List[str] = self.v4_file.read_text().splitlines(keepends=True) + v4_config_dict = config.get_ser2netv4(self.v4_file) + v4_aliases = {k: v for k, v in v4_config_dict.items() if not k.startswith("_") and 7000 < v.get('port', 0) <= 7999} + else: + v4_config: List[str] = DEF_V4_FILE.read_text().splitlines(keepends=True) + v4_aliases = {} + + v4_user_lines = [v["v4line"] for k, v in v3_aliases.items() if k not in v4_aliases.keys()] + + # Gather any trace-file references from v3 config and ensure they are defined in v4 config. + if "trace-" in "".join(v4_user_lines): + trace_lines = [line.strip().split(":")[-1].strip() for line in v4_user_lines if "trace-" in line] + new_defs = [ref.lstrip("*") for ref in trace_lines if ref.startswith("*")] # trace-file defs referenced in old config + if new_defs: + existing_defs, get_next_empty, insert_idx = [], False, 19 + for idx, line in enumerate(v4_config): + if "define: &" in line and "banner" not in line and "/" in line: + existing_defs += [line.split("&")[-1].split()[0]] # define: &trace3 /var/log/ser2net/trace3... + elif get_next_empty: + if not line.strip() or not line.strip().startswith("define:"): + get_next_empty = False + insert_idx = idx + elif "# TRACEFILE DEFINITIONS" in line.strip(): + get_next_empty = True + else: + continue + + missing_defs = [ref for ref in new_defs if ref not in existing_defs] + if missing_defs: + missing_lines = [line for line in config.ser2net_conf.get("_v4_tracefiles", "").splitlines() for d in missing_defs if not line.startswith("#") and f'define: &{d}' in line] + if missing_lines: + print(f" [magenta]-[/] Adding trace-file definition{'s' if len(missing_defs) > 1 else ''} [cyan]{'[/], [cyan]'.join(missing_defs)}[/] referenced in v3 config.") + v4_config.insert(insert_idx, "\n".join(missing_lines) + "\n") + + if v4_user_lines: + new_cnt = len([line for line in v4_user_lines if line.startswith("connection:")]) + print(f" [magenta]-[/] Adding {new_cnt} alias definitions converted from {config.ser2net_file}") + v4_config += v4_user_lines + if len(v3_aliases) != new_cnt: + print(f" [magenta]-[/] {len(v3_aliases) - new_cnt} aliases found in ser2net v3 config were skipped as they are already defined in the existing v4 config.") + elif v3_aliases and v4_aliases: + print(f' [dark_orange3][blink]!![/blink] WARNING[/]: All {len(v3_aliases)} alias definitions found in the ser2net v3 config were ignored as they are already defined in the existing v4 config.') + else: + print(f' [dark_orange3][blink]!![/blink] WARNING[/]: No User defined aliases found in {V3_FILE.name}.') + + return "".join(v4_config) + + @staticmethod + def confirm(prompt: str = None) -> bool: + prompt = prompt or DEF_PROMPT + print(DEF_PROMPT) + choice = Confirm.ask("Proceed?") + print(f"[green]{'':-^77}[/]\n") + + return choice + + +if __name__ == "__main__": + if shutil.os.getuid() != 0: + print("[bright_red]Error:[/] Must be run as root. Launch with [cyan]consolepi-convert[/].") + sys.exit(1) + elif not Path("/etc/ser2net.conf").exists(): + print("[bright_red]Error:[/] No v3 config found ([cyan]/etc/ser2net.conf[/]).") + sys.exit(1) + else: + convert = Convert() + convert() From e995dab46014c680010b245c5a92660117617862 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 23:44:58 -0500 Subject: [PATCH 28/30] Update docs --- README.md | 160 ++++++++++++++++++++++++++++++++------------------- changelog.md | 7 +++ 2 files changed, 108 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 7c7638a3..36c003d1 100644 --- a/README.md +++ b/README.md @@ -69,49 +69,91 @@ wget -q https://raw.githubusercontent.com/Pack3tL0ss/ConsolePi/master/installer/ # Known Issues -Bullseye updates the ser2net available from the package repo from 3.x to 4.x this is a significant change. Functionality wise it's a very good thing, ser2net 4.x brings a lot more flexibility and connectivity options. - -However: ser2net 4.x uses `/etc/ser2net.yaml` as it's config. -`consolepi-menu`: Uses `/etc/ser2net.conf` (used by ser2net 3.x) to extract the serial settings for any defined adapters. Adapters can be defined via `consolepi-addconsole` or via the "Predictable Console ports" section at the end of the install. The `rn` (rename) adapter option within the menu, also updates `/etc/ser2net.conf`. The installer creates a default `ser2net.conf` with the default serial settings configured for adapters that lack an alias (ttyUSB0, ttyACM0, etc). - -So - - If you use `consolepi-menu` and never access the adapters directly via TELNET (which utilizes ser2net), then this won't impact you. - - If you don't use ser2net (access adapters directly via TELNET), - - If you do access adapters directly via TELNET, then you need to populate `/etc/ser2net.yaml` as you like, re-building your 3.x setup in the new 4.x file/format. There are a lot more options available (including accessing them via SSH vs. TELNET). - - If you use both. Let the menu rename/define update the old `/etc/ser2net.conf`, but to also have the adapter available directly, for now, you'll have to manually create an equivalent entry in `ser2net.yaml` +- :bangbang: *Not an issue, but a breaking change in ser2net if you've upgraded.* Bullseye updates the ser2net available from the package repo from 3.x to 4.x this is a significant change. Functionality wise it's a very good thing, ser2net 4.x brings a lot more flexibility and connectivity options, but it means if you already have aliases that were configured (via `consolepi-addconsole` or the rename(`rn`) option in `consolepi-menu`) against the ser2net v3 config `/etc/ser2net.conf`... Those are no longer what ser2net uses for the TELNET connections. + + Good news though. As of v2023-6.0 the various options mentioned above for adding/updating/renaming adapter aliases now work with the v4 config (`/etc/ser2net.yaml`). Not only that, but there is now a migration command `consolepi-convert` which will migrate an existing ser2net v3 config to v4. + + if the v3 file still exists, the menu and rename options will continue to use it (it will warn you on menu launch if the v3 config is still in place with ser2netv4 installed). If you run `consolepi-convert` it will migrate the config, and if ser2net v4 is installed it will stash the old v3 config in the bak dir. + +- ser2net v4 config parsing errors: I have seen on multiple occasions ser2net throw parsing errors after bootup. `systemctl restart ser2net` has resolved it, so it's not an actual file format issue. Investigating why this occasionally happens. + The errors look something like this: + ```shell + ---------------- // STATUS OF ser2net.service \\ --------------- + ● ser2net.service - Serial port to network proxy + Loaded: loaded (/lib/systemd/system/ser2net.service; enabled; vendor preset: enabled) + Active: active (running) since Tue 2023-07-11 14:34:27 CDT; 1 weeks 0 days ago + Docs: man:ser2net(8) + Main PID: 407 (ser2net) + Tasks: 1 (limit: 2057) + CPU: 131ms + CGroup: /system.slice/ser2net.service + └─407 /usr/sbin/ser2net -n -c /etc/ser2net.yaml -P /run/ser2net.pid + + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 361 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 369 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 377 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 385 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 393 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 401 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 409 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 420 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 431 column 0 + Jul 11 14:34:28 ConsolePi3 ser2net[407]: Invalid port name/number: Invalid data to parameter on line 442 column 0 + ``` + Again simply restarting the service seems to resolve the issue. I suspect it may be a race-condition on bootup with the initialization of udev. -ConsolePi will eventually be updated to detect the ser2net version installed, and extract/update to the corresponding config. This won't convert your ser2net.conf to ser2net.yaml, it will just update the ConsolePi functions to use the new config (`ser2net.yaml`) . +- `consolepi-extras`: The options related to installing ansible and the aruba modules need to be updated as some things have changed upstream. They *might* work, but I would refer to ansible documentation, and [Aruba devhub](https://devhub.arubanetworks.com) for manual install instructions for now. # What's New Prior Changes can be found in the - [ChangeLog](changelog.md) +### July 2023 (v2023-6.0) + - :sparkles: Add full support for ser2netv4 add/change/rename via rename(`rn`) option in the menu, and the `consolepi-addconsole`. + - :sparkles: Add `consolepi-convert` command, which will parse an existing ser2netv3 config (`/etc/ser2net.conf`) and create/update a ser2netv4 config (`/etc/ser2net.yaml`) + - :zap: Convert remote ConsolePi updates to async (they were already using threading) + - :zap: Convert remote ConsolePi updates to async (they were already using threading) + - :loud_sound: Update Spinner with the name of the remote as reachability is being check for remote ConsolePis. Make failures persistent (spinner shows what failed and continues one line down.) + - :construction: (Requires manual setup for now see issue [#119](https://github.com/Pack3tL0ss/ConsolePi/issues/119)) Add ability to ssh directly to an adapter specifying adapter by name + - i.e. `ssh -t -p 2202 ` + - real example `ssh -t consolepi4 -p 2202 r1-8360-TOP` will connect to the defined udev alias `/dev/r1-8360-TOP` connected to remote ConsolePi ConsolePi4 (you could use ip vs hostname) + > The examples uses a predictable device name (`r1-8360-TOP`) vs. the default /dev/ttyUSB# Use consolepi-addconsole or the rename(`rn`) option in `consolepi-menu` to discover and apply predictable names to connected serial adapters. + - This feature retains power-control, so if `r1-8360-TOP` has an outlet linked to it, connecting to the device will automatically verify the outlet is on, and turn it on if not. See [Power Control Setup](readme_content/power.md#power-control-setup) for more details. + - This is a work in progress. The sshd config still needs to be automated but can be manually created. Just place the following in a new file /etc/ssh/sshd_config.d/consolepi.conf and restart ssh `systemctl restart ssh` + ```shell + Port 22 + Port 2202 + AddressFamily any + ListenAddress 0.0.0.0 + + Match LocalPort 2202 + ForceCommand /etc/ConsolePi/src/remote_launcher.py $SSH_ORIGINAL_COMMAND + ``` + - In future release additional flags will be passed on to picocom i.e. `ssh -t -p 2202 [any flags picocom supports]` + - :bangbang: The `-t` option is crucial, otherwise there is no tty which causes strange behavior in the terminal (tab completion via the connected device among other things break). Will research if there is a way to attach it on the server side. + ### June 2023 (v2023-5.0) - ser2netv4 Parsing. Rename is not refactored yet, but parsing the baud rate from defined adapters now works with ser2netv3 and ser2netv4. - Rename still functional if still using ser2netv3 - If ser2netv4 is installed but the ser2netv3 config file still exists (`/etc/ser2net.conf`). ConsolePi will continue to use the v3 config for parsing. This is to allow time for manual conversion to the v4 format (`/etc/ser2net.yaml`) - Fix issue introduced in v2022-4.x (which should have been v2023-xx.yy). Issue relates to handling optional requirement for RPi.GPIO module. -### Sep 2022 (v2022-3.0) **Breaking Change for silent installs** - - Changed cmd-line flags for `consolepi-image` and `consolepi-install`/`consolepi-upgrade`. Use `--help` with those commands to see the changes. - - This is a breaking change for silent install. If using an install.conf file refer to the new example as some varirables have changed. - - Re-worked `consolepi-image` script ([consolepi-image-creator.sh](installer/consolepi-image-creator.sh)) to configure consolepi as the default user on the image. - - This is necessary for headless installs, as there is no default pi user anymore. - - Updated installation script... worked-around some dependencies that required rust build environment. - - Various other improvements to both of the above mentioned scripts. + # Planned enhancements - - The ser2net update highlighted in [Known Issues](#known-issues) (partially complete as of v2023-5.0) + - Complete automation of feature to provide ssh direct to adapter discussed in [issue #119](https://github.com/Pack3tL0ss/ConsolePi/issues/119) - Non RPI & wsl (menu accessing all remotes) support for the installer. Can be done now, but normally at least portions need to be tweaked manually. - Ability to pick a non sequential port when using `rn` in menu or `consolepi-addconsole` - *Most excited about* launch menu in byobu session (tmux). With any connections in a new tab. - Eventually... formatting tweaks, and TUI. Also consolepi-commands turn into `consolepi command [options]` with auto complete and help text for all (transition to typer CLI). - - Investigate options to provide ssh direct to adapter discussed in [issue #119](https://github.com/Pack3tL0ss/ConsolePi/issues/119) # Features ## **Feature Summary Image** ![consolepi-menu image](https://raw.githubusercontent.com/Pack3tL0ss/ConsolePi/master/readme_content/ConsolePi_features.jpg) ## Serial Console Server -This is the core feature of ConsolePi. Connect USB to serial adapters to ConsolePi (or use the onboard UART(s)), then access the devices on those adapters via the ConsolePi. Supports TELNET directly to the adapter, or connect to ConsolePi via SSH or BlueTooth and select the adapter from `consolepi-menu`. The menu will show connection options for any locally connected adapters, as well as connections to any remote ConsolePis discovered via Cluster/sync. The menu has a lot of other features beyond connecting to local adapters, as shown in the image above. - +**This is the core feature of ConsolePi.** + - Connect USB to serial adapters to ConsolePi (or use the onboard UART(s)), then access the devices on those adapters via the ConsolePi. + - Supports TELNET directly to the adapter. Use `consolepi-addconsole` or the rename(`rn`) option in `consolepi-menu` to create predictable alias for a USB to serial adapter (vs the default /dev/ttyUSB0...) and assign the TELNET port associated with it. + - Connect to ConsolePi via SSH or BlueTooth and select the adapter from `consolepi-menu`. The menu will show connection options for any locally connected adapters, as well as connections to any remote ConsolePis discovered via Cluster/sync. The menu has a lot of other features beyond connecting to local adapters, as shown in the image above. + - When connecting to the ConsolePi via bluetooth, default behavior is to auto-login and launch a limited function menu. Given this user is automatically logged in, the user has limited rights, hence the limited function menu (allows access to locally attached adapters). - The `consolepi` user can also be configured to auto-launch the menu on login (option during install), this user doesn't auto-login, so it's created with typical rights and launches the full menu. > To disable auto-login via Bluetooth, modify /etc/systemd/system/rfcomm.service and remove `-a blue` from the end of the `ExecStart` line. Then issue the following to create an empty file telling `consolepi-upgrade` not to update the file on upgrade: `touch /etc/ConsolePi/overrides/rfcomm.service`. @@ -250,7 +292,7 @@ The Power Control Function allows you to control power to external outlets. Con ### Outlet Linkages Example Outlet Linkage. In this case the switch "rw-6200T-sw" has 2 ports linked. Both are on a dli web power switch. One of the ports is for this switch, the other is for the up-stream switch that serves this rack. When connecting to the switch, ConsolePi will ensure the linked outlets are powered ON. *ConsolePi does **not** power-off the outlets when you disconnect.* -```bash +```shell ------------------------------------------------------------------------------------ Ensuring r2-6200T-sw Linked Outlets (labpower2:[1, 4]) are Powered ON @@ -273,7 +315,8 @@ ConsolePi supports Zero Touch Provisioning(ZTP) of devices via wired ethernet/DH ## ConsolePi API -ConsolePi includes an API with the following available methods (All Are GET methods via http port 5000 currently). +ConsolePi includes an API with the following available methods +> All Are GET methods the default port is 5000, which can be overridden in the config via the `api_port` key in the optional `OVERRIDES` stanza. See [ConsolePi.yaml.example](ConsolePi.yaml.example). /api/v1.0/ * adapters: returns list of local adapters @@ -285,7 +328,7 @@ The swagger interface is @ `/api/docs` or `/api/redoc`. You can browse/try the The API is used by ConsolePi to verify reachability and ensure adapter data is current on menu-load. -> The API is currently unsecured, it uses http, and Auth is not implemented *yet*. It currently only supports GET requests and doesn't provide any sensitive (credential) data. Authentication on the API is a roadmap item. +> The API is currently unsecured, it uses http, and Auth is not implemented *yet*. It currently only supports GET requests and doesn't provide any sensitive (credential) data. ## ConsolePi Extras Toward the end of the install, and via `consolepi-extras` anytime after the install, you are provided with options to automate the deployment (and removal for most) of some additional tools. This is a selection of tools not required for ConsolePi, but often desired, or useful for the kind of folks that would be using ConsolePi. @@ -299,9 +342,9 @@ Toward the end of the install, and via `consolepi-extras` anytime after the inst If you have a Linux system available you can use [ConsolePi image creator](#3.-consolepi-image-creator) to burn the image to a micro-sd, enable SSH, pre-configure a WLAN (optional), mass-import configurations (if ran from a ConsolePi, optional), and PreConfigure ConsolePi settings (optional). This script is especially useful for doing headless installations. **The Following Applies to All Automated Installation methods** -> Note Previous versions of ConsolePi supported import from either the users home-dir (i.e. `/home/pi`) or from a `consolepi-stage` subdir in the users home-dir (i.e. `/home/pi/ConsolePi-stage`). The import logic directly from the home-dir has not been removed, but going forward any new imports will only be tested using the `consolePi-stage` directory for simplicity. +> Note Previous versions of ConsolePi supported import from either the users home-dir (i.e. `/home/wade`) or from a `consolepi-stage` subdir in the users home-dir (i.e. `/home/wade/ConsolePi-stage`). The import logic directly from the home-dir has not been removed, but going forward any new imports will only be tested using the `consolePi-stage` directory for simplicity. -ConsolePi will **optionally** use pre-configured settings for the following if they are placed in the a `consolepi-stage` subdir in the users home folder (i.e. `/home/pi/consolepi-stage`). This is optional, the installer will prompt for the information if not pre-configured. It will prompt you to verify either way. *Imports only occur during initial install not upgrades.* +ConsolePi will **optionally** use pre-configured settings for the following if they are placed in the a `consolepi-stage` subdir in the users home folder (i.e. `/home/wade/consolepi-stage`). This is optional, the installer will prompt for the information if not pre-configured. It will prompt you to verify either way. *Imports only occur during initial install not upgrades.* - ConsolePi.yaml: This is the main configuration file where all configurable settings are defined. If provided in the `consolepi-stage` dir the installer will ask for verification then create the working config `/etc/ConsolePi/ConsolePi.yaml` @@ -326,7 +369,7 @@ ConsolePi will **optionally** use pre-configured settings for the following if t > > There may be a better way, but this is working on all my Pi3/4s, on my Pi Zero Ws installing these packages breaks wpa_supplicant entirely. For those I currently just use the psk SSID (which I expect most would do, but good tip anyway for the cool kids using certs) -- authorized_keys/known_hosts: If either of these ssh related files are found they will be placed in both the /home/pi/.ssh and /root/.ssh directories (ownership is adjusted appropriately). +- authorized_keys/known_hosts: If either of these ssh related files are found they will be placed in both the /home/wade/.ssh and /root/.ssh directories (ownership is adjusted appropriately). - rpi-poe-overlay.dts: This is a custom overlay file for the official Rpi PoE hat. If the dts is found in the stage dir, a dtbo (overlay binary) is created from it and placed in /boot/overlays. A custom overlay for the PoE hat can be used to adjust what temp triggers the fan, and how fast the fan will run at each temp threshold. > Refer to google for more info, be aware some apt upgrades update the overlays overwriting your customization. I use a separate script I run occasionally which creates a dtbo then compares it to the one in /boot/overlays, and updates if necessary (to revery back to my custom settings) @@ -410,7 +453,7 @@ The Following optional arguments are more for dev, but can be useful in some oth Examples: This example specifies a config file with -C (telling it to get some info from the specified config) as well as the silent install option (no prompts) - > consolepi-upgrade -C /home/pi/consolepi-stage/installer.conf -silent + > consolepi-upgrade -C /home/wade/consolepi-stage/installer.conf -silent Alternatively the necessary arguments can be passed in via cmd line arguments NOTE: Showing minimum required options for a silent install. ConsolePi.yaml has to exist @@ -505,7 +548,7 @@ Examples: This example overrides the default RaspiOS image type (lite) in favor of the desktop image and configures a psk SSID (use single quotes if special characters exist) sudo ./consolepi-image-creator.sh --img_type=desktop --ssid=MySSID --psk='ConsolePi!!!' This example passes the -C option to the installer (telling it to get some info from the specified config) as well as the silent install option (no prompts) - sudo ./consolepi-image-creator.sh --cmd_line='-C /home/pi/consolepi-stage/installer.conf -silent' + sudo ./consolepi-image-creator.sh --cmd_line='-C /home/wade/consolepi-stage/installer.conf -silent' ``` ```bash # ----------------------------------- // DEFAULTS \\ ----------------------------------- @@ -528,28 +571,26 @@ Examples: # -------------------------------------------------------------------------------------- ``` **What the script does** -- automatically pull the most recent RaspiOS image (lite by default) if one is not found in the script-dir (whatever dir you run it from) - - It will check to see if a more current image is available and prompt for image selection even if an image exists in the script dir. -- Make an attempt to determine the correct drive to be flashed, and display details ... User to verify/confirm before writing. - + - automatically pull the most recent RaspiOS image (lite by default) if one is not found in the script-dir (whatever dir you run it from) + - It will check to see if a more current image is available and prompt for image selection even if an image exists in the script dir. + - Make an attempt to determine the correct drive to be flashed, and display details ... User to verify/confirm before writing. > As a fail-safe the script will exit if it finds more than 1 USB storage device on the system. -- Flash image to micro-sd card -- Enable SSH (handy for headless install) - - ***if img_only=true the script stops here*** - -- The entire stage dir (consolepi-stage) is moved to the /home/pi dir on the micro-sd if found in the script dir. This can be used to pre-stage a number of config files the installer will detect and use, along with anything else you'd like on the ConsolePi image. -- Pre-Configure a psk or open WLAN via parameters in script. Useful for headless installation, you just need to determine what IP address ConsolePi gets from DHCP if doing a headless install. -- You can also pre-configure WLAN by placing a wpa_supplicant.conf file in the stage dir. This will be copied to the /etc/wpa_supplicant dir on the micro-sd card. This method supports the typical methods along with EAP-TLS with certificates. Just place the cert files referenced in the provided wpa_supplicant.conf file in a 'cert' folder inside the stage dir. ( Only works for a single EAP-TLS SSID or rather a single set of certs ), the image creator will then move the certs to the micro-sd to the path specified in the provided wap_supplicant.conf. -- create a quick command 'consolepi-install' to simplify the command string to pull the installer from this repo and launch. If cmd_line= argument is provided to consolepi-image-creator.sh those arguments are passed on to the auto-install. -- The ConsolePi installer will start on first login, as long as the RaspberryPi has internet access. This can be disabled with `--auto_install=false`. - - > If you set `--auto_install=false`, `--cmd_line=...` is ignored. You would specify arguments for the installer manually. -- If the `consolepi-image-creator.sh` script is ran from a ConsolePi, the script will detect that it's a ConsolePi and offer to pre-stage it's existing settings. If a file has already been pre-staged (via consolepi-stage dir) it will skip it. It will give you the chance to edit ConsolePi.yaml if pre-staged, so you can deploy multiple ConsolePis and edit the specifics for each as you stage them. -- Entire home directory imports: If you place /root and/or /home/pi inside the consolepi-stage directory. Those contents/sub-dirs will be imported to the respective users directory on the image. - - You can even pre-stage a users home directory for a user that doesn't exist. When the installer runs, you are given the option to create new users. Once created if a folder is found in consolepi-stage for that user (i.e. `home/pi/consolepi-stage/home/larry`), the contents will be copied from the `consolepi-stage` dir to `/home/larry`. - -The install script (not this image-creator, the installer that actually installs ConsolePi) will look for and if found import a number of items from the consolepi-stage directory. Gdrive credentials, ovpn settings, ssh keys refer to *TODO link to section highlighting imports* + - Flash image to micro-sd card + - pre-configure default consolepi user via userconf (RaspiOS no longer boots with a default `pi` user) + - Enable SSH (to facilitate headless install) + > ***if img_only=true the script stops here*** + - The entire stage dir (consolepi-stage) is moved to the /home/consolepi dir on the micro-sd if found in the script dir. This can be used to pre-stage a number of config files the installer will detect and use, along with anything else you'd like on the ConsolePi image. + - Pre-Configure a psk or open WLAN via parameters in script. Useful for headless installation, you just need to determine what IP address ConsolePi gets from DHCP if doing a headless install. + - You can also pre-configure WLAN by placing a wpa_supplicant.conf file in the stage dir. This will be copied to the /etc/wpa_supplicant dir on the micro-sd card. This method supports the typical methods along with EAP-TLS with certificates. Just place the cert files referenced in the provided wpa_supplicant.conf file in a 'cert' folder inside the stage dir. ( Only works for a single EAP-TLS SSID or rather a single set of certs ), the image creator will then move the certs to the micro-sd to the path specified in the provided wap_supplicant.conf. + - create a quick command 'consolepi-install' to simplify the command string to pull the installer from this repo and launch. If cmd_line= argument is provided to consolepi-image-creator.sh those arguments are passed on to the auto-install. + - The ConsolePi installer will start on first login, as long as the RaspberryPi has internet access. This can be disabled with `--auto_install=false`. + + > If you set `--auto_install=false`, `--cmd_line=...` is ignored. You would specify arguments for the installer manually. + - If the `consolepi-image-creator.sh` script is ran from a ConsolePi, the script will detect that it's a ConsolePi and offer to pre-stage it's existing settings. If a file has already been pre-staged (via consolepi-stage dir) it will skip it. It will give you the chance to edit ConsolePi.yaml if pre-staged, so you can deploy multiple ConsolePis and edit the specifics for each as you stage them. + - Entire home directory imports: If you place /root and/or /home/pi inside the consolepi-stage directory. Those contents/sub-dirs will be imported to the respective users directory on the image. + - You can even pre-stage a users home directory for a user that doesn't exist. When the installer runs, you are given the option to create new users. Once created if a folder is found in consolepi-stage for that user (i.e. `home/pi/consolepi-stage/home/larry`), the contents will be copied from the `consolepi-stage` dir to `/home/larry`. + + The install script (not this image-creator, the installer that actually installs ConsolePi) will look for and if found import a number of items from the consolepi-stage directory. Gdrive credentials, ovpn settings, ssh keys refer to *TODO link to section highlighting imports* **This capture highlights what the script does and what it pulls via mass import if ran from an existing ConsolePi** ```bash @@ -589,11 +630,11 @@ Press enter to accept sda as the destination drive or specify the correct device Device to flash with image [sda]: Getting latest raspios image (lite) -Using image 2020-05-27-raspios-buster-lite-armhf found in /home/pi. It is the current release +Using image 2023-05-03-raspios-bullseye-armhf-lite found in /home/wade. It is the current release !!! Last chance to abort !!! -About to write image 2020-05-27-raspios-buster-lite-armhf.img to sda, Continue? (y/n|exit): y +About to write image 2023-05-03-raspios-bullseye-armhf-lite.img to sda, Continue? (y/n|exit): y ______ __ ____ _ / ____/___ ____ _________ / /__ / __ \(_) / / / __ \/ __ \/ ___/ __ \/ / _ \/ /_/ / / @@ -602,7 +643,7 @@ About to write image 2020-05-27-raspios-buster-lite-armhf.img to sda, Continue? https://github.com/Pack3tL0ss/ConsolePi -Now Writing image 2020-05-27-raspios-buster-lite-armhf.img to sda standby... +Now Writing image 2023-05-03-raspios-bullseye-armhf-lite.img to sda standby... This takes a few minutes 1849688064 bytes (1.8 GB, 1.7 GiB) copied, 221 s, 8.4 MB/s @@ -632,8 +673,8 @@ Image written to flash - no Errors ~ /etc/wpa_supplicant/wpa_supplicant.conf...............................Skipped - Already Staged ~ /etc/udev/rules.d/10-ConsolePi.rules..................................Skipped - Already Staged ~ /etc/ser2net.conf.....................................................Skipped - Already Staged - ~ /home/pi/.ssh/authorized_keys.........................................Skipped - Already Staged - ~ /home/pi/.ssh/known_hosts.............................................Skipped - Already Staged + ~ /home/wade/.ssh/authorized_keys.........................................Skipped - Already Staged + ~ /home/wade/.ssh/known_hosts.............................................Skipped - Already Staged ~ /etc/ConsolePi/cloud/gdrive/.credentials/credentials.json.............Imported ~ /etc/ConsolePi/cloud/gdrive/.credentials/token.pickle.................Imported ~ /etc/openvpn/client/ConsolePi.ovpn....................................Skipped - Already Staged @@ -679,9 +720,9 @@ The Use Cases > If you did want the system to advertise itself on the network, so other ConsolePis could discover it: Repeat the commands above related to `consolepi-mdnsbrowse.service` but swap in `consolepi-mdnsreg.service`. 2. ConsolePi running on wsl-ubuntu (Windows Subsystem for Linux) - - Use Case... I just wanted to see if it would work. I also have it open a lot so handy to be able to just run from there. + - Use Case... I always have a wsl terminal open, so I use the `consolepi-menu` in wsl, which displays all the discovered remote consolepis. - No local-adapters wsl would be remote only. - - Install process: Same as above with the exception of leave out the consolpi-mdnsbrowse bit (no systemd on wsl) + - Install process: Same as above with the exception of leave out the consolpi-mdnsbrowse bit (no systemd on wsl). - It works as expected, with the minor caveat that it's only source to get remote details is via cloud-sync. Adapter data is still refreshed on menu-load by querying the remote directly. You also can not create the cloud credentials files (do the initial Authorization) in wsl. That needs to be done on another system and copied over. > For Alternative installs. Use `consolepi-sync` to update ConsolePi rather than `consolepi-upgrade`. *or* just `git pull` from the `/etc/ConsolePi` directory. `consolepi-upgrade` is the installer (which will detect ConsolePi is already installed and run as upgrade), has not been tested on non RaspberryPi installs yet. @@ -729,7 +770,7 @@ HOSTS: - mm1 will use `wade_arubaos_pub.key` as the ssh private key/identity rather than the default identity file (typically `~/.ssh/id_rsa`) > The `key` specified can be in a number of places. > 1. The script always looks in `~/.ssh` first - > 2. full path and relative (to cwd) path are also valid in the config (i.e. `key: /home/pi/mykeys/wade_arubaos_pub.key`) + > 2. full path and relative (to cwd) path are also valid in the config (i.e. `key: /home/wade/mykeys/wade_arubaos_pub.key`) > 3. Lastly if you create the dir `/etc/ConsolePi/.ssh` and place the key there. It will be copied to the users .ssh dir (`~/.ssh`) on menu launch and permissions/ownership will be updated appropriately for the logged in user. > > Option 3 has the benefit of providing a single global identity file for the host regardless of what user you are logged in as on the ConsolePi. If the file in `/etc/ConsolePi/.ssh` is updated, the menu will detect the change, and copy the new file. @@ -972,6 +1013,7 @@ There are a few convenience commands created for ConsolePi during the automated - `consolepi-version`: Displays ConsolePi version (which is in the format YYYY-MajorRel.MinorRel) - `consolepi-wlanreset`: Disables hotspot if enabled, disconnects from AP if connected as a station. Then starts wpa_supplicant. Otherwise attempts to reset WLAN adapter to initial bootup state, autohotspot will not run (use `consolepi-autohotspot` after reset to force autohotspot logic to run). - `consolepi-wlanscan`: Scan and display SSIDs visible to this ConsolePi, does not impact existing connection. +- `consolepi-convert`: A utility that parses ser2net v3 config (`/etc/ser2net.conf`) and creates a ConsolePi compatible ser2net v4 config (`/etc/ser2net.yaml`). Buster brings with it an upgrade from ser2netv3 to v4. This script will migrate any existing adapter deffinitions (configured via `consolepi-addconsole` or the `rn` option in the menu) to v4 and create / update the v4 config. - `consolepi-help`: Shows this output ## Upgrading ConsolePi diff --git a/changelog.md b/changelog.md index 85ece924..e9b6fe57 100644 --- a/changelog.md +++ b/changelog.md @@ -1,4 +1,11 @@ # CHANGELOG +### Sep 2022 (v2022-3.0) **Breaking Change for silent installs** + - Changed cmd-line flags for `consolepi-image` and `consolepi-install`/`consolepi-upgrade`. Use `--help` with those commands to see the changes. + - This is a breaking change for silent install. If using an install.conf file refer to the new example as some varirables have changed. + - Re-worked `consolepi-image` script ([consolepi-image-creator.sh](installer/consolepi-image-creator.sh)) to configure consolepi as the default user on the image. + - This is necessary for headless installs, as there is no default pi user anymore. + - Updated installation script... worked-around some dependencies that required rust build environment. + - Various other improvements to both of the above mentioned scripts. ### Nov 2021 (v2021-1.5) - Fix: RPI.GPIO set to use 0.7.1a4+ to accommodate known issue with python3.9 (bullseye default) - Fix: bluetooth.service template updated for bullseye (dynamically handles both bullseye where exec path changed and prev rel) From 6290565f801668fc6944b2d483e6b26c3a0c0c08 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 23:55:14 -0500 Subject: [PATCH 29/30] set process title --- src/consolepi-api.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/consolepi-api.py b/src/consolepi-api.py index 249b8612..56e3b5a1 100644 --- a/src/consolepi-api.py +++ b/src/consolepi-api.py @@ -12,14 +12,22 @@ to ensure the remote is reachable and that the data is current. ''' import sys + +import setproctitle +import uvicorn # NoQA +from rich.traceback import install + sys.path.insert(0, '/etc/ConsolePi/src/pypkg') -from consolepi import config, log # NoQA -from consolepi.consolepi import ConsolePi # NoQA +from consolepi import config, log # type: ignore # NoQA +from consolepi.consolepi import ConsolePi # type: ignore # NoQA from fastapi import FastAPI # NoQA from pydantic import BaseModel # NoQA from time import time # NoQA from starlette.requests import Request # NoQA -import uvicorn # NoQA + +install(show_locals=True) + +setproctitle.setproctitle("consolepi-api") cpi = ConsolePi() From 7ccbff917c8b3028c2a6ca41406effa64627ea67 Mon Sep 17 00:00:00 2001 From: wade ~ Pack3tL0ss Date: Tue, 18 Jul 2023 23:55:23 -0500 Subject: [PATCH 30/30] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 36c003d1..0db3d960 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Prior Changes can be found in the - [ChangeLog](changelog.md) - :zap: Convert remote ConsolePi updates to async (they were already using threading) - :zap: Convert remote ConsolePi updates to async (they were already using threading) - :loud_sound: Update Spinner with the name of the remote as reachability is being check for remote ConsolePis. Make failures persistent (spinner shows what failed and continues one line down.) + - The various consolepi-services that run as daemons (for remote discovery) now display a descriptive process name (i.e. when running `top` and the like) vs generically `python3` - :construction: (Requires manual setup for now see issue [#119](https://github.com/Pack3tL0ss/ConsolePi/issues/119)) Add ability to ssh directly to an adapter specifying adapter by name - i.e. `ssh -t -p 2202 ` - real example `ssh -t consolepi4 -p 2202 r1-8360-TOP` will connect to the defined udev alias `/dev/r1-8360-TOP` connected to remote ConsolePi ConsolePi4 (you could use ip vs hostname)