v2023-6.0

.static.yaml

@@ -2,7 +2,7 @@
 # This file should not be modified.  Static variables/paths used throughout ConsolePi
 # Versioning = YYYY.Major.Patch/Minor
 ---
-CONSOLEPI_VER: 2023-5.2
+CONSOLEPI_VER: 2023-6.0
 INSTALLER_VER: 67
 CFG_FILE_VER: 10
 CONFIG_FILE_YAML: /etc/ConsolePi/ConsolePi.yaml

changelog.md

@@ -1,4 +1,11 @@
 # CHANGELOG
+### Sep 2022 (v2022-3.0)  **Breaking Change for silent installs**
+  - Changed cmd-line flags for `consolepi-image` and `consolepi-install`/`consolepi-upgrade`.  Use `--help` with those commands to see the changes.
+    - This is a breaking change for silent install.  If using an install.conf file refer to the new example as some varirables have changed.
+  - Re-worked `consolepi-image` script ([consolepi-image-creator.sh](installer/consolepi-image-creator.sh)) to configure consolepi as the default user on the image.
+    - This is necessary for headless installs, as there is no default pi user anymore.
+  - Updated installation script... worked-around some dependencies that required rust build environment.
+  - Various other improvements to both of the above mentioned scripts.
 ### Nov 2021 (v2021-1.5)
   - Fix: RPI.GPIO set to use 0.7.1a4+ to accommodate known issue with python3.9 (bullseye default)
   - Fix: bluetooth.service template updated for bullseye (dynamically handles both bullseye where exec path changed and prev rel)

installer/requirements.txt

@@ -52,3 +52,6 @@ Werkzeug>=0.15.4
 # wrapt>=1.11.2
 zeroconf>=0.23.0
 rich
+setproctitle
+aiohttp
+asyncio

src/consolepi-api.py

@@ -12,14 +12,22 @@
 to ensure the remote is reachable and that the data is current.
 '''
 import sys
+
+import setproctitle
+import uvicorn  # NoQA
+from rich.traceback import install
+
 sys.path.insert(0, '/etc/ConsolePi/src/pypkg')
-from consolepi import config, log  # NoQA
-from consolepi.consolepi import ConsolePi  # NoQA
+from consolepi import config, log  # type: ignore # NoQA
+from consolepi.consolepi import ConsolePi  # type: ignore # NoQA
 from fastapi import FastAPI  # NoQA
 from pydantic import BaseModel  # NoQA
 from time import time  # NoQA
 from starlette.requests import Request  # NoQA
-import uvicorn  # NoQA
+
+install(show_locals=True)
+
+setproctitle.setproctitle("consolepi-api")
 
 
 cpi = ConsolePi()

src/consolepi-commands/consolepi-convert

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo /etc/ConsolePi/venv/bin/python3 /etc/ConsolePi/src/convert-ser2net.py "$@"

src/consolepi-commands/consolepi-editudev

@@ -1,5 +1,15 @@
 #!/usr/bin/env bash
 
-sudo nano /etc/udev/rules.d/10-ConsolePi.rules /etc/ser2net.conf
+[ -f /etc/ser2net.yaml ] && ser2net_file=/etc/ser2net.yaml
+[ -f /etc/ser2net.yml ] && ser2net_file=/etc/ser2net.yml
+[ -f /etc/ser2net.conf ] && ser2net_file=/etc/ser2net.conf
+
+if [ -n "$ser2net_file" ]; then
+    sudo nano /etc/udev/rules.d/10-ConsolePi.rules "$ser2net_file"
+else
+    sudo nano /etc/udev/rules.d/10-ConsolePi.rules
+    echo '!!! No ser2net file found to edit.  Run consolepi-upgrade to init ser2net config.'
+fi
+
 
 echo use consolepi-showaliases to validate udev and ser2net, check for orphans

src/consolepi-menu.py

@@ -11,6 +11,7 @@
 from collections import OrderedDict as od
 from typing import Union
 from halo import Halo
+import asyncio
 # from rich.console import Console
 
 # --// ConsolePi imports \\--
@@ -217,7 +218,7 @@ def power_menu(self, calling_menu: str = 'main_menu'):
         cpi = self.cpi
         pwr = cpi.pwr
         # menu = self.menu
-        menu = Menu()
+        menu = Menu("power_menu")
         states = self.states
         menu_actions = {
             # 'b': self.main_menu,
@@ -891,7 +892,7 @@ def rename_menu(self, direct_launch: bool = False, from_name: str = None):
             if choice == 'r':
                 local.adapters = local.build_adapter_dict(refresh=True)
                 if not direct_launch:
-                    remotes.data = remotes.get_remote(data=config.remote_update())
+                    remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update()))
             loc = local.adapters
             rem = remotes.data if not direct_launch else []
 
@@ -956,7 +957,7 @@ def rename_menu(self, direct_launch: bool = False, from_name: str = None):
             if choice.isdigit() and int(choice) >= rem_item:
                 print('Triggering Refresh due to Remote Name Change')
                 # remotes.refresh(bypass_cloud=True)  # NoQA TODO would be more ideal just to query the remote involved in the rename and update the dict
-                remotes.data = remotes.get_remote(data=config.remote_update(), rename=True)
+                remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update(), rename=True))
 
             # TODO Temp need more elegant way to handle back to main_menu
             elif menu_actions.get(choice, {}) is None and choice == "b":
@@ -973,12 +974,7 @@ def refresh_local(self):
         cpi = self.cpi
         remotes = cpi.remotes
         cpi.local.adapters = cpi.local.build_adapter_dict(refresh=True)
-        remotes.data = remotes.get_remote(data=config.remote_update())
-        # loc = cpi.local.adapters
-        # rem = remotes.data
-
-    # def toggle_show_legend(self):
-    #     self.menu.show_legend = not self.menu.show_legend
+        remotes.data = asyncio.run(remotes.get_remote(data=config.remote_update()))
 
     # ------ // MAIN MENU \\ ------ #
     def main_menu(self):
@@ -1168,10 +1164,10 @@ def rshell_menu(self):
     def con_menu(self, rename: bool = False, con_dict: dict = None):
         # menu = self.cpi.menu
         menu = Menu("con_menu")
-        menu.legend_options = {
-            'back': ['b', 'Back'],
-            'x': ['x', 'Exit']
-        }
+        # menu.legend_options = {
+        #     'back': ['b', 'Back'],
+        #     'x': ['x', 'Exit']
+        # }
         menu_actions = {
             '1': self.baud_menu,
             '2': self.data_bits_menu,
@@ -1194,7 +1190,7 @@ def con_menu(self, rename: bool = False, con_dict: dict = None):
             self.flow = con_dict['flow']
             self.sbits = con_dict['sbits']
         while True:
-            header = ' Connection Settings Menu '
+            header = 'Connection Settings Menu'
             mlines = []
             mlines.append('Baud [{}]'.format(self.baud))
             mlines.append('Data Bits [{}]'.format(self.data_bits))
@@ -1203,7 +1199,7 @@ def con_menu(self, rename: bool = False, con_dict: dict = None):
             legend = {'opts': ['back', 'x'],
                       'overrides': {'back': ['b', 'Back {}'.format(' (Apply Changes to Files)' if rename else '')]}
                       }
-            menu.print_menu(mlines, header=header, legend=legend, menu_actions=menu_actions)
+            menu.print_menu(mlines, header=header, legend=legend, menu_actions=menu_actions, hide_legend=False)
             ch = self.wait_for_input(locs=locals()).lower
             try:
                 if ch == 'b':
@@ -1212,13 +1208,14 @@ def con_menu(self, rename: bool = False, con_dict: dict = None):
 
             except KeyError as e:
                 if ch:
-                    log.show('Invalid selection {}, please try again.'.format(e))
+                    log.show(f'Invalid selection {e}, please try again.')
                     log.clear()
 
     # -- // BAUD MENU \\ --
     def baud_menu(self):
         # config = self.cpi.config
-        menu = self.cpi.menu
+        # menu = self.cpi.menu
+        menu = Menu("baud_menu")
         menu_actions = od([
             ('1', 300),
             ('2', 1200),
@@ -1243,6 +1240,7 @@ def baud_menu(self):
                 print(' {0}. {1}'.format(key, _cur_baud if _cur_baud != self.baud else '[{}]'.format(_cur_baud)))
 
             legend = menu.format_legend(legend={"opts": 'back'})
+            menu.page.legend.hide = False  # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu"
             footer = menu.format_footer()
             print(legend)
             print(footer)
@@ -1277,7 +1275,8 @@ def baud_menu(self):
 
     # -- // DATA BITS MENU \\ --
     def data_bits_menu(self):
-        menu = self.cpi.menu
+        # menu = self.cpi.menu
+        menu = Menu("data_bits")
         valid = False
         while not valid:
             if not config.debug:
@@ -1287,6 +1286,7 @@ def data_bits_menu(self):
             print('\n Default 8, Current [{}], Valid range 5-8'.format(self.data_bits))
 
             legend = menu.format_legend(legend={"opts": 'back'})
+            menu.page.legend.hide = False  # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu"
             footer = menu.format_footer()
             print(legend)
             print(footer)
@@ -1310,7 +1310,7 @@ def data_bits_menu(self):
 
     # -- // PARITY MENU \\ --
     def parity_menu(self):
-        menu = self.cpi.menu
+        menu = Menu("parity_menu")
 
         def print_menu():
             if not config.debug:
@@ -1323,6 +1323,7 @@ def print_menu():
             print(f" 3. {'[Even]' if self.parity == 'e' else 'Even'}")
 
             legend = menu.format_legend(legend={"opts": 'back'})
+            menu.page.legend.hide = False  # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu"
             footer = menu.format_footer()
             print(legend)
             print(footer)
@@ -1350,7 +1351,7 @@ def print_menu():
 
     # -- // FLOW MENU \\ --
     def flow_menu(self):
-        menu = self.cpi.menu
+        menu = Menu("flow_menu")
 
         def print_menu():
             if not config.debug:
@@ -1363,6 +1364,7 @@ def print_menu():
             print(f" 2. {'[Xon/Xoff]' if self.flow == 'x' else 'Xon/Xoff'} (software)")
             print(f" 3. {'[RTS/CTS]' if self.flow == 'h' else 'RTS/CTS'} (hardware)")
             legend = menu.format_legend(legend={"opts": 'back'})
+            menu.page.legend.hide = False  # TODO make elegant this prevents "Use 'TL' to Toggle Legend from being displayed in footer given we are not giving an option in this menu"
             footer = menu.format_footer()
             print(legend)
             print(footer)

src/convert-ser2net.py

@@ -0,0 +1,150 @@
+#!/etc/ConsolePi/venv/bin/python3
+
+import sys
+from pathlib import Path
+import shutil
+from rich import print
+from rich.prompt import Confirm
+from datetime import datetime
+from typing import List
+
+sys.path.insert(0, '/etc/ConsolePi/src/pypkg')
+from consolepi import config  # type: ignore # NoQA
+
+DEF_V4_FILE = Path("/etc/ConsolePi/src/ser2net.yaml")
+V3_FILE = Path("/etc/ser2net.conf")
+BAK_DIR = Path("/etc/ConsolePi/bak")
+DEF_PROMPT = f"""
+[green]--------------------[/] [cyan]ser2net[/] [bright_red]v3[/] to [green3]v4[/] conversion utility [green]--------------------[/]
+This utility will parse the [cyan]ser2net[/] v3 configuration file [cyan italic]{V3_FILE.name}[/]
+and create a ConsolePi compatible ser2net v4 config [cyan italic]ser2net.yaml[/].
+
+If ser2net v4 is installed the v3 config will be moved to {BAK_DIR}.
+"""
+
+
+class Convert:
+    def __init__(self):
+        self.v4_file: Path = self.get_v4_config_name()
+
+    def __call__(self):
+        if "-y" in " ".join(sys.argv[1:]).lower() or self.confirm():
+            _ = self.check_v4_config()
+            v4_lines = self.gather_config_data()
+            if self.v4_file.exists() and self.v4_file.read_text() == v4_lines:
+                print(f'  [magenta]-[/] Process resulted in no change in {self.v4_file.name} content.  Existing v4 config left as is.')
+            else:
+                print(f'  [magenta]-[/] Writing converted config to {self.v4_file}')
+                self.v4_file.write_text(v4_lines)
+                self.backup_v3_config()
+            print()
+        else:
+            print("[bright_red]Aborted.")
+
+    def get_v4_config_name(self) -> Path:
+        new_file = Path("/etc/ser2net.yaml")
+        if not new_file.exists() and Path("/etc/ser2net.yml").exists():
+            new_file = Path("/etc/ser2net.yml")
+
+        return new_file
+
+    def check_v4_config(self) -> bool:
+        if self.v4_file.exists():
+            if "ConsolePi" not in self.v4_file.read_text().splitlines()[2]:
+                now = datetime.now()
+                _bak_file = BAK_DIR / f'{self.v4_file.name}.{now.strftime("%F_%H%M")}'
+                print(f'  [magenta]-[/] Backing up existing non ConsolePi version of [cyan]{self.v4_file}[/] to [cyan]{_bak_file}[/].')
+                shutil.move(self.v4_file, _bak_file)
+                do_ser2net = True
+                print("  [magenta]-[/] Preparing default ConsolePi ser2net v4 base config.")
+            else:
+                print(f'  [magenta]-[/] Found existing ConsolePi compatible version of [cyan]{self.v4_file}[reset].')
+                do_ser2net = False
+        else:
+            do_ser2net = True
+            print("  [magenta]-[/] Preparing default ConsolePi ser2net v4 base config.")
+
+        return do_ser2net
+
+    def backup_v3_config(self):
+        if int("".join(config.ser2net_ver.split(".")[0])) > 3:
+            if not self.v4_file.exists:
+                print(f'  [dark_orange3][blink]!![/blink] WARNING[/]: ser2net v4 Configuration ({self.v4_file}) not found.  Keeping ser2net v3 file ({V3_FILE}) in place')
+            else:
+                now = datetime.now()
+                _bak_file = BAK_DIR / f'{V3_FILE.name}.{now.strftime("%F_%H%M")}'
+                print(f"  [magenta]-[/] ser2net [cyan]v{config.ser2net_ver}[/] installed.  Backing up v3 config {V3_FILE} to {_bak_file}")
+                shutil.move(V3_FILE, _bak_file)
+        else:
+            print(f"  [magenta]-[/] ser2net [cyan]v{config.ser2net_ver}[/] installed.  Keeping v3 config {V3_FILE} in place.")
+
+    def gather_config_data(self) -> str:
+        v3_aliases = {k: v for k, v in config.ser2net_conf.items() if not k.startswith("_") and 7000 < v.get('port', 0) <= 7999}
+        if self.v4_file.exists():
+            v4_config: List[str] = self.v4_file.read_text().splitlines(keepends=True)
+            v4_config_dict = config.get_ser2netv4(self.v4_file)
+            v4_aliases = {k: v for k, v in v4_config_dict.items() if not k.startswith("_") and 7000 < v.get('port', 0) <= 7999}
+        else:
+            v4_config: List[str] = DEF_V4_FILE.read_text().splitlines(keepends=True)
+            v4_aliases = {}
+
+        v4_user_lines = [v["v4line"] for k, v in v3_aliases.items() if k not in v4_aliases.keys()]
+
+        # Gather any trace-file references from v3 config and ensure they are defined in v4 config.
+        if "trace-" in "".join(v4_user_lines):
+            trace_lines = [line.strip().split(":")[-1].strip() for line in v4_user_lines if "trace-" in line]
+            new_defs = [ref.lstrip("*") for ref in trace_lines if ref.startswith("*")]  # trace-file defs referenced in old config
+            if new_defs:
+                existing_defs, get_next_empty, insert_idx = [], False, 19
+                for idx, line in enumerate(v4_config):
+                    if "define: &" in line and "banner" not in line and "/" in line:
+                        existing_defs += [line.split("&")[-1].split()[0]]  # define: &trace3 /var/log/ser2net/trace3...
+                    elif get_next_empty:
+                        if not line.strip() or not line.strip().startswith("define:"):
+                            get_next_empty = False
+                            insert_idx = idx
+                    elif "# TRACEFILE DEFINITIONS" in line.strip():
+                        get_next_empty = True
+                    else:
+                        continue
+
+                missing_defs = [ref for ref in new_defs if ref not in existing_defs]
+                if missing_defs:
+                    missing_lines = [line for line in config.ser2net_conf.get("_v4_tracefiles", "").splitlines() for d in missing_defs if not line.startswith("#") and f'define: &{d}' in line]
+                    if missing_lines:
+                        print(f"  [magenta]-[/] Adding trace-file definition{'s' if len(missing_defs) > 1 else ''} [cyan]{'[/], [cyan]'.join(missing_defs)}[/] referenced in v3 config.")
+                        v4_config.insert(insert_idx, "\n".join(missing_lines) + "\n")
+
+        if v4_user_lines:
+            new_cnt = len([line for line in v4_user_lines if line.startswith("connection:")])
+            print(f"  [magenta]-[/] Adding {new_cnt} alias definitions converted from {config.ser2net_file}")
+            v4_config += v4_user_lines
+            if len(v3_aliases) != new_cnt:
+                print(f"  [magenta]-[/] {len(v3_aliases) - new_cnt} aliases found in ser2net v3 config were skipped as they are already defined in the existing v4 config.")
+        elif v3_aliases and v4_aliases:
+            print(f'  [dark_orange3][blink]!![/blink] WARNING[/]: All {len(v3_aliases)} alias definitions found in the ser2net v3 config were ignored as they are already defined in the existing v4 config.')
+        else:
+            print(f'  [dark_orange3][blink]!![/blink] WARNING[/]: No User defined aliases found in {V3_FILE.name}.')
+
+        return "".join(v4_config)
+
+    @staticmethod
+    def confirm(prompt: str = None) -> bool:
+        prompt = prompt or DEF_PROMPT
+        print(DEF_PROMPT)
+        choice = Confirm.ask("Proceed?")
+        print(f"[green]{'':-^77}[/]\n")
+
+        return choice
+
+
+if __name__ == "__main__":
+    if shutil.os.getuid() != 0:
+        print("[bright_red]Error:[/] Must be run as root.  Launch with [cyan]consolepi-convert[/].")
+        sys.exit(1)
+    elif not Path("/etc/ser2net.conf").exists():
+        print("[bright_red]Error:[/] No v3 config found ([cyan]/etc/ser2net.conf[/]).")
+        sys.exit(1)
+    else:
+        convert = Convert()
+        convert()