22.10

Neon Core 22.10

.dockerignore

@@ -0,0 +1 @@
+docker

.github/workflows/setup_tests.yml

@@ -29,6 +29,24 @@ jobs:
       - name: Test Skills Installation
         run: |
           pip install .[skills_required,skills_essential,skills_default,skills_extended]
+  pi_image:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test chroot installation
+        uses: pguyot/arm-runner-action@v2
+        with:
+          optimize_image: false
+          base_image: https://2222.us/app/files/neon_images/pi/ubuntu_22_04.img.xz
+          cpu: cortex-a53
+          copy_repository_path: /core
+          commands: |
+            bash /core/test/pi_setup.sh
+            . /core/venv/bin/activate
+            neon-audio init-plugin -p coqui
+            neon-speech init-plugin -p deepspeech_stream_local
+
   legacy-remote:
     strategy:
       matrix:

.gitignore

@@ -33,4 +33,10 @@ test_conf.json
 
 # Neon dev files
 .*.tmp
-./ngi*.yml
+./ngi*.yml
+
+# Docker files
+docker/xdg/*
+docker/skills/*
+docker/config/*
+!docker/config/neon.yaml

Dockerfile

@@ -52,5 +52,4 @@ CMD ["/root/run.sh"]
 
 FROM base as default_skills
 RUN pip install .[skills_required,skills_essential,skills_default,skills_extended]
-# TODO: Default skill installation is a temporary step until all skills are pip installable
-RUN neon-install-default-skills
+# Default skills from configuration are installed at container creation

README.md

@@ -1,6 +1,11 @@
 
 # Table of Contents
 0. [Quick Start](#quick-start)
+   * [a. Prerequisite Setup](#a-prerequisite-setup)
+   * [b. Running Neon](#b-running-neon)
+   * [c. Interacting with Neon](#c-interacting-with-neon)
+   * [d. Skill Development](#d-skill-development)
+   * [e. Persistent Data](#e-persistent-data)
 1. [Optional Service Account Setup](#optional-service-account-setup)  
    * [a. Google Cloud Speech](#a-google-cloud-speech-setup)  
    * [b. Amazon Polly and Translate](#b-amazon-polly-and-translate-setup)
@@ -33,6 +38,7 @@ Neon Core is only tested on Ubuntu, but should be compatible with any linux dist
 
 > *Note*: By default, only the `root` user has permissions to interact with Docker under Ubuntu.
 > To allow the current user to modify Docker containers, you can add them to the `docker` group with:
+> 
 > `sudo usermod -aG docker $USER && newgrp`
 
 ## b. Running Neon
@@ -42,6 +48,13 @@ assume that the repository is cloned to: `~/NeonCore`.
 > *Note*: The `docker` directory includes required hidden files. If you copy files, make sure to include any hidden
 > files. In must Ubuntu distros, you can toggle hidden file visibility in the file explorer with `CTRL` + `h`.
 
+> *Note*: If you run `docker` commands with `sudo`, make sure to use the `-E` flag to preserve runtime envvars.
+
+> *Note*: Some Docker implementations don't handle relative paths.
+> If you encounter errors, try updating the paths in `.env` to absolute paths.
+> Also note that any environment variables will override the default values in `.env`.
+> In BASH shells, you can list all current envvars with `env`
+
 You can start all core modules with:
 ```shell
 # cd into the directory containing docker-compose.yml
@@ -100,12 +113,22 @@ to:
     image: ghcr.io/neongeckocom/neon_skills:dev
 ```
 
-## e. Configuration
-The `neon.yaml` file included in the `docker` directory contains a default configuration
-that may be modified to specify different plugins and other runtime settings.
-The `docker` directory is mounted read-only to `/config` in each of the containers,
-so model files may be placed there and the configuration updated to use different STT/TTS plugins with
-local models.
+## e. Persistent Data
+The `xdg/config` directory is mounted to each of the Neon containers as `XDG_CONFIG_HOME`.
+`xdg/config/neon/neon.yaml` can be modified to change core configuration values.
+`xdg/config/neon/skills` contains settings files for each loaded skill
+
+The `xdg/data` directory is mounted to each of the Neon containers as `XDG_DATA_HOME`.
+`xdg/data/neon/filesystem` contains skill filesystem files.
+`xdg/data/neon/resources` contains user skill resource files.
+
+The `xdg/cache` directory is mounted to each of the Neon containers as `XDG_CACHE_HOME`.
+Any cache information should be recreated as needed if manually removed and includes things like
+STT/TTS model files, TTS audio files, and other downloaded files.
+
+> *Note*: When Docker creates files on the host filesystem, they are owned by `root`.
+> In order to modify anything in the `xdg` directory, you may need to take ownership with:
+> `sudo chown -R $USER:$USER xdg`
 
 # Optional Service Account Setup
 There are several online services that may be used with Neon. Speech-to-Text (STT) and Text-to-Speech (TTS) may be run 

docker/.env

@@ -1,2 +1,3 @@
 NEON_SKILLS_DIR=./skills/
-NEON_CONFIG_PATH=./config/
+NEON_CONFIG_PATH=./config/
+NEON_XDG_PATH=./xdg

docker/config/neon.yaml

@@ -1,6 +1,3 @@
-play_wav_cmdline: "play %1"
-play_mp3_cmdline: "play %1"
-play_ogg_cmdline: "play %1"
 websocket:
   host: neon-messagebus
 gui_websocket:
@@ -12,6 +9,10 @@ ready_settings:
   - speech
   - audio
   - gui
-#MQ:
-#  server: mq.2021.us
-#  port: 25672
+skills:
+  default_skills:
+    # Jokes skill included because it cannot be pip installed to the image
+    - https://github.com/JarbasSkills/skill-icanhazdadjokes/tree/dev
+MQ:
+  server: mq.2021.us
+  port: 25672

docker/docker-compose.yml

@@ -1,4 +1,3 @@
-# TODO: Handle XDG_CONFIG_HOME envvar for persistent mounted config
 version: '3.1'
 networks:
   neon-core:
@@ -8,6 +7,11 @@ volumes:
       type: config
       o: bind
       device: ${NEON_CONFIG_PATH}
+  xdg:
+    driver_opts:
+      type: xdg
+      o: bind
+      device: ${NEON_XDG_PATH}
 services:
   neon-messagebus:
     container_name: neon-messagebus
@@ -20,6 +24,12 @@ services:
         - messagebus
     volumes:
     - config:/config:ro
+    - xdg:/xdg:rw
+    environment:
+    - XDG_CONFIG_HOME=/xdg/config
+    - XDG_DATA_HOME=/xdg/data
+    - XDG_CACHE_HOME=/xdg/cache
+    - XDG_STATE_HOME=/xdg/state
   neon-speech:
     container_name: neon-speech
     image: ghcr.io/neongeckocom/neon_speech:dev
@@ -29,9 +39,14 @@ services:
     - config:/config:ro
     - ~/.config/pulse/cookie:/tmp/pulse_cookie:ro
     - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro
+    - xdg:/xdg:rw
     environment:
     - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native
     - PULSE_COOKIE=/tmp/pulse_cookie
+    - XDG_CONFIG_HOME=/xdg/config
+    - XDG_DATA_HOME=/xdg/data
+    - XDG_CACHE_HOME=/xdg/cache
+    - XDG_STATE_HOME=/xdg/state
     devices:
     - /dev/snd:/dev/snd
   neon-skills:
@@ -46,9 +61,14 @@ services:
     - ~/.config/pulse/cookie:/tmp/pulse_cookie:ro
     - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro
     - ${NEON_SKILLS_DIR}:/skills:ro
+    - xdg:/xdg:rw
     environment:
     - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native
     - PULSE_COOKIE=/tmp/pulse_cookie
+    - XDG_CONFIG_HOME=/xdg/config
+    - XDG_DATA_HOME=/xdg/data
+    - XDG_CACHE_HOME=/xdg/cache
+    - XDG_STATE_HOME=/xdg/state
     devices:
     - /dev/snd:/dev/snd
   neon-audio:
@@ -62,10 +82,15 @@ services:
     - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro
     - /run/dbus/system_dbus_socket:/run/dbus/system_dbus_socket:rw
     - /tmp/.X11-unix:/tmp/.X11-unix:ro
+    - xdg:/xdg:rw
     environment:
     - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native
     - PULSE_COOKIE=/tmp/pulse_cookie
     - DISPLAY=${DISPLAY}
+    - XDG_CONFIG_HOME=/xdg/config
+    - XDG_DATA_HOME=/xdg/data
+    - XDG_CACHE_HOME=/xdg/cache
+    - XDG_STATE_HOME=/xdg/state
     devices:
     - /dev/snd:/dev/snd
   neon-gui:
@@ -78,4 +103,10 @@ services:
     ports:
     - 18181:18181
     volumes:
-    - config:/config:ro
+    - config:/config:ro
+    - xdg:/xdg:rw
+    environment:
+    - XDG_CONFIG_HOME=/xdg/config
+    - XDG_DATA_HOME=/xdg/data
+    - XDG_CACHE_HOME=/xdg/cache
+    - XDG_STATE_HOME=/xdg/state

docker/xdg/config/neon/skills/skill-homescreen-lite.openvoiceos/settings.json

@@ -0,0 +1,8 @@
+{
+  "weather_skill": "skill-weather.neongeckocom",
+  "datetime_skill": "skill-date_time.neongeckocom",
+  "examples_skill": "skill-about.neongeckocom",
+  "wallpaper": "default.jpg",
+  "examples_enabled": true,
+  "__mycroft_skill_firstrun": false
+}

docker/xdg/data/neon/resources/skill-demo.neongeckocom/locale/en-us/gui_demo.txt

@@ -0,0 +1,4 @@
+what time is it
+what is microsoft trading at
+how far away is mars
+what is the forecast for seattle

docker/xdg/data/neon/resources/skill-demo.neongeckocom/locale/en-us/language_demo.txt

@@ -0,0 +1,25 @@
+who are you
+change my response language to spanish
+who are you
+change my response language to french
+who are you
+change my response language to german
+who are you
+change my response language to italian
+who are you
+change my response language to polish
+who are you
+change my response language to ukrainian
+who are you
+change my response language to hungarian
+who are you
+change my response language to greek
+who are you
+change my response language to swedish
+who are you
+change my response language to dutch
+who are you
+change my response language to finnish
+who are you
+change my response language to irish
+who are you

docker_overlay/etc/OpenVoiceOS/ovos.conf

@@ -1,13 +1,13 @@
 {
     "module_overrides": {
         "neon_core": {
-            "xdg": true,
             "base_folder": "neon",
             "config_filename": "neon.yaml",
-            "default_config_path": "/neon_core/configuration/neon.yaml"
+            "default_config_path": "/neon_core/neon_core/configuration/neon.yaml"
         }
     },
     "submodule_mappings": {
-        "neon_core": "neon_core"
+        "neon_core": "neon_core",
+        "neon_core.skills.skill_manager": "neon_core"
     }
 }

docker_overlay/root/run.sh

@@ -27,6 +27,7 @@
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# Plugin installation must occur in a separate thread, before module load, for the entry point to be loaded.
+# Python package installation must occur in a separate thread, before module load, for the entry point to be loaded.
+neon install-default-skills
 neon install-skill-requirements /skills
 neon run-skills

neon_core/configuration/__init__.py

@@ -29,7 +29,7 @@
 from ovos_config.config import Configuration
 from os.path import exists, isdir, dirname
 from os import makedirs
-from neon_utils.logger import LOG
+from ovos_utils.log import LOG
 
 """
 Neon modules should import config from this module since module_overrides will