From 4eec8c5ff3704acd11f81bed13d9f91f26252f9e Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:59:56 -0800 Subject: [PATCH 01/70] Update Test Automation (#173) * Upload test artifacts to close #140 Increase test timeout to account for actions runners Add diagnostic utils tests to GH actions * Formatting cleanup --- .github/workflows/unit_tests.yml | 37 ++++++++++++++++++++++++++++---- test/test_diagnostic_utils.py | 1 - test/test_run_neon.py | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e5f403730..a63ac015f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -43,14 +43,28 @@ jobs: pip install -r requirements/test.txt env: GITHUB_TOKEN: ${{secrets.neon_token}} + - name: Test Skill Utils run: | - pytest test/test_skill_utils.py + pytest test/test_skill_utils.py --doctest-modules --junitxml=tests/skill-utils-test-results.xml env: GITHUB_TOKEN: ${{secrets.neon_token}} - - name: Test Language + - name: Upload Skill Utils test results + uses: actions/upload-artifact@v2 + with: + name: skill-utils-test-results + path: tests/skill-utils-test-results.xml + + - name: Test Diagnostic Utils run: | - pytest test/test_language.py + pytest test/test_diagnostic_utils.py --doctest-modules --junitxml=tests/diagnostic-utils-test-results.xml + env: + GITHUB_TOKEN: ${{secrets.neon_token}} + - name: Upload Diagnostic Utils test results + uses: actions/upload-artifact@v2 + with: + name: diagnostic-utils-test-results + path: tests/diagnostic-utils-test-results.xml unit_tests: strategy: @@ -72,6 +86,21 @@ jobs: pip install -r requirements/test.txt env: GITHUB_TOKEN: ${{secrets.neon_token}} + - name: Test Configuration Module run: | - pytest test/test_configuration.py \ No newline at end of file + pytest test/test_configuration.py --doctest-modules --junitxml=tests/configuration-test-results.xml + - name: Upload Configuration test results + uses: actions/upload-artifact@v2 + with: + name: configuration-test-results + path: tests/configuration-test-results.xml + + - name: Test Language + run: | + pytest test/test_language.py --doctest-modules --junitxml=tests/language-test-results.xml + - name: Upload Language test results + uses: actions/upload-artifact@v2 + with: + name: language-test-results + path: tests/language-test-results.xml \ No newline at end of file diff --git a/test/test_diagnostic_utils.py b/test/test_diagnostic_utils.py index 88bd7c2c3..a74c0ce5b 100644 --- a/test/test_diagnostic_utils.py +++ b/test/test_diagnostic_utils.py @@ -61,7 +61,6 @@ def setUp(self) -> None: self.report_metric.reset_mock() neon_utils.metrics_utils.report_metric = self.report_metric - def test_send_diagnostics_default(self): from neon_core.util.diagnostic_utils import send_diagnostics send_diagnostics() diff --git a/test/test_run_neon.py b/test/test_run_neon.py index a3be33489..93b59484d 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -47,7 +47,7 @@ def setUpClass(cls) -> None: cls.bus = MessageBusClient() cls.bus.run_in_thread() cls.bus.connected_event.wait() - cls.bus.wait_for_message("mycroft.ready", 360) + cls.bus.wait_for_message("mycroft.ready", 600) @classmethod def tearDownClass(cls) -> None: From 598e966a342a53420eb9ec5d65a1e7eaadec145e Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 24 Nov 2021 19:00:17 +0000 Subject: [PATCH 02/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 36b0034dc..04eae1c82 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.1" +__version__ = "21.10.2a0" From 9cd28ba26aed368bae119c4d9c3c9c8b920b9b1e Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:01:45 -0800 Subject: [PATCH 03/70] Add any missing license file headers (#172) Consolidate all LOG imports to neon_utils Refactor superseded `mycroft` imports Mark stt and tts modules as deprecated --- neon_core/configuration/__init__.py | 25 ++++++++++++++++++++++++ neon_core/dialog/__init__.py | 4 +++- neon_core/gui/resting_screen.py | 2 +- neon_core/messagebus/__init__.py | 5 +++-- neon_core/messagebus/service/__init__.py | 7 ++++--- neon_core/processing_modules/__init__.py | 9 +++++---- neon_core/skills/__main__.py | 25 ++++++++++++++++++++++++ neon_core/skills/decorators.py | 10 ++++++---- neon_core/skills/display_service.py | 2 +- neon_core/skills/neon_skill.py | 25 ++++++++++++++++++++++++ neon_core/skills/service.py | 21 ++++++++++---------- neon_core/skills/skill_store.py | 18 ++++++++--------- neon_core/stt/__init__.py | 4 ++++ neon_core/tts/__init__.py | 3 +++ neon_core/util/skill_utils.py | 2 +- 15 files changed, 125 insertions(+), 37 deletions(-) diff --git a/neon_core/configuration/__init__.py b/neon_core/configuration/__init__.py index 587c69df4..37cc663a0 100644 --- a/neon_core/configuration/__init__.py +++ b/neon_core/configuration/__init__.py @@ -1,3 +1,28 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from mycroft.configuration.config import Configuration, LocalConf def get_private_keys(): diff --git a/neon_core/dialog/__init__.py b/neon_core/dialog/__init__.py index c16d40b20..2b080e72a 100644 --- a/neon_core/dialog/__init__.py +++ b/neon_core/dialog/__init__.py @@ -22,10 +22,12 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from os.path import join +from neon_utils import LOG + from mycroft.dialog import MustacheDialogRenderer, load_dialogs from mycroft.util import resolve_resource_file -from mycroft.util.log import LOG def get(phrase, lang=None, context=None): diff --git a/neon_core/gui/resting_screen.py b/neon_core/gui/resting_screen.py index a65998bd4..eb6749f1a 100644 --- a/neon_core/gui/resting_screen.py +++ b/neon_core/gui/resting_screen.py @@ -26,7 +26,7 @@ import time from threading import Lock -from neon_utils.log_utils import LOG +from neon_utils import LOG from mycroft_bus_client import Message, MessageBusClient from neon_utils.skills.mycroft_skill import MycroftSkill diff --git a/neon_core/messagebus/__init__.py b/neon_core/messagebus/__init__.py index 86a9c7eec..da6a54ca6 100644 --- a/neon_core/messagebus/__init__.py +++ b/neon_core/messagebus/__init__.py @@ -22,13 +22,14 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json +from threading import Event from mycroft_bus_client import MessageBusClient, Message + from mycroft.messagebus.service.event_handler import MessageBusEventHandler from mycroft.util import create_daemon from mycroft.messagebus.load_config import load_message_bus_config from mycroft.util.json_helper import merge_dict -import json -from threading import Event def get_messagebus(running=True): diff --git a/neon_core/messagebus/service/__init__.py b/neon_core/messagebus/service/__init__.py index b47683fde..27c3bcccf 100644 --- a/neon_core/messagebus/service/__init__.py +++ b/neon_core/messagebus/service/__init__.py @@ -30,14 +30,15 @@ """ import asyncio import sys +import tornado.options + from os.path import expanduser, isfile from threading import Thread +from tornado import web, ioloop +from neon_utils import LOG -import tornado.options from mycroft.messagebus.load_config import load_message_bus_config from mycroft.messagebus.service.event_handler import MessageBusEventHandler -from mycroft.util.log import LOG -from tornado import web, ioloop class NeonBusService(Thread): diff --git a/neon_core/processing_modules/__init__.py b/neon_core/processing_modules/__init__.py index ab6b9dcff..da0b0f9e8 100644 --- a/neon_core/processing_modules/__init__.py +++ b/neon_core/processing_modules/__init__.py @@ -23,15 +23,16 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from threading import Thread, Event -from os.path import join, basename import os import time import sys import gc -from glob import glob import imp -from mycroft.util.log import LOG + +from glob import glob +from threading import Thread, Event +from os.path import join, basename +from neon_utils import LOG DEBUG = True diff --git a/neon_core/skills/__main__.py b/neon_core/skills/__main__.py index 64f66d570..4b473adf7 100644 --- a/neon_core/skills/__main__.py +++ b/neon_core/skills/__main__.py @@ -1,3 +1,28 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from mycroft.lock import Lock from mycroft.util import reset_sigint_handler, wait_for_exit_signal from neon_core.skills.service import NeonSkillService diff --git a/neon_core/skills/decorators.py b/neon_core/skills/decorators.py index 2aaded410..7ad338e28 100644 --- a/neon_core/skills/decorators.py +++ b/neon_core/skills/decorators.py @@ -23,14 +23,16 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Decorators for use with MycroftSkill methods""" -from functools import wraps +import time import threading + +from functools import wraps from inspect import signature -import time -from mycroft.messagebus import Message +from mycroft_bus_client import Message +from ovos_utils import create_killable_daemon + from mycroft.skills.mycroft_skill.decorators import intent_handler, \ intent_file_handler, resting_screen_handler, skill_api_method -from ovos_utils import create_killable_daemon class AbortEvent(StopIteration): diff --git a/neon_core/skills/display_service.py b/neon_core/skills/display_service.py index ce33b9d13..7eef24561 100644 --- a/neon_core/skills/display_service.py +++ b/neon_core/skills/display_service.py @@ -24,7 +24,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time -from mycroft.messagebus.message import Message +from mycroft_bus_client import Message from neon_core.messagebus import get_messagebus from os.path import abspath diff --git a/neon_core/skills/neon_skill.py b/neon_core/skills/neon_skill.py index 62edc2360..82ea56f6b 100644 --- a/neon_core/skills/neon_skill.py +++ b/neon_core/skills/neon_skill.py @@ -1,3 +1,28 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import re from os import walk from os.path import join, exists diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index 2154f4ad5..84049bc1d 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -24,22 +24,23 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time +from neon_utils.configuration_utils import get_neon_skills_config, \ + get_neon_lang_config, get_neon_local_config +from neon_utils.net_utils import check_online +from neon_utils import LOG +from neon_utils.metrics_utils import announce_connection + +from neon_core.skills.fallback_skill import FallbackSkill +from neon_core.skills.intent_service import NeonIntentService +from neon_core.skills.skill_manager import NeonSkillManager +from neon_core.util.diagnostic_utils import report_metric + from mycroft.skills.api import SkillApi from mycroft.skills.event_scheduler import EventScheduler from mycroft.skills.msm_wrapper import MsmException from mycroft.util import start_message_bus_client from mycroft.configuration.locale import set_default_lang, set_default_tz -from mycroft.util.log import LOG from mycroft.util.process_utils import ProcessStatus, StatusCallbackMap -from neon_core.skills.fallback_skill import FallbackSkill -from neon_core.skills.intent_service import NeonIntentService -from neon_core.skills.skill_manager import NeonSkillManager -from neon_utils.configuration_utils import get_neon_skills_config, \ - get_neon_lang_config, get_neon_local_config -from neon_utils.net_utils import check_online - -from neon_core.util.diagnostic_utils import report_metric -from neon_utils.metrics_utils import announce_connection def on_started(): diff --git a/neon_core/skills/skill_store.py b/neon_core/skills/skill_store.py index 62726ae4c..94f1f14ec 100644 --- a/neon_core/skills/skill_store.py +++ b/neon_core/skills/skill_store.py @@ -25,17 +25,15 @@ from ovos_skills_manager.osm import OVOSSkillsManager from ovos_skills_manager.skill_entry import SkillEntry -# from neon_core.configuration import Configuration +from neon_utils import LOG +from neon_utils.net_utils import check_online +from neon_utils.authentication_utils import repo_is_neon +from neon_utils.configuration_utils import get_neon_skills_config +from datetime import datetime, timedelta + from neon_core.messagebus import get_messagebus from neon_core.util.skill_utils import get_remote_entries from mycroft.skills.event_scheduler import EventSchedulerInterface -from mycroft.util import connected -from mycroft.util.log import LOG -# from os.path import expanduser -from datetime import datetime, timedelta - -from neon_utils.authentication_utils import repo_is_neon -from neon_utils.configuration_utils import get_neon_skills_config class SkillsStore: @@ -76,7 +74,7 @@ def handle_update(self, _): try: self.install_default_skills(update=True) except Exception as e: - if connected(): + if check_online(): # if there is internet log the error LOG.exception(e) LOG.error("skills update failed") @@ -88,7 +86,7 @@ def handle_sync_appstores(self, _): try: self.osm.sync_appstores() except Exception as e: - if connected(): + if check_online(): # if there is internet log the error LOG.exception(e) LOG.error("appstore sync failed") diff --git a/neon_core/stt/__init__.py b/neon_core/stt/__init__.py index e14809a95..087474ef7 100644 --- a/neon_core/stt/__init__.py +++ b/neon_core/stt/__init__.py @@ -23,4 +23,8 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # backwards compatibility + from neon_speech.stt import * +from neon_utils import LOG +LOG.warning(f"This reference is deprecated. Import from neon_speech.stt directly!") +# TODO: Deprecate in neon_core 22.04 diff --git a/neon_core/tts/__init__.py b/neon_core/tts/__init__.py index 478cb23b6..abc77bfce 100644 --- a/neon_core/tts/__init__.py +++ b/neon_core/tts/__init__.py @@ -24,3 +24,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from neon_audio.tts import TTS, PlaybackThread, TTSValidator, TTSFactory, load_tts_plugin +from neon_utils import LOG +LOG.warning(f"This reference is deprecated. Import from neon_audio.tts directly!") +# TODO: Deprecate in neon_core 22.04 diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index 049d7297e..aaf819ce9 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -27,7 +27,7 @@ from ovos_skills_manager.osm import OVOSSkillsManager from ovos_skills_manager.session import SESSION as requests, set_github_token, clear_github_token from neon_utils.configuration_utils import get_neon_skills_config -from neon_utils.log_utils import LOG +from neon_utils import LOG def install_skills_from_list(skills_to_install: list, config: dict = None): From 3b4259edbb6e5ab2e53b802f0e4a0b41990b45bd Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 24 Nov 2021 19:02:07 +0000 Subject: [PATCH 04/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 04eae1c82..4d1541ba3 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a0" +__version__ = "21.10.2a1" From 5972551614633fbbb242e37bad4cbdd0016b9126 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 1 Dec 2021 16:03:12 -0800 Subject: [PATCH 05/70] Update diagnostic_utils and tests to handle json dumped dicts (#176) * Update diagnostic_utils and tests to handle json dumped dicts * Increase timeout for setup tests * Debug util test failures * Write test config changes to resolve test failures * Add line to debug skill installation errors * revert unit test changes --- .github/workflows/setup_tests.yml | 2 +- neon_core/util/diagnostic_utils.py | 5 +++-- test/test_diagnostic_utils.py | 9 +++++---- test/test_skill_utils.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/setup_tests.yml b/.github/workflows/setup_tests.yml index 2e15a5adb..bbc85a474 100644 --- a/.github/workflows/setup_tests.yml +++ b/.github/workflows/setup_tests.yml @@ -50,7 +50,7 @@ jobs: matrix: python-version: [ 3.8 ] runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@v2 - name: Set up python ${{ matrix.python-version }} diff --git a/neon_core/util/diagnostic_utils.py b/neon_core/util/diagnostic_utils.py index 5d4164e64..396e2a936 100644 --- a/neon_core/util/diagnostic_utils.py +++ b/neon_core/util/diagnostic_utils.py @@ -61,6 +61,7 @@ def send_diagnostics(allow_logs=True, allow_transcripts=True, allow_config=True) if allow_logs: logs = dict() try: + LOG.info(f"Reading logs from: {logs_dir}") for log in glob.glob(f'{logs_dir}/*.log'): if os.path.basename(log) == "start.log": pass @@ -92,8 +93,8 @@ def send_diagnostics(allow_logs=True, allow_transcripts=True, allow_config=True) data = {"host": socket.gethostname(), "startup": startup, - "configurations": json.dumps(configs), - "logs": json.dumps(logs), + "configurations": json.dumps(configs) if configs else None, + "logs": json.dumps(logs) if logs else None, "transcripts": transcripts} report_metric("diagnostics", **data) return data diff --git a/test/test_diagnostic_utils.py b/test/test_diagnostic_utils.py index a74c0ce5b..64aa77898 100644 --- a/test/test_diagnostic_utils.py +++ b/test/test_diagnostic_utils.py @@ -50,6 +50,7 @@ def setUpClass(cls) -> None: local_config["dirVars"]["docsDir"] = test_dir local_config["dirVars"]["logsDir"] = test_dir local_config["dirVars"]["diagsDir"] = test_dir + local_config.write_changes() @classmethod def tearDownClass(cls) -> None: @@ -70,8 +71,8 @@ def test_send_diagnostics_default(self): data = args.kwargs self.assertIsInstance(data, dict) self.assertIsInstance(data["host"], str) - self.assertIsInstance(data["configurations"], dict) - self.assertIsInstance(data["logs"], dict) + self.assertIsInstance(data["configurations"], str) + self.assertIsInstance(data["logs"], str) self.assertIsInstance(data["transcripts"], str) def test_send_diagnostics_no_extras(self): @@ -97,7 +98,7 @@ def test_send_diagnostics_allow_logs(self): self.assertIsInstance(data, dict) self.assertIsInstance(data["host"], str) self.assertIsNone(data["configurations"]) - self.assertIsInstance(data["logs"], dict) + self.assertIsInstance(data["logs"], str) self.assertIsNone(data["transcripts"]) def test_send_diagnostics_allow_transcripts(self): @@ -122,7 +123,7 @@ def test_send_diagnostics_allow_config(self): data = args.kwargs self.assertIsInstance(data, dict) self.assertIsInstance(data["host"], str) - self.assertIsInstance(data["configurations"], dict) + self.assertIsInstance(data["configurations"], str) self.assertIsNone(data["logs"]) self.assertIsNone(data["transcripts"]) diff --git a/test/test_skill_utils.py b/test/test_skill_utils.py index 014377dae..e85524c45 100644 --- a/test/test_skill_utils.py +++ b/test/test_skill_utils.py @@ -83,7 +83,7 @@ def test_install_skills_from_list_with_auth(self): def test_install_skills_default(self): install_skills_default(SKILL_CONFIG) skill_dirs = [d for d in os.listdir(SKILL_DIR) if os.path.isdir(os.path.join(SKILL_DIR, d))] - self.assertEqual(len(skill_dirs), len(get_remote_entries(SKILL_CONFIG["default_skills"]))) + self.assertEqual(len(skill_dirs), len(get_remote_entries(SKILL_CONFIG["default_skills"])), f"{skill_dirs}\n\n{get_remote_entries(SKILL_CONFIG['default_skills'])}") if __name__ == '__main__': From 8786f485675cc75faaa7968df5093b97a41ceeee Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 2 Dec 2021 00:03:32 +0000 Subject: [PATCH 06/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 4d1541ba3..704d904ed 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a1" +__version__ = "21.10.2a2" From 716dbff87884d52a3ed6957ab95c6d18bf3fe570 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 10 Dec 2021 10:15:51 -0800 Subject: [PATCH 07/70] Remove display module and add optional dependency (#179) --- neon_core/display/__init__.py | 348 ------------------ neon_core/display/__main__.py | 57 --- neon_core/display/services/__init__.py | 222 ----------- .../services/opencv_backend/__init__.py | 104 ------ requirements/vision.txt | 6 +- 5 files changed, 1 insertion(+), 736 deletions(-) delete mode 100644 neon_core/display/__init__.py delete mode 100644 neon_core/display/__main__.py delete mode 100644 neon_core/display/services/__init__.py delete mode 100644 neon_core/display/services/opencv_backend/__init__.py diff --git a/neon_core/display/__init__.py b/neon_core/display/__init__.py deleted file mode 100644 index a479388cc..000000000 --- a/neon_core/display/__init__.py +++ /dev/null @@ -1,348 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import imp -import sys -from os import listdir -from os.path import abspath, dirname, basename, isdir, join -from threading import Lock - -from neon_core.configuration import Configuration -from mycroft.messagebus.message import Message -from mycroft.util.log import LOG - - -MAINMODULE = '__init__' -sys.path.append(abspath(dirname(__file__))) - - -def create_service_descriptor(service_folder): - """Prepares a descriptor that can be used together with imp. - - Args: - service_folder: folder that shall be imported. - - Returns: - Dict with import information - """ - info = imp.find_module(MAINMODULE, [service_folder]) - return {"name": basename(service_folder), "info": info} - - -def get_services(services_folder): - """ - Load and initialize services from all subfolders. - - Args: - services_folder: base folder to look for services in. - - Returns: - Sorted list of display services. - """ - LOG.info("Loading services from " + services_folder) - services = [] - possible_services = listdir(services_folder) - for i in possible_services: - location = join(services_folder, i) - if (isdir(location) and - not MAINMODULE + ".py" in listdir(location)): - for j in listdir(location): - name = join(location, j) - if (not isdir(name) or - not MAINMODULE + ".py" in listdir(name)): - continue - try: - services.append(create_service_descriptor(name)) - except Exception: - LOG.error('Failed to create service from ' + name, - exc_info=True) - if (not isdir(location) or - not MAINMODULE + ".py" in listdir(location)): - continue - try: - services.append(create_service_descriptor(location)) - except Exception: - LOG.error('Failed to create service from ' + location, - exc_info=True) - return sorted(services, key=lambda p: p.get('name')) - - -def load_services(config, bus, path=None): - """ - Search though the service directory and load any services. - - Args: - config: configuration dict for the display backends. - bus: Mycroft messagebus - - Returns: - List of started services. - """ - if path is None: - path = dirname(abspath(__file__)) + '/services/' - service_directories = get_services(path) - services = [] - for descriptor in service_directories: - LOG.info('Loading ' + descriptor['name']) - try: - service_module = imp.load_module(descriptor["name"] + MAINMODULE, - *descriptor["info"]) - except Exception as e: - LOG.error('Failed to import module ' + descriptor['name'] + '\n' + - repr(e)) - continue - - if hasattr(service_module, 'load_service'): - try: - s = service_module.load_service(config, bus) - services += s - except Exception as e: - LOG.error('Failed to load service. ' + repr(e)) - return services - - -class DisplayService: - """ Display Service class. - Handles display of images and selecting proper backend for - the image to be displayed. - """ - - def __init__(self, bus): - """ - Args: - bus: Mycroft messagebus - """ - self.bus = bus - self.config = Configuration.get().get("Display") - self.service_lock = Lock() - - self.default = None - self.services = [] - self.current = None - - bus.once('open', self.load_services_callback) - - def load_services_callback(self): - """ - Main callback function for loading services. Sets up the globals - service and default and registers the event handlers for the - subsystem. - """ - - self.services = load_services(self.config, self.bus) - - # Register end of picture callback - for s in self.services: - s.set_display_start_callback(self.display_start) - - # Find default backend - default_name = self.config.get('default-backend', '') - LOG.info('Finding default backend...') - for s in self.services: - if s.name == default_name: - self.default = s - LOG.info('Found ' + self.default.name) - break - else: - self.default = None - LOG.info('no default found') - - # Setup event handlers - self.bus.on('mycroft.display.service.display', self._display) - self.bus.on('mycroft.display.service.queue', self._queue) - self.bus.on('mycroft.display.service.stop', self._stop) - self.bus.on('mycroft.display.service.clear', self._clear) - self.bus.on('mycroft.display.service.close', self._close) - self.bus.on('mycroft.display.service.reset', self._reset) - self.bus.on('mycroft.display.service.next', self._next) - self.bus.on('mycroft.display.service.prev', self._prev) - self.bus.on('mycroft.display.service.height', self._set_height) - self.bus.on('mycroft.display.service.width', self._set_width) - self.bus.on('mycroft.display.service.fullscreen', self._set_fullscreen) - self.bus.on('mycroft.display.service.picture_info', self._picture_info) - self.bus.on('mycroft.display.service.list_backends', self._list_backends) - - def get_prefered(self, utterance=""): - # Find if the user wants to use a specific backend - for s in self.services: - if s.name in utterance: - prefered_service = s - LOG.debug(s.name + ' would be prefered') - break - else: - prefered_service = None - return prefered_service - - def display_start(self, picture): - """ - Callback method called from the services to indicate start of - playback of a picture. - """ - self.bus.emit(Message('mycroft.display.displaying_picture', - data={'picture': picture})) - - def _set_fullscreen(self, message=None): - value = message.data["value"] - if self.current: - self.current.change_fullscreen(value) - - def _set_height(self, message=None): - value = message.data["value"] - if self.current: - self.current.set_height(value) - - def _set_width(self, message=None): - value = message.data["value"] - if self.current: - self.current.set_width(value) - - def _close(self, message=None): - if self.current: - self.current.close() - - def _clear(self, message=None): - if self.current: - self.current.clear() - - def _reset(self, message=None): - if self.current: - self.current.reset() - else: - LOG.error("No active display to reset") - - def _next(self, message=None): - if self.current: - self.current.next() - - def _prev(self, message=None): - if self.current: - self.current.previous() - - def _stop(self, message=None): - LOG.debug('stopping display services') - with self.service_lock: - if self.current: - name = self.current.name - if self.current.stop(): - self.bus.emit(Message("mycroft.stop.handled", - {"by": "display:" + name})) - - self.current = None - - def _queue(self, message): - if self.current: - pictures = message.data['pictures'] - self.current.add_pictures(pictures) - else: - self._display(message) - - def _display(self, message): - """ - Handler for mycroft.display.service.play. Starts display of a - picturelist. Also determines if the user requested a special - service. - - Args: - message: message bus message, not used but required - """ - try: - pictures = message.data['pictures'] - prefered_service = self.get_prefered(message.data.get("utterance", "")) - - if isinstance(pictures[0], str): - uri_type = pictures[0].split(':')[0] - else: - uri_type = pictures[0][0].split(':')[0] - - # check if user requested a particular service - if prefered_service and uri_type in prefered_service.supported_uris(): - selected_service = prefered_service - # check if default supports the uri - elif self.default and uri_type in self.default.supported_uris(): - LOG.debug("Using default backend ({})".format(self.default.name)) - selected_service = self.default - else: # Check if any other service can play the media - LOG.debug("Searching the services") - for s in self.services: - if uri_type in s.supported_uris(): - LOG.debug("Service {} supports URI {}".format(s, uri_type)) - selected_service = s - break - else: - LOG.info('No service found for uri_type: ' + uri_type) - return - selected_service.clear_pictures() - selected_service.add_pictures(pictures) - selected_service.display() - self.current = selected_service - except Exception as e: - LOG.exception(e) - - def _picture_info(self, message): - """ - Returns picture info on the message bus. - - Args: - message: message bus message, not used but required - """ - if self.current: - picture_info = self.current.picture_info() - else: - picture_info = {} - self.bus.emit(Message('mycroft.display.service.picture_info_reply', - data=picture_info)) - - def _list_backends(self, message): - """ Return a dict of available backends. """ - data = {} - for s in self.services: - info = { - 'supported_uris': s.supported_uris(), - 'default': s == self.default - } - data[s.name] = info - self.bus.emit(message.response(data)) - - def shutdown(self): - for s in self.services: - try: - LOG.info('shutting down ' + s.name) - s.shutdown() - except Exception as e: - LOG.error('shutdown of ' + s.name + ' failed: ' + repr(e)) - - # remove listeners - self.bus.remove('mycroft.display.service.display', self._display) - self.bus.remove('mycroft.display.service.queue', self._queue) - self.bus.remove('mycroft.display.service.stop', self._stop) - self.bus.remove('mycroft.display.service.clear', self._clear) - self.bus.remove('mycroft.display.service.close', self._close) - self.bus.remove('mycroft.display.service.reset', self._reset) - self.bus.remove('mycroft.display.service.next', self._next) - self.bus.remove('mycroft.display.service.prev', self._prev) - self.bus.remove('mycroft.display.service.height', self._set_height) - self.bus.remove('mycroft.display.service.width', self._set_width) - self.bus.remove('mycroft.display.service.fullscreen', self._set_fullscreen) - self.bus.remove('mycroft.display.service.picture_info', self._picture_info) - self.bus.remove('mycroft.display.service.list_backends', self._list_backends) diff --git a/neon_core/display/__main__.py b/neon_core/display/__main__.py deleted file mode 100644 index 1c75f88b6..000000000 --- a/neon_core/display/__main__.py +++ /dev/null @@ -1,57 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - Mycroft display service. - - This handles display of pictures - - TODO videos - TODO multiple displays -""" -from neon_core.configuration import Configuration -from neon_core.messagebus import get_messagebus -from mycroft.util import reset_sigint_handler, wait_for_exit_signal, \ - create_echo_function, check_for_signal -from mycroft.util.log import LOG -from neon_core.display import DisplayService - - -def main(): - """ Main function. Run when file is invoked. """ - reset_sigint_handler() - check_for_signal("isSpeaking") - bus = get_messagebus() # Connect to the Mycroft Messagebus - Configuration.set_config_update_handlers(bus) - - LOG.info("Starting Display Services") - bus.on('message', create_echo_function('Display', ['mycroft.display.service'])) - - display = DisplayService(bus) # Connect audio service instance to message bus - - wait_for_exit_signal() - display.shutdown() - - -main() diff --git a/neon_core/display/services/__init__.py b/neon_core/display/services/__init__.py deleted file mode 100644 index 76b06131f..000000000 --- a/neon_core/display/services/__init__.py +++ /dev/null @@ -1,222 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from mycroft.util.log import LOG -from mycroft.util import resolve_resource_file -from time import sleep - - -class DisplayBackend: - """ - Base class for all display backend implementations. - Args: - config: configuration dict for the instance - bus: websocket object - """ - - def __init__(self, bus, config=None, name="dummy display"): - self.name = name - self.bus = bus - self.config = config - self.index = 0 - self._is_displaying = False - self.pictures = [] - self.width = 1600 - self.height = 900 - self.fullscreen = False - self._display_start_callback = None - self.default_picture = resolve_resource_file("ui/neon_logo.png") - - @staticmethod - def supported_uris(): - """ - Returns: list of supported uri types. - """ - return ['file'] #, 'http', 'https'] - - def handle_fullscreen(self, new_value, old_value): - # display was told to change fullscreen status - pass - - def handle_reset(self): - # display was told to reset to default state - # usually a logo - if self.default_picture is not None: - self.handle_display(self.default_picture) - - def handle_stop(self): - # display was told to stop displaying - self.handle_reset() - - def handle_close(self): - # display was told to close window - pass - - def handle_display(self, picture): - # display was told to display picture - pass - - def handle_clear(self): - # display was told to clear - # usually a black image - pass - - def handle_height_change(self, new_value, old_value): - # change display height in pixels - pass - - def handle_width_change(self, new_value, old_value): - # change display width in pixels - pass - - def display(self): - """ - Display self.index in Pictures List of paths - """ - if len(self.pictures): - pic = self.pictures[self.index] - self.handle_display(pic) - else: - LOG.error("Nothing to display") - - def clear_pictures(self): - self.pictures = [] - self.index = 0 - - def add_pictures(self, picture_list): - """ - add pics - """ - self.pictures.extend(picture_list) - - def reset(self): - """ - Reset Display. - """ - self.index = 0 - self.pictures = [] - self.handle_reset() - - def clear(self): - """ - Clear Display. - """ - self.handle_clear() - - def next(self): - """ - Skip to next pic in playlist. - """ - self.index += 1 - if self.index > len(self.pictures): - self.index = 0 - self.display() - - def previous(self): - """ - Skip to previous pic in playlist. - """ - self.index -= 1 - if self.index > 0: - self.index = len(self.pictures) - self.display() - - def lock(self): - """ - Set Lock Flag so nothing else can display - """ - pass - - def unlock(self): - """ - Unset Lock Flag so nothing else can display - """ - pass - - def change_index(self, index): - """ - Change picture index - """ - self.index = index - self.display() - - def change_fullscreen(self, value=True): - """ - toogle fullscreen - """ - old = self.fullscreen - self.fullscreen = value - self.handle_fullscreen(value, old) - - def change_height(self, value=900): - """ - change display height - """ - old = self.height - self.height = int(value) - self.handle_height_change(int(value), old) - - def change_width(self, value=1600): - """ - change display width - """ - old = self.width - self.width = int(value) - self.handle_width_change(int(value), old) - - def stop(self): - """ - Stop display. - """ - self._is_displaying = False - self.handle_stop() - - def close(self): - self.stop() - sleep(0.5) - self.handle_close() - - def shutdown(self): - """ Perform clean shutdown """ - self.stop() - self.close() - - def set_display_start_callback(self, callback_func): - """ - Register callback on display start, should be called as each - picture in picture list is displayed - """ - self._display_start_callback = callback_func - - def picture_info(self): - ret = {} - ret['artist'] = 'unknown' - ret['path'] = None - if len(self.pictures): - ret['path'] = self.pictures[self.index] - else: - ret['path'] = self.default_picture - ret["is_displaying"] = self._is_displaying - return ret - diff --git a/neon_core/display/services/opencv_backend/__init__.py b/neon_core/display/services/opencv_backend/__init__.py deleted file mode 100644 index 0efccdb45..000000000 --- a/neon_core/display/services/opencv_backend/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from mycroft.display.services import DisplayBackend -from mycroft.util.log import LOG -from mycroft.messagebus import Message -import imutils -import cv2 -import numpy as np - - -class OpenCVService(DisplayBackend): - """ - Display backend for opencv package. - """ - def __init__(self, bus, config, name="opencv"): - super().__init__(bus, config, name) - self.bus.on("opencv.display", self._display) - self.current_image = None - - def _display(self, message=None): - self._prepare_window() - self._is_displaying = True - cv2.imshow("OpenCV Display", self.current_image) - cv2.waitKey(0) - - def _prepare_window(self): - if self._is_displaying: - cv2.destroyWindow("OpenCV Display") - - cv2.namedWindow("OpenCV Display", cv2.WND_PROP_FULLSCREEN) - if self.fullscreen: - cv2.setWindowProperty("OpenCV Display", cv2.WND_PROP_FULLSCREEN, - cv2.WINDOW_FULLSCREEN) - else: - cv2.setWindowProperty("OpenCV Display", cv2.WND_PROP_FULLSCREEN, - not cv2.WINDOW_FULLSCREEN) - cv2.resizeWindow("OpenCV Display", self.width, self.height) - - def handle_display(self, picture): - LOG.info('Call OpenCVDisplay') - path = picture.replace("file://", "") - image = cv2.imread(path) - image = imutils.resize(image, self.width, self.height) - self.current_image = image - # NOTE message is needed because otherwise opencv will block - self.bus.emit(Message("opencv.display")) - - def handle_fullscreen(self, new_value, old_value): - # re-render - self._display() - - def handle_height_change(self, new_value, old_value): - # re-render - self._display() - - def handle_width_change(self, new_value, old_value): - # re-render - self._display() - - def handle_clear(self): - """ - Clear Display. - """ - # Create a black image - image = np.zeros((512, 512, 3), np.uint8) - if not self.fullscreen: - image = imutils.resize(image, self.width, self.height) - self.current_image = image - self._display() - - def handle_close(self): - LOG.info('OpenCVDisplayClose') - cv2.destroyAllWindows() - self._is_displaying = False - - -def load_service(base_config, bus): - backends = base_config.get('backends', []) - services = [(b, backends[b]) for b in backends - if backends[b]['type'] == 'opencv'] - instances = [OpenCVService(bus, s[1], s[0]) for s in services] - return instances diff --git a/requirements/vision.txt b/requirements/vision.txt index c65e7a17c..2a9b80fca 100644 --- a/requirements/vision.txt +++ b/requirements/vision.txt @@ -1,5 +1 @@ -# vision stuff -numpy -imutils -opencv-python -opencv-contrib-python \ No newline at end of file +neon_display \ No newline at end of file From e55ea1ad01c3dc2b70e6b864f9af54ed5583ccfa Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 10 Dec 2021 18:16:10 +0000 Subject: [PATCH 08/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 704d904ed..0bbea94e9 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a2" +__version__ = "21.10.2a3" From 38a3543872311a805143d35b80c578a3ded43799 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 13 Dec 2021 11:56:41 -0800 Subject: [PATCH 09/70] Extract gui module to neon_gui (#181) --- neon_core/gui/__init__.py | 26 -- neon_core/gui/__main__.py | 38 -- neon_core/gui/gui.py | 742 -------------------------------- neon_core/gui/resting_screen.py | 229 ---------- neon_core/gui/service.py | 183 -------- neon_core/launcher.py | 2 +- neon_core/run_neon.py | 4 +- requirements/requirements.txt | 2 + 8 files changed, 5 insertions(+), 1221 deletions(-) delete mode 100644 neon_core/gui/__init__.py delete mode 100644 neon_core/gui/__main__.py delete mode 100644 neon_core/gui/gui.py delete mode 100644 neon_core/gui/resting_screen.py delete mode 100644 neon_core/gui/service.py diff --git a/neon_core/gui/__init__.py b/neon_core/gui/__init__.py deleted file mode 100644 index b1792f31c..000000000 --- a/neon_core/gui/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from mycroft.enclosure import * diff --git a/neon_core/gui/__main__.py b/neon_core/gui/__main__.py deleted file mode 100644 index 3d060374a..000000000 --- a/neon_core/gui/__main__.py +++ /dev/null @@ -1,38 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from neon_core.gui.service import NeonGUIService -from mycroft.util import wait_for_exit_signal - - -def main(): - gui = NeonGUIService(daemonic=True) - gui.start() - wait_for_exit_signal() - gui.shutdown() - - -if __name__ == "__main__": - main() diff --git a/neon_core/gui/gui.py b/neon_core/gui/gui.py deleted file mode 100644 index c8557e7e3..000000000 --- a/neon_core/gui/gui.py +++ /dev/null @@ -1,742 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from os.path import join -from collections import namedtuple -from threading import Lock -from ovos_utils import wait_for_exit_signal -from ovos_utils import resolve_resource_file -from neon_utils import LOG -from mycroft_bus_client import Message - -from neon_core.configuration import Configuration -from neon_core.messagebus import get_messagebus - - -class SkillGUI: - """SkillGUI - Interface to the Graphical User Interface - - Values set in this class are synced to the GUI, accessible within QML - via the built-in sessionData mechanism. For example, in Python you can - write in a skill: - self.gui['temp'] = 33 - self.gui.show_page('Weather.qml') - Then in the Weather.qml you'd access the temp via code such as: - text: sessionData.time - """ - - def __init__(self, skill): - self.__session_data = {} # synced to GUI for use by this skill's pages - self.page = None # the active GUI page (e.g. QML template) to show - self.skill = skill - self.on_gui_changed_callback = None - self.config = Configuration.get() - - @property - def remote_url(self): - """Returns configuration value for url of remote-server.""" - return self.config.get('remote-server') - - def build_message_type(self, event): - """Builds a message matching the output from the enclosure.""" - return '{}.{}'.format(self.skill.skill_id, event) - - def setup_default_handlers(self): - """Sets the handlers for the default messages.""" - msg_type = self.build_message_type('set') - self.skill.add_event(msg_type, self.gui_set) - - def register_handler(self, event, handler): - """Register a handler for GUI events. - - When using the triggerEvent method from Qt - triggerEvent("event", {"data": "cool"}) - - Arguments: - event (str): event to catch - handler: function to handle the event - """ - msg_type = self.build_message_type(event) - self.skill.add_event(msg_type, handler) - - def set_on_gui_changed(self, callback): - """Registers a callback function to run when a value is - changed from the GUI. - - Arguments: - callback: Function to call when a value is changed - """ - self.on_gui_changed_callback = callback - - def gui_set(self, message): - """Handler catching variable changes from the GUI. - - Arguments: - message: Messagebus message - """ - for key in message.data: - self[key] = message.data[key] - if self.on_gui_changed_callback: - self.on_gui_changed_callback() - - def __setitem__(self, key, value): - """Implements set part of dict-like behaviour with named keys.""" - self.__session_data[key] = value - - if self.page: - # emit notification (but not needed if page has not been shown yet) - data = self.__session_data.copy() - data.update({'__from': self.skill.skill_id}) - self.skill.bus.emit(Message("gui.value.set", data)) - - def __getitem__(self, key): - """Implements get part of dict-like behaviour with named keys.""" - return self.__session_data[key] - - def __contains__(self, key): - """Implements the "in" operation.""" - return self.__session_data.__contains__(key) - - def clear(self): - """Reset the value dictionary, and remove namespace from GUI.""" - self.__session_data = {} - self.page = None - self.skill.bus.emit(Message("gui.clear.namespace", - {"__from": self.skill.skill_id})) - - def send_event(self, event_name, params=None): - """Trigger a gui event. - - Arguments: - event_name (str): name of event to be triggered - params: json serializable object containing any parameters that - should be sent along with the request. - """ - params = params or {} - self.skill.bus.emit(Message("gui.event.send", - {"__from": self.skill.skill_id, - "event_name": event_name, - "params": params})) - - def show_page(self, name, override_idle=None): - """Begin showing the page in the GUI - - Arguments: - name (str): Name of page (e.g "mypage.qml") to display - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - self.show_pages([name], 0, override_idle) - - def show_pages(self, page_names, index=0, override_idle=None): - """Begin showing the list of pages in the GUI. - - Arguments: - page_names (list): List of page names (str) to display, such as - ["Weather.qml", "Forecast.qml", "Details.qml"] - index (int): Page number (0-based) to show initially. For the - above list a value of 1 would start on "Forecast.qml" - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - if not isinstance(page_names, list): - raise ValueError('page_names must be a list') - - if index > len(page_names): - raise ValueError('Default index is larger than page list length') - - self.page = page_names[index] - - # First sync any data... - data = self.__session_data.copy() - data.update({'__from': self.skill.skill_id}) - self.skill.bus.emit(Message("gui.value.set", data)) - - # Convert pages to full reference - page_urls = [] - for name in page_names: - if name.startswith("SYSTEM"): - page = resolve_resource_file(join('ui', name)) - else: - page = self.skill.find_resource(name, 'ui') - if page: - if self.config.get('remote'): - page_urls.append(self.remote_url + "/" + page) - else: - page_urls.append("file://" + page) - else: - raise FileNotFoundError("Unable to find page: {}".format(name)) - - self.skill.bus.emit(Message("gui.page.show", - {"page": page_urls, - "index": index, - "__from": self.skill.skill_id, - "__idle": override_idle})) - - def remove_page(self, page): - """Remove a single page from the GUI. - - Arguments: - page (str): Page to remove from the GUI - """ - return self.remove_pages([page]) - - def remove_pages(self, page_names): - """Remove a list of pages in the GUI. - - Arguments: - page_names (list): List of page names (str) to display, such as - ["Weather.qml", "Forecast.qml", "Other.qml"] - """ - if not isinstance(page_names, list): - raise ValueError('page_names must be a list') - - # Convert pages to full reference - page_urls = [] - for name in page_names: - page = self.skill.find_resource(name, 'ui') - if page: - page_urls.append("file://" + page) - else: - raise FileNotFoundError("Unable to find page: {}".format(name)) - - self.skill.bus.emit(Message("gui.page.delete", - {"page": page_urls, - "__from": self.skill.skill_id})) - - def show_text(self, text, title=None, override_idle=None): - """Display a GUI page for viewing simple text. - - Arguments: - text (str): Main text content. It will auto-paginate - title (str): A title to display above the text content. - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - self.clear() - self["text"] = text - self["title"] = title - self.show_page("SYSTEM_TextFrame.qml", override_idle) - - def show_image(self, url, caption=None, - title=None, fill=None, - override_idle=None): - """Display a GUI page for viewing an image. - - Arguments: - url (str): Pointer to the image - caption (str): A caption to show under the image - title (str): A title to display above the image content - fill (str): Fill type supports 'PreserveAspectFit', - 'PreserveAspectCrop', 'Stretch' - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - self.clear() - self["image"] = url - self["title"] = title - self["caption"] = caption - self["fill"] = fill - self.show_page("SYSTEM_ImageFrame.qml", override_idle) - - def show_html(self, html, resource_url=None, override_idle=None): - """Display an HTML page in the GUI. - - Arguments: - html (str): HTML text to display - resource_url (str): Pointer to HTML resources - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - self.clear() - self["html"] = html - self["resourceLocation"] = resource_url - self.show_page("SYSTEM_HtmlFrame.qml", override_idle) - - def show_url(self, url, override_idle=None): - """Display an HTML page in the GUI. - - Arguments: - url (str): URL to render - override_idle (boolean, int): - True: Takes over the resting page indefinitely - (int): Delays resting page for the specified number of - seconds. - """ - self.clear() - self["url"] = url - self.show_page("SYSTEM_UrlFrame.qml", override_idle) - - def shutdown(self): - """Shutdown gui interface. - - Clear pages loaded through this interface and remove the skill - reference to make ref counting warning more precise. - """ - self.clear() - self.skill = None - - -Namespace = namedtuple('Namespace', ['name', 'pages']) -write_lock = Lock() -namespace_lock = Lock() - -RESERVED_KEYS = ['__from', '__idle'] - - -def _get_page_data(message): - """ Extract page related data from a message. - - Args: - message: messagebus message object - Returns: - tuple (page, namespace, index) - Raises: - ValueError if value is missing. - """ - data = message.data - # Note: 'page' can be either a string or a list of strings - if 'page' not in data: - raise ValueError("Page missing in data") - if 'index' in data: - index = data['index'] - else: - index = 0 - page = data.get("page", "") - namespace = data.get("__from", "") - return page, namespace, index - - -class GUIManager: - def __init__(self, bus=None): - config = Configuration.get() - self.lang = config['lang'] - self.config = config - - # Establish Enclosure's websocket connection to the messagebus - self.bus = bus or get_messagebus() - - # This datastore holds the data associated with the GUI provider. Data - # is stored in Namespaces, so you can have: - # self.datastore["namespace"]["name"] = value - # Typically the namespace is a meaningless identifier, but there is a - # special "SYSTEM" namespace. - self.datastore = {} - - # self.loaded is a list, each element consists of a namespace named - # tuple. - # The namespace namedtuple has the properties "name" and "pages" - # The name contains the namespace name as a string and pages is a - # mutable list of loaded pages. - # - # [Namespace name, [List of loaded qml pages]] - # [ - # ["SKILL_NAME", ["page1.qml, "page2.qml", ... , "pageN.qml"] - # [...] - # ] - self.loaded = [] # list of lists in order. - self.explicit_move = True # Set to true to send reorder commands - - # Listen for new GUI clients to announce themselves on the main bus - self.active_namespaces = [] - self.bus.on("mycroft.gui.connected", self.on_gui_client_connected) - self.register_gui_handlers() - - # First send any data: - self.bus.on("gui.value.set", self.on_gui_set_value) - self.bus.on("gui.page.show", self.on_gui_show_page) - self.bus.on("gui.page.delete", self.on_gui_delete_page) - self.bus.on("gui.clear.namespace", self.on_gui_delete_namespace) - self.bus.on("gui.event.send", self.on_gui_send_event) - self.bus.on("gui.status.request", self.handle_gui_status_request) - - def run(self): - try: - if not self.bus.started_running: - self.bus.run_forever() - else: - wait_for_exit_signal() - except Exception as e: - LOG.error("Error: {0}".format(e)) - self.stop() - - def stop(self): - pass - - ###################################################################### - # GUI client API - @property - def gui_connected(self): - from neon_core.gui.service import GUIWebsocketHandler - """Returns True if at least 1 gui is connected, else False""" - return len(GUIWebsocketHandler.clients) > 0 - - def handle_gui_status_request(self, message): - """Reply to gui status request, allows querying if a gui is - connected using the message bus""" - self.bus.emit(message.reply("gui.status.request.response", - {"connected": self.gui_connected})) - - @staticmethod - def send(msg_dict): - from neon_core.gui.service import GUIWebsocketHandler - """ Send to all registered GUIs. """ - for connection in GUIWebsocketHandler.clients: - try: - connection.send(msg_dict) - except Exception as e: - LOG.exception(repr(e)) - - def on_gui_send_event(self, message): - """ Send an event to the GUIs. """ - try: - data = {'type': 'mycroft.events.triggered', - 'namespace': message.data.get('__from'), - 'event_name': message.data.get('event_name'), - 'params': message.data.get('params')} - self.send(data) - except Exception as e: - LOG.error('Could not send event ({})'.format(repr(e))) - - def on_gui_set_value(self, message): - data = message.data - namespace = data.get("__from", "") - - # Pass these values on to the GUI renderers - for key in data: - if key not in RESERVED_KEYS: - try: - self.set(namespace, key, data[key]) - except Exception as e: - LOG.exception(repr(e)) - - def set(self, namespace, name, value): - """ Perform the send of the values to the connected GUIs. """ - if namespace not in self.datastore: - self.datastore[namespace] = {} - if self.datastore[namespace].get(name) != value: - self.datastore[namespace][name] = value - - # If the namespace is loaded send data to GUI - if namespace in [ns.name for ns in self.loaded]: - msg = {"type": "mycroft.session.set", - "namespace": namespace, - "data": {name: value}} - self.send(msg) - - def on_gui_delete_page(self, message): - """ Bus handler for removing pages. """ - page, namespace, _ = _get_page_data(message) - try: - with namespace_lock: - self.remove_pages(namespace, page) - except Exception as e: - LOG.exception(repr(e)) - - def on_gui_delete_namespace(self, message): - """ Bus handler for removing namespace. """ - try: - namespace = message.data['__from'] - with namespace_lock: - self.remove_namespace(namespace) - except Exception as e: - LOG.exception(repr(e)) - - def on_gui_show_page(self, message): - try: - page, namespace, index = _get_page_data(message) - # Pass the request to the GUI(s) to pull up a page template - with namespace_lock: - self.show(namespace, page, index) - except Exception as e: - LOG.exception(repr(e)) - - def __find_namespace(self, namespace): - for i, skill in enumerate(self.loaded): - if skill[0] == namespace: - return i - return None - - def __insert_pages(self, namespace, pages): - """ Insert pages into the namespace - - Args: - namespace (str): Namespace to add to - pages (list): Pages (str) to insert - """ - LOG.debug("Inserting new pages") - if not isinstance(pages, list): - raise ValueError('Argument must be list of pages') - - self.send({"type": "mycroft.gui.list.insert", - "namespace": namespace, - "position": len(self.loaded[0].pages), - "data": [{"url": p} for p in pages] - }) - # Insert the pages into local reprensentation as well. - updated = Namespace(self.loaded[0].name, self.loaded[0].pages + pages) - self.loaded[0] = updated - - def __remove_page(self, namespace, pos): - """ Delete page. - - Args: - namespace (str): Namespace to remove from - pos (int): Page position to remove - """ - LOG.debug("Deleting {} from {}".format(pos, namespace)) - self.send({"type": "mycroft.gui.list.remove", - "namespace": namespace, - "position": pos, - "items_number": 1 - }) - # Remove the page from the local reprensentation as well. - self.loaded[0].pages.pop(pos) - # Add a check to return any display to idle from position 0 - if pos == 0 and len(self.loaded[0].pages) == 0: - self.bus.emit(Message("mycroft.device.show.idle")) - - def __insert_new_namespace(self, namespace, pages): - """ Insert new namespace and pages. - - This first sends a message adding a new namespace at the - highest priority (position 0 in the namespace stack) - - Args: - namespace (str): The skill namespace to create - pages (list): Pages to insert (name matches QML) - """ - LOG.debug("Inserting new namespace") - self.send({"type": "mycroft.session.list.insert", - "namespace": "mycroft.system.active_skills", - "position": 0, - "data": [{"skill_id": namespace}] - }) - - # Load any already stored Data - data = self.datastore.get(namespace, {}) - for key in data: - msg = {"type": "mycroft.session.set", - "namespace": namespace, - "data": {key: data[key]}} - self.send(msg) - - LOG.debug("Inserting new page") - self.send({"type": "mycroft.gui.list.insert", - "namespace": namespace, - "position": 0, - "data": [{"url": p} for p in pages] - }) - # Make sure the local copy is updated - self.loaded.insert(0, Namespace(namespace, pages)) - - def __move_namespace(self, from_pos, to_pos): - """ Move an existing namespace to a new position in the stack. - - Args: - from_pos (int): Position in the stack to move from - to_pos (int): Position to move to - """ - LOG.debug("Activating existing namespace") - # Seems like the namespace is moved to the top automatically when - # a page change is done. Deactivating this for now. - if self.explicit_move: - LOG.debug("move {} to {}".format(from_pos, to_pos)) - self.send({"type": "mycroft.session.list.move", - "namespace": "mycroft.system.active_skills", - "from": from_pos, "to": to_pos, - "items_number": 1}) - # Move the local representation of the skill from current - # position to position 0. - self.loaded.insert(to_pos, self.loaded.pop(from_pos)) - - def __switch_page(self, namespace, pages): - """ Switch page to an already loaded page. - - Args: - pages (list): pages (str) to switch to - namespace (str): skill namespace - """ - try: - num = self.loaded[0].pages.index(pages[0]) - except Exception as e: - LOG.exception(repr(e)) - num = 0 - - LOG.debug('Switching to already loaded page at ' - 'index {} in namespace {}'.format(num, namespace)) - self.send({"type": "mycroft.events.triggered", - "namespace": namespace, - "event_name": "page_gained_focus", - "data": {"number": num}}) - - def show(self, namespace, page, index): - """ Show a page and load it as needed. - - Args: - page (str or list): page(s) to show - namespace (str): skill namespace - index (int): ??? TODO: Unused in code ??? - - TODO: - Update sync to match. - - Separate into multiple functions/methods - """ - - LOG.debug("GUIConnection activating: " + namespace) - pages = page if isinstance(page, list) else [page] - - # find namespace among loaded namespaces - try: - index = self.__find_namespace(namespace) - if index is None: - # This namespace doesn't exist, insert them first so they're - # shown. - self.__insert_new_namespace(namespace, pages) - return - else: # Namespace exists - if index > 0: - # Namespace is inactive, activate it by moving it to - # position 0 - self.__move_namespace(index, 0) - - # Find if any new pages needs to be inserted - new_pages = [p for p in pages if p not in self.loaded[0].pages] - if new_pages: - self.__insert_pages(namespace, new_pages) - else: - # No new pages, just switch - self.__switch_page(namespace, pages) - except Exception as e: - LOG.exception(repr(e)) - - def remove_namespace(self, namespace): - """ Remove namespace. - - Args: - namespace (str): namespace to remove - """ - index = self.__find_namespace(namespace) - if index is None: - return - else: - LOG.debug("Removing namespace {} at {}".format(namespace, index)) - self.send({"type": "mycroft.session.list.remove", - "namespace": "mycroft.system.active_skills", - "position": index, - "items_number": 1 - }) - # Remove namespace from loaded namespaces - self.loaded.pop(index) - - def remove_pages(self, namespace, pages): - """ Remove the listed pages from the provided namespace. - - Args: - namespace (str): The namespace to modify - pages (list): List of page names (str) to delete - """ - try: - index = self.__find_namespace(namespace) - if index is None: - return - else: - # Remove any pages that doesn't exist in the namespace - pages = [p for p in pages if p in self.loaded[index].pages] - # Make sure to remove pages from the back - indexes = [self.loaded[index].pages.index(p) for p in pages] - indexes = sorted(indexes) - indexes.reverse() - for page_index in indexes: - self.__remove_page(namespace, page_index) - except Exception as e: - LOG.exception(repr(e)) - - ###################################################################### - # GUI client socket - # - # The basic mechanism is: - # 1) GUI client announces itself on the main messagebus - # 2) Mycroft prepares a port for a socket connection to this GUI - # 3) The port is announced over the messagebus - # 4) The GUI connects on the socket - # 5) Connection persists for graphical interaction indefinitely - # - # If the connection is lost, it must be renegotiated and restarted. - def on_gui_client_connected(self, message): - # GUI has announced presence - LOG.info('GUI HAS ANNOUNCED!') - port = self.config["gui_websocket"]["base_port"] - LOG.debug("on_gui_client_connected") - gui_id = message.data.get("gui_id") - - LOG.debug("Heard announcement from gui_id: {}".format(gui_id)) - - # Announce connection, the GUI should connect on it soon - self.bus.emit(Message("mycroft.gui.port", - {"port": port, - "gui_id": gui_id})) - - def register_gui_handlers(self): - # TODO: Register handlers for standard (Mark 1) events - # self.bus.on('enclosure.eyes.on', self.on) - # self.bus.on('enclosure.eyes.off', self.off) - # self.bus.on('enclosure.eyes.blink', self.blink) - # self.bus.on('enclosure.eyes.narrow', self.narrow) - # self.bus.on('enclosure.eyes.look', self.look) - # self.bus.on('enclosure.eyes.color', self.color) - # self.bus.on('enclosure.eyes.level', self.brightness) - # self.bus.on('enclosure.eyes.volume', self.volume) - # self.bus.on('enclosure.eyes.spin', self.spin) - # self.bus.on('enclosure.eyes.timedspin', self.timed_spin) - # self.bus.on('enclosure.eyes.reset', self.reset) - # self.bus.on('enclosure.eyes.setpixel', self.set_pixel) - # self.bus.on('enclosure.eyes.fill', self.fill) - - # self.bus.on('enclosure.mouth.reset', self.reset) - # self.bus.on('enclosure.mouth.talk', self.talk) - # self.bus.on('enclosure.mouth.think', self.think) - # self.bus.on('enclosure.mouth.listen', self.listen) - # self.bus.on('enclosure.mouth.smile', self.smile) - # self.bus.on('enclosure.mouth.viseme', self.viseme) - # self.bus.on('enclosure.mouth.text', self.text) - # self.bus.on('enclosure.mouth.display', self.display) - # self.bus.on('enclosure.mouth.display_image', self.display_image) - # self.bus.on('enclosure.weather.display', self.display_weather) - - # self.bus.on('recognizer_loop:record_begin', self.mouth.listen) - # self.bus.on('recognizer_loop:record_end', self.mouth.reset) - # self.bus.on('recognizer_loop:audio_output_start', self.mouth.talk) - # self.bus.on('recognizer_loop:audio_output_end', self.mouth.reset) - pass diff --git a/neon_core/gui/resting_screen.py b/neon_core/gui/resting_screen.py deleted file mode 100644 index eb6749f1a..000000000 --- a/neon_core/gui/resting_screen.py +++ /dev/null @@ -1,229 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import time - -from threading import Lock -from neon_utils import LOG -from mycroft_bus_client import Message, MessageBusClient -from neon_utils.skills.mycroft_skill import MycroftSkill - - -def compare_origin(m1, m2): - origin1 = m1.data["__from"] if isinstance(m1, Message) else m1 - origin2 = m2.data["__from"] if isinstance(m2, Message) else m2 - return origin1 == origin2 - - -class RestingScreen: - """ - Implementation of resting screens. - This class handles registration and override of resting screens, - encapsulating the system. - """ - - def __init__(self, bus: MessageBusClient = None): - bus = bus or MessageBusClient() - if not bus.started_running: - bus.run_in_thread() - - skill = MycroftSkill() - skill.skill_id = "resting_screen.neon" - skill.bind(bus) - self.bus = skill.bus - self.gui = skill.gui - self.settings = {} - self.schedule_event = skill.schedule_event - self.cancel_scheduled_event = skill.cancel_scheduled_event - self.has_show_page = False # resets with each handler - self.override_animations = False - self.resting_screen = None - - self.screens = {} - self.override_idle = None - self.next = 0 # Next time the idle screen should trigger - self.lock = Lock() - self.override_set_time = time.monotonic() - - # Preselect Time and Date as resting screen - self.gui["selected"] = self.settings.get("selected", "OVOSHomescreen") - self.gui.set_on_gui_changed(self.save) - self._init_listeners() - self.collect() - - def _init_listeners(self): - self.bus.on("mycroft.mark2.register_idle", self.on_register) - self.bus.on("mycroft.mark2.reset_idle", self.restore) - self.bus.on("mycroft.device.show.idle", self.show) - self.bus.on("gui.page.show", self.on_gui_page_show) - self.bus.on("gui.page_interaction", self.on_gui_page_interaction) - - self.gui.register_handler("mycroft.device.show.idle", self.show) - self.gui.register_handler("mycroft.device.set.idle", self.set) - - def on_register(self, message): - """Handler for catching incoming idle screens.""" - if "name" in message.data and "id" in message.data: - self.screens[message.data["name"]] = message.data["id"] - LOG.info("Registered {}".format(message.data["name"])) - else: - LOG.error("Malformed idle screen registration received") - - def save(self): - """Handler to be called if the settings are changed by the GUI. - Stores the selected idle screen. - """ - LOG.debug("Saving resting screen") - self.settings["selected"] = self.gui["selected"] - self.gui["selectedScreen"] = self.gui["selected"] - - def collect(self): - """Trigger collection and then show the resting screen.""" - self.bus.emit(Message("mycroft.mark2.collect_idle")) - time.sleep(1) - self.show() - - def set(self, message): - """Set selected idle screen from message.""" - self.gui["selected"] = message.data["selected"] - self.save() - - def show(self, _=None): - """Show the idle screen or return to the skill that's overriding idle.""" - LOG.debug("Showing idle screen") - screen = None - if self.override_idle: - LOG.debug("Returning to override idle screen") - # Restore the page overriding idle instead of the normal idle - self.bus.emit(self.override_idle[0]) - elif len(self.screens) > 0 and "selected" in self.gui: - # TODO remove hard coded value - LOG.info("Showing Idle screen for " "{}".format(self.gui["selected"])) - screen = self.screens.get(self.gui["selected"]) - - LOG.info(screen) - if screen: - self.bus.emit(Message("{}.idle".format(screen))) - - def restore(self, _=None): - """Remove any override and show the selected resting screen.""" - if self.override_idle and time.monotonic() - self.override_idle[1] > 2: - self.override_idle = None - self.show() - - def stop(self): - if time.monotonic() > self.override_set_time + 7: - self.restore() - - def override(self, message=None): - """Override the resting screen. - Arguments: - message: Optional message to use for to restore - the expected override screen after - another screen has been displayed. - """ - self.override_set_time = time.monotonic() - if message: - self.override_idle = (message, time.monotonic()) - - def cancel_override(self): - """Remove the override screen.""" - self.override_idle = None - - def on_gui_page_interaction(self, _): - """ Reset idle timer to 30 seconds when page is flipped. """ - LOG.info("Resetting idle counter to 30 seconds") - self.start_idle_event(30) - - def on_gui_page_show(self, message): - # Some skill other than the handler is showing a page - self.has_show_page = True - - # If a skill overrides the animations do not show any - override_animations = message.data.get("__animations", False) - if override_animations: - # Disable animations - LOG.info("Disabling all animations for page") - self.override_animations = True - else: - LOG.info("Displaying all animations for page") - self.override_animations = False - - # If a skill overrides the idle do not switch page - override_idle = message.data.get("__idle") - if override_idle is True: - # Disable idle screen - LOG.info("Cancelling Idle screen") - self.cancel_idle_event() - self.override(message) - elif isinstance(override_idle, int) and override_idle is not False: - LOG.info( - "Overriding idle timer to" " {} seconds".format(override_idle) - ) - self.override(None) - self.start_idle_event(override_idle) - elif message.data["page"] and not message.data["page"][0].endswith( - "idle.qml" - ): - # Check if the show_page deactivates a previous idle override - # This is only possible if the page is from the same skill - LOG.info("Cancelling idle override") - if override_idle is False and compare_origin( - message, self.override_idle[0] - ): - # Remove the idle override page if override is set to false - self.cancel_override() - # Set default idle screen timer - self.start_idle_event(30) - - def cancel_idle_event(self): - """Cancel the event monitoring current system idle time.""" - self.next = 0 - self.cancel_scheduled_event("IdleCheck") - - def start_idle_event(self, offset=60, weak=False): - """Start an event for showing the idle screen. - Arguments: - offset: How long until the idle screen should be shown - weak: set to true if the time should be able to be overridden - """ - with self.lock: - if time.monotonic() + offset < self.next: - LOG.info("No update, before next time") - return - - LOG.info("Starting idle event") - try: - if not weak: - self.next = time.monotonic() + offset - # Clear any existing checker - self.cancel_scheduled_event("IdleCheck") - time.sleep(0.5) - self.schedule_event( - self.show, int(offset), name="IdleCheck" - ) - LOG.info("Showing idle screen in " "{} seconds".format(offset)) - except Exception as e: - LOG.exception(repr(e)) diff --git a/neon_core/gui/service.py b/neon_core/gui/service.py deleted file mode 100644 index 4b8b0d1e9..000000000 --- a/neon_core/gui/service.py +++ /dev/null @@ -1,183 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import asyncio -import json -import sys -import tornado.options -import tornado.web as web - -from tornado import ioloop -from tornado.websocket import WebSocketHandler -from threading import Thread -from typing import Optional, Awaitable -from mycroft_bus_client import Message -from neon_utils import LOG - -from neon_core.configuration import Configuration -from neon_core.gui.gui import GUIManager, write_lock -from neon_core.gui.resting_screen import RestingScreen - -########################################################################## -# GUIConnection -########################################################################## - -gui_app_settings = { - 'debug': True -} - - -class GUIWebsocketHandler(WebSocketHandler): - """The socket pipeline between the GUI and Mycroft.""" - clients = [] - - def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: - pass - - def open(self): - GUIWebsocketHandler.clients.append(self) - LOG.info('New Connection opened!') - self.synchronize() - - def on_close(self): - LOG.info('Closing {}'.format(id(self))) - GUIWebsocketHandler.clients.remove(self) - - def synchronize(self): - """ Upload namespaces, pages and data to the last connected. """ - namespace_pos = 0 - gui_service = self.application.gui_service - - for namespace, pages in gui_service.loaded: - LOG.info('Sync {}'.format(namespace)) - # Insert namespace - self.send({"type": "mycroft.session.list.insert", - "namespace": "mycroft.system.active_skills", - "position": namespace_pos, - "data": [{"skill_id": namespace}] - }) - # Insert pages - self.send({"type": "mycroft.gui.list.insert", - "namespace": namespace, - "position": 0, - "data": [{"url": p} for p in pages] - }) - # Insert data - data = gui_service.datastore.get(namespace, {}) - for key in data: - self.send({"type": "mycroft.session.set", - "namespace": namespace, - "data": {key: data[key]} - }) - namespace_pos += 1 - - def on_message(self, message): - LOG.info("Received: {}".format(message)) - msg = json.loads(message) - if (msg.get('type') == "mycroft.events.triggered" and - (msg.get('event_name') == 'page_gained_focus' or - msg.get('event_name') == 'system.gui.user.interaction')): - # System event, a page was changed - msg_type = 'gui.page_interaction' - msg_data = {'namespace': msg['namespace'], - 'page_number': msg['parameters'].get('number'), - 'skill_id': msg['parameters'].get('skillId')} - elif msg.get('type') == "mycroft.events.triggered": - # A normal event was triggered - msg_type = '{}.{}'.format(msg['namespace'], msg['event_name']) - msg_data = msg['parameters'] - - elif msg.get('type') == 'mycroft.session.set': - # A value was changed send it back to the skill - msg_type = '{}.{}'.format(msg['namespace'], 'set') - msg_data = msg['data'] - else: - LOG.error(f"Unhandled message type: {msg.get('type')}") - return - message = Message(msg_type, msg_data) - LOG.info('Forwarding to bus...') - self.application.gui_service.bus.emit(message) - LOG.info('Done!') - - def write_message(self, *arg, **kwarg): - """Wraps WebSocketHandler.write_message() with a lock. """ - try: - asyncio.get_event_loop() - except RuntimeError: - asyncio.set_event_loop(asyncio.new_event_loop()) - - with write_lock: - super().write_message(*arg, **kwarg) - - def send(self, data): - """Send the given data across the socket as JSON - - Args: - data (dict): Data to transmit - """ - s = json.dumps(data) - LOG.info('Sending {}'.format(s)) - self.write_message(s) - - def check_origin(self, origin): - """Disable origin check to make js connections work.""" - return True - - -class NeonGUIService(Thread): - def __init__(self, config=None, debug=False, daemonic=False): - super().__init__() - config_core = Configuration.get() - self.config = config or config_core['gui_websocket'] - self.debug = debug - self.setDaemon(daemonic) - - def run(self): - LOG.info('Starting GUI service...') - self._init_gui() - self._init_tornado() - self._listen() - LOG.info('GUI service started!') - ioloop.IOLoop.instance().start() - - @staticmethod - def _init_tornado(): - # Disable all tornado logging so mycroft loglevel isn't overridden - tornado.options.parse_command_line(sys.argv + ['--logging=None']) - # get event loop for this thread - asyncio.set_event_loop(asyncio.new_event_loop()) - - def _init_gui(self): - self.gui_manager = GUIManager() - RestingScreen() - - def _listen(self): - routes = [(self.config['route'], GUIWebsocketHandler)] - application = web.Application(routes, debug=True) - application.gui_service = self.gui_manager - application.listen(self.config['base_port'], self.config['host']) - - def shutdown(self): - pass # TODO diff --git a/neon_core/launcher.py b/neon_core/launcher.py index 3e85abf23..d937dd3b5 100644 --- a/neon_core/launcher.py +++ b/neon_core/launcher.py @@ -26,7 +26,7 @@ from mycroft.util import wait_for_exit_signal, reset_sigint_handler from neon_core.messagebus.service import NeonBusService from neon_core.skills.service import NeonSkillService -from neon_core.gui.service import NeonGUIService +from neon_gui.service import NeonGUIService from time import sleep reset_sigint_handler() diff --git a/neon_core/run_neon.py b/neon_core/run_neon.py index 756f427c3..ac818228d 100644 --- a/neon_core/run_neon.py +++ b/neon_core/run_neon.py @@ -124,7 +124,7 @@ def _stop_all_core_processes(): for pid, cmdline in procs.items(): if cmdline and (any(pname in cmdline[-1] for pname in ("mycroft.messagebus.service", "neon_speech_client", "neon_audio_client", "neon_core.messagebus.service", - "neon_core.skills", "neon_core.gui", + "neon_core.skills", "neon_core.gui", "neon_gui_service", "neon_core_server", "neon_enclosure_client", "neon_core_client", "mycroft-gui-app", "NGI.utilities.gui", "run_neon.py") @@ -167,7 +167,7 @@ def start_neon(): _start_process("mycroft-gui-app") _start_process("neon_enclosure_client") # _start_process("neon_core_client") - _start_process(["python3", "-m", "neon_core.gui"]) + _start_process(["neon_gui_service"]) try: STOP_MODULES.wait() diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 903a71952..11c9c0063 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,6 +6,8 @@ mock_msm neon_speech~=0.3 neon_audio~=0.4 neon_enclosure~=0.1,>=0.1.2 +neon_gui +# TODO: Version spec GUI # utils neon-utils>=0.12.6 From 8a2e118ef9a6d59300ccf8997cd3a7599fb91fae Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Mon, 13 Dec 2021 19:57:02 +0000 Subject: [PATCH 10/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 0bbea94e9..93ea8ac82 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a3" +__version__ = "21.10.2a4" From c126fea050c1a786ed16c59183f8072804a4181f Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 17 Dec 2021 11:41:56 -0800 Subject: [PATCH 11/70] Extract Messagebus to separate module and update references (#178) * Extract Messagebus to separate module and update references * Fix dependency version * Move signal bus init into Skill Service class * Remove signal changes to submit via separate PR --- neon_core/__init__.py | 1 - neon_core/launcher.py | 5 +- neon_core/messagebus/__init__.py | 85 ++--------------------- neon_core/messagebus/service/__init__.py | 87 ------------------------ neon_core/messagebus/service/__main__.py | 48 ------------- neon_core/run_neon.py | 5 +- neon_core/skills/skill_store.py | 2 +- requirements/requirements.txt | 4 +- setup.py | 1 - test/test_run_modules.py | 10 ++- 10 files changed, 19 insertions(+), 229 deletions(-) delete mode 100644 neon_core/messagebus/service/__init__.py delete mode 100644 neon_core/messagebus/service/__main__.py diff --git a/neon_core/__init__.py b/neon_core/__init__.py index 3f5fe143b..aa939d7d7 100644 --- a/neon_core/__init__.py +++ b/neon_core/__init__.py @@ -119,7 +119,6 @@ def setup_ovos_config(): from neon_core.skills import NeonSkill, NeonFallbackSkill from neon_core.skills.intent_service import NeonIntentService - __all__ = ['NEON_ROOT_PATH', 'NeonIntentService', 'NeonSkill', diff --git a/neon_core/launcher.py b/neon_core/launcher.py index d937dd3b5..eb613efc2 100644 --- a/neon_core/launcher.py +++ b/neon_core/launcher.py @@ -24,11 +24,12 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from mycroft.lock import Lock from mycroft.util import wait_for_exit_signal, reset_sigint_handler -from neon_core.messagebus.service import NeonBusService +from neon_messagebus.service import NeonBusService from neon_core.skills.service import NeonSkillService from neon_gui.service import NeonGUIService from time import sleep + reset_sigint_handler() # Create PID file, prevent multiple instances of this service # TODO should also detect old services Locks @@ -37,6 +38,7 @@ # launch websocket listener bus = NeonBusService(daemonic=True) bus.start() +bus.started.wait(30) # launch GUI websocket listener gui = NeonGUIService(daemonic=True) @@ -51,3 +53,4 @@ skills.shutdown() gui.shutdown() bus.shutdown() +lock.delete() \ No newline at end of file diff --git a/neon_core/messagebus/__init__.py b/neon_core/messagebus/__init__.py index da6a54ca6..8daedd1bc 100644 --- a/neon_core/messagebus/__init__.py +++ b/neon_core/messagebus/__init__.py @@ -22,85 +22,8 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json -from threading import Event -from mycroft_bus_client import MessageBusClient, Message - -from mycroft.messagebus.service.event_handler import MessageBusEventHandler -from mycroft.util import create_daemon -from mycroft.messagebus.load_config import load_message_bus_config -from mycroft.util.json_helper import merge_dict - - -def get_messagebus(running=True): - config = load_message_bus_config() - bus = MessageBusClient(host=config.host, port=config.port, - route=config.route, ssl=config.ssl) - if running: - bus_connected = Event() - # Set the bus connected event when connection is established - bus.once('open', bus_connected.set) - create_daemon(bus.run_forever) - # Wait for connection - bus_connected.wait() - return bus - - -def send_message(message, data=None, context=None, bus=None): - auto_close = bus is None - bus = bus or get_messagebus() - if isinstance(message, str): - if isinstance(data, dict) or isinstance(context, dict): - message = Message(message, data, context) - else: - try: - message = json.loads(message) - except: - message = Message(message) - if isinstance(message, dict): - message = Message(message["type"], - message.get("data"), - message.get("context")) - if not isinstance(message, Message): - raise ValueError - bus.emit(message) - if auto_close: - bus.close() - - -def send_binary_data_message(binary_data, msg_type="mycroft.binary.data", - msg_data=None, msg_context=None, bus=None): - msg_data = msg_data or {} - msg = { - "type": msg_type, - "data": merge_dict(msg_data, {"binary": binary_data.hex()}), - "context": msg_context or None - } - send_message(msg, bus=bus) - - -def send_binary_file_message(filepath, msg_type="mycroft.binary.file", - msg_context=None, bus=None): - with open(filepath, 'rb') as f: - binary_data = f.read() - msg_data = {"path": filepath} - send_binary_data_message(binary_data, msg_type=msg_type, msg_data=msg_data, - msg_context=msg_context, bus=bus) - - -def decode_binary_message(message): - if isinstance(message, str): - try: # json string - message = json.loads(message) - binary_data = message.get("binary") or message["data"]["binary"] - except: # hex string - binary_data = message - elif isinstance(message, dict): - # data field or serialized message - binary_data = message.get("binary") or message["data"]["binary"] - else: - # message object - binary_data = message.data["binary"] - # decode hex string - return bytearray.fromhex(binary_data) +from neon_messagebus.util.message_utils import get_messagebus +from neon_utils import LOG +LOG.warning("This reference is deprecated; import from neon_messagebus directly") +# TODO: Deprecate in neon_core 22.04 diff --git a/neon_core/messagebus/service/__init__.py b/neon_core/messagebus/service/__init__.py deleted file mode 100644 index 27c3bcccf..000000000 --- a/neon_core/messagebus/service/__init__.py +++ /dev/null @@ -1,87 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" Message bus service for mycroft-core - -The message bus facilitates inter-process communication between mycroft-core -processes. It implements a websocket server so can also be used by external -systems to integrate with the Mycroft system. -""" -import asyncio -import sys -import tornado.options - -from os.path import expanduser, isfile -from threading import Thread -from tornado import web, ioloop -from neon_utils import LOG - -from mycroft.messagebus.load_config import load_message_bus_config -from mycroft.messagebus.service.event_handler import MessageBusEventHandler - - -class NeonBusService(Thread): - def __init__(self, config=None, debug=False, daemonic=False): - super().__init__() - self.config = config or load_message_bus_config() - self.debug = debug - self.setDaemon(daemonic) - - def run(self): - LOG.info('Starting message bus service...') - self._init_tornado() - self._listen() - LOG.info('Message bus service started!') - ioloop.IOLoop.instance().start() - - def _init_tornado(self): - # Disable all tornado logging so mycroft loglevel isn't overridden - tornado.options.parse_command_line(sys.argv + ['--logging=None']) - # get event loop for this thread - asyncio.set_event_loop(asyncio.new_event_loop()) - - def _listen(self): - routes = [(self.config.route, MessageBusEventHandler)] - application = web.Application(routes, debug=self.debug) - ssl_options = None - if self.config.ssl: - cert = expanduser(self.config.ssl_cert) - key = expanduser(self.config.ssl_key) - if not isfile(key) or not isfile(cert): - LOG.error( - "ssl keys dont exist, falling back to unsecured socket") - else: - LOG.info("using ssl key at " + key) - LOG.info("using ssl certificate at " + cert) - ssl_options = {"certfile": cert, "keyfile": key} - if ssl_options: - LOG.info("wss listener started") - application.listen(self.config.port, self.config.host, - ssl_options=ssl_options) - else: - LOG.info("ws listener started") - application.listen(self.config.port, self.config.host) - - def shutdown(self): - pass # TODO diff --git a/neon_core/messagebus/service/__main__.py b/neon_core/messagebus/service/__main__.py deleted file mode 100644 index 353817e8f..000000000 --- a/neon_core/messagebus/service/__main__.py +++ /dev/null @@ -1,48 +0,0 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" Message bus service for mycroft-core - -The message bus facilitates inter-process communication between mycroft-core -processes. It implements a websocket server so can also be used by external -systems to integrate with the Mycroft system. -""" -from mycroft.lock import Lock # creates/supports PID locking file -from mycroft.util import wait_for_exit_signal, reset_sigint_handler -from neon_core.messagebus.service import NeonBusService - - -def main(): - reset_sigint_handler() - # Create PID file, prevent multiple instances of this service - lock = Lock("bus") - # TODO debug should be False by default - service = NeonBusService(debug=True, daemonic=True) - service.start() - wait_for_exit_signal() - service.shutdown() - - -if __name__ == "__main__": - main() diff --git a/neon_core/run_neon.py b/neon_core/run_neon.py index ac818228d..7738a5cf4 100644 --- a/neon_core/run_neon.py +++ b/neon_core/run_neon.py @@ -123,7 +123,7 @@ def _stop_all_core_processes(): procs = {p.pid: p.cmdline() for p in psutil.process_iter()} for pid, cmdline in procs.items(): if cmdline and (any(pname in cmdline[-1] for pname in ("mycroft.messagebus.service", "neon_speech_client", - "neon_audio_client", "neon_core.messagebus.service", + "neon_audio_client", "neon_messagebus_service", "neon_core.skills", "neon_core.gui", "neon_gui_service", "neon_core_server", "neon_enclosure_client", "neon_core_client", "mycroft-gui-app", @@ -155,7 +155,8 @@ def start_neon(): _stop_all_core_processes() _cycle_logs() - _start_process(["python3", "-m", "neon_core.messagebus.service"]) or STOP_MODULES.set() + _start_process(["neon_messagebus_service"]) or STOP_MODULES.set() + bus.connected_event.wait() _start_process("neon_speech_client") or STOP_MODULES.set() _start_process("neon_audio_client") or STOP_MODULES.set() _start_process(["python3", "-m", "neon_core.skills"]) or STOP_MODULES.set() diff --git a/neon_core/skills/skill_store.py b/neon_core/skills/skill_store.py index 94f1f14ec..fa4a142b2 100644 --- a/neon_core/skills/skill_store.py +++ b/neon_core/skills/skill_store.py @@ -31,7 +31,7 @@ from neon_utils.configuration_utils import get_neon_skills_config from datetime import datetime, timedelta -from neon_core.messagebus import get_messagebus +from neon_messagebus.util.message_utils import get_messagebus from neon_core.util.skill_utils import get_remote_entries from mycroft.skills.event_scheduler import EventSchedulerInterface diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 11c9c0063..eec5793c3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,8 +1,10 @@ # mycroft +# TODO: Reduce ovos-core installed deps here ovos-core[all]==0.0.1 mock_msm # neon core modules +neon_messagebus neon_speech~=0.3 neon_audio~=0.4 neon_enclosure~=0.1,>=0.1.2 @@ -10,7 +12,7 @@ neon_gui # TODO: Version spec GUI # utils -neon-utils>=0.12.6 +neon-utils~=0.12 rapidfuzz kthread ovos_utils>=0.0.12 diff --git a/setup.py b/setup.py index b1311265b..134c219a6 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,6 @@ def get_requirements(requirements_filename: str): include_package_data=True, entry_points={ 'console_scripts': [ - 'neon_messagebus_service=neon_core.messagebus.service.__main__:main', 'neon_skills_service=neon_core.skills.__main__:main', 'neon_gui_service=neon_core.gui.__main__:main', 'neon-install-default-skills=neon_core.util.skill_utils:install_skills_default', diff --git a/test/test_run_modules.py b/test/test_run_modules.py index bacc68373..19fc18fae 100644 --- a/test/test_run_modules.py +++ b/test/test_run_modules.py @@ -24,7 +24,6 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import sys import unittest from multiprocessing import Process @@ -32,9 +31,7 @@ from mycroft_bus_client import MessageBusClient, Message from neon_speech.__main__ import main as neon_speech_main from neon_audio.__main__ import main as neon_audio_main - -sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from neon_core.messagebus.service.__main__ import main as messagebus_service +from neon_messagebus.service import NeonBusService AUDIO_FILE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "audio_files") @@ -48,7 +45,8 @@ class TestModules(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.bus_thread = Process(target=messagebus_service, daemon=False) + cls.messagebus_service = NeonBusService() + cls.messagebus_service.start() cls.speech_thread = Process(target=neon_speech_main, daemon=False) cls.audio_thread = Process(target=neon_audio_main, daemon=False) cls.bus_thread.start() @@ -61,7 +59,7 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.bus.close() - cls.bus_thread.terminate() + cls.messagebus_service.shutdown() cls.speech_thread.terminate() cls.audio_thread.terminate() From aad941f19bd9e6ab1796f6b5b1e38d11870e2a2f Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 17 Dec 2021 11:42:14 -0800 Subject: [PATCH 12/70] Refactor default skill installation (#188) * Add `get_neon_skills_data` to preload skill json Update skill_utils tests * Extend skills timeout to troubleshoot test failure * Refactor skill util tests Fallback to non-cached requests if returned 400 * Refactor skill utils to use OSM/OSI instead of git directly --- neon_core/util/skill_utils.py | 62 +++++++++++++++++++++++++++++++---- test/test_run_neon.py | 2 +- test/test_skill_utils.py | 12 +++++-- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index aaf819ce9..24d7952e1 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -23,29 +23,75 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from os.path import expanduser +import json +import requests + +from os import listdir +from tempfile import mkdtemp +from shutil import rmtree +from os.path import expanduser, join +from ovos_skills_manager.skill_entry import SkillEntry from ovos_skills_manager.osm import OVOSSkillsManager -from ovos_skills_manager.session import SESSION as requests, set_github_token, clear_github_token +from ovos_skills_manager.session import SESSION, set_github_token, clear_github_token +from ovos_skills_manager.github import normalize_github_url, get_branch_from_github_url, download_url_from_github_url +from ovos_skill_installer import download_extract_zip from neon_utils.configuration_utils import get_neon_skills_config from neon_utils import LOG +def get_neon_skills_data(skill_meta_repository: str = "https://github.com/neongeckocom/neon_skills", + branch: str = "master", + repo_metadata_path: str = "skill_metadata") -> dict: + """ + Get skill data from configured neon_skills repository. + :param skill_meta_repository: URL of skills repository containing metadata + :param branch: branch of repository to checkout + :param repo_metadata_path: Path to repo directory containing json metadata files + """ + skills_data = dict() + temp_download_dir = mkdtemp() + zip_url = download_url_from_github_url(skill_meta_repository, branch) + base_dir = join(temp_download_dir, "neon_skill_meta") + download_extract_zip(zip_url, temp_download_dir, "neon_skill_meta.zip", base_dir) + + meta_dir = join(base_dir, repo_metadata_path) + for entry in listdir(meta_dir): + with open(join(meta_dir, entry)) as f: + skill_entry = json.load(f) + skills_data[normalize_github_url(skill_entry["url"])] = skill_entry + rmtree(temp_download_dir) + return skills_data + + def install_skills_from_list(skills_to_install: list, config: dict = None): """ Installs the passed list of skill URLs - :param skills_to_install: list or skill URLs to install + :param skills_to_install: list of skill URLs to install :param config: optional dict configuration """ config = config or get_neon_skills_config() skill_dir = expanduser(config["directory"]) osm = OVOSSkillsManager() + skills_catalog = get_neon_skills_data() token_set = False if config.get("neon_token"): token_set = True set_github_token(config["neon_token"]) for url in skills_to_install: try: - osm.install_skill_from_url(url, skill_dir) + normalized_url = normalize_github_url(url) + # Check if this skill is in the Neon list + if normalized_url in skills_catalog: + branch = get_branch_from_github_url(url) + # Set URL and branch to requested spec + skills_catalog[normalized_url]["url"] = normalized_url + skills_catalog[normalized_url]["branch"] = branch + entry = SkillEntry.from_json(skills_catalog.get(normalized_url), False) + else: + LOG.warning(f"Requested Skill not in Neon skill store ({url})") + entry = osm.skill_entry_from_url(url) + # try: + osm.install_skill(entry, skill_dir) LOG.info(f"Installed {url} to {skill_dir}") except Exception as e: LOG.error(e) @@ -70,8 +116,12 @@ def install_skills_default(config: dict = None): def get_remote_entries(url): """ parse url and return a list of SkillEntry, expects 1 skill per line, can be a skill_id or url""" - r = requests.get(url) - if r.status_code == 200: + r = SESSION.get(url) + if not r.ok: + LOG.warning(f"Cached response returned: {r.status_code}") + SESSION.cache.delete_url(r.url) + r = requests.get(url) + if r.ok: return [s for s in r.text.split("\n") if s.strip()] else: LOG.error(f"{url} request failed with code: {r.status_code}") diff --git a/test/test_run_neon.py b/test/test_run_neon.py index 93b59484d..08690acfd 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -131,7 +131,7 @@ def test_audio_module(self): # self.assertIsInstance(data["success"], bool) def test_skills_module(self): - response = self.bus.wait_for_response(Message('mycroft.skills.is_ready')) + response = self.bus.wait_for_response(Message('mycroft.skills.is_ready'), timeout=10) self.assertTrue(response.data['status']) response = self.bus.wait_for_response(Message("skillmanager.list"), "mycroft.skills.list") diff --git a/test/test_skill_utils.py b/test/test_skill_utils.py index e85524c45..1ac2432d1 100644 --- a/test/test_skill_utils.py +++ b/test/test_skill_utils.py @@ -42,7 +42,7 @@ ] SKILL_DIR = os.path.join(os.path.dirname(__file__), "test_skills") SKILL_CONFIG = { - "default_skills": "https://raw.githubusercontent.com/NeonGeckoCom/neon-skills-submodules/dev/.utilities/" + "default_skills": "https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/" "DEFAULT-SKILLS-DEV", "neon_token": os.environ.get("GITHUB_TOKEN"), "directory": SKILL_DIR @@ -83,7 +83,15 @@ def test_install_skills_from_list_with_auth(self): def test_install_skills_default(self): install_skills_default(SKILL_CONFIG) skill_dirs = [d for d in os.listdir(SKILL_DIR) if os.path.isdir(os.path.join(SKILL_DIR, d))] - self.assertEqual(len(skill_dirs), len(get_remote_entries(SKILL_CONFIG["default_skills"])), f"{skill_dirs}\n\n{get_remote_entries(SKILL_CONFIG['default_skills'])}") + self.assertEqual(len(skill_dirs), len(get_remote_entries(SKILL_CONFIG["default_skills"])), + f"{skill_dirs}\n\n{get_remote_entries(SKILL_CONFIG['default_skills'])}") + + def test_get_neon_skills_data(self): + neon_skills = get_neon_skills_data() + self.assertIsInstance(neon_skills, dict) + for skill in neon_skills: + self.assertIsInstance(neon_skills[skill], dict) + self.assertEqual(skill, normalize_github_url(neon_skills[skill]["url"])) if __name__ == '__main__': From 398b194c4f919ead849cee0580d88639b3195dcd Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 17 Dec 2021 19:42:35 +0000 Subject: [PATCH 13/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 93ea8ac82..f3403f5e1 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a4" +__version__ = "21.10.2a5" From c6a9b867bb9033dd8ba868f4354f587e2ab3c85a Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 13 Jan 2022 11:32:16 -0800 Subject: [PATCH 14/70] Fix Setup Test Failures (#197) * Loosen remote dependency specs * Cleanup dependency specs * Test dependency resolution changes * Test dependency resolution changes * Fix typo in requirements * Troubleshooting dependency errors * Troubleshooting dependency errors * Troubleshooting remote dependency errors Try updated pip version * Troubleshooting remote dependency errors * Update amazon translate dependency version * Remove some dependency patches * Replace ovos-utils version patch * loosen ovos_utils version spec * Add back OPM version patch * Troubleshooting skills module test failures * Troubleshooting inconsistent GH test failures * Update dependency specs Add `ready_settings` to troubleshoot test failures * Increase test timeout Update neon.conf default skills URL * Troubleshooting test timeout failures * Rollback changes to troubleshoot GH Actions failures * Update neon-utils dep spec Fix typos in setup_tests uploads * Increment neon-utils version * Update actions uploads to keep logs from all runs * Cleanup setup workflow and add upload of local test logs * Implement core test timeouts * Ensure artifacts are always uploaded * Troubleshooting local test failures * Troubleshoot artifact upload paths * Troubleshoot artifact upload paths * Troubleshoot artifact upload paths * Specify config path in dev setup tests * Replace remote unit tests Comment patch in test setup script * Blacklist skills to suppress logging and troubleshoot failures Fix typo in test results upload * Troubleshoot test failure Cleanup Changes * Troubleshoot local test failure due to model download * Annotate patched speech module test --- .github/workflows/setup_tests.yml | 24 +++++++++++++++-------- neon_core/configuration/neon.conf | 2 +- requirements/client.txt | 6 +++--- requirements/local_speech_processing.txt | 5 ++--- requirements/pi.txt | 3 --- requirements/remote_speech_processing.txt | 6 +++--- requirements/requirements.txt | 17 ++++++++-------- setup.sh | 2 +- test/setup_dev_local.sh | 4 +++- test/setup_remote.sh | 4 +++- test/test_run_neon.py | 19 ++++++++++++++++-- 11 files changed, 58 insertions(+), 34 deletions(-) diff --git a/.github/workflows/setup_tests.yml b/.github/workflows/setup_tests.yml index bbc85a474..f1dbc6644 100644 --- a/.github/workflows/setup_tests.yml +++ b/.github/workflows/setup_tests.yml @@ -11,7 +11,7 @@ jobs: matrix: python-version: [ 3.6, 3.7, 3.8 ] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v2 - name: Set up python ${{ matrix.python-version }} @@ -29,6 +29,7 @@ jobs: GOOGLE_KEY: ${{secrets.google_api_key}} AWS_CREDS: ${{secrets.amazon_creds}} - name: Test Core Setup + timeout-minutes: 10 run: | . test/.venv/bin/activate pytest test/test_setup_remote.py --junitxml=tests/remote-setup-results.xml @@ -36,21 +37,21 @@ jobs: - name: Upload Core Setup test results uses: actions/upload-artifact@v2 with: - name: pytest-results-3.6 + name: pytest-results-remote-${{ matrix.python-version }} path: tests/remote-test-results.xml - if: ${{ always() }} + if: always() - name: Upload Core Setup logs uses: actions/upload-artifact@v2 with: - name: pytest-results-3.6 + name: core-logs-remote-${{ matrix.python-version }} path: ~/.local/share/neon/logs/*.log - if: ${{ always() }} + if: always() dev_local: strategy: matrix: python-version: [ 3.8 ] runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 30 steps: - uses: actions/checkout@v2 - name: Set up python ${{ matrix.python-version }} @@ -63,6 +64,7 @@ jobs: env: NEON_TOKEN: ${{secrets.neon_token}} - name: Test Core Setup + timeout-minutes: 10 run: | . test/.venv/bin/activate pip install pytest pytest-timeout @@ -71,6 +73,12 @@ jobs: - name: Upload Core Setup test results uses: actions/upload-artifact@v2 with: - name: pytest-results-3.6 + name: pytest-results-local-${{ matrix.python-version }} path: tests/dev_local-test-results.xml - if: ${{ always() }} + if: always() + - name: Upload Core Setup logs + uses: actions/upload-artifact@v2 + with: + name: core-logs-dev-local-${{ matrix.python-version }} + path: test/logs/*.log + if: always() diff --git a/neon_core/configuration/neon.conf b/neon_core/configuration/neon.conf index fc4954b1d..3442f2324 100644 --- a/neon_core/configuration/neon.conf +++ b/neon_core/configuration/neon.conf @@ -254,7 +254,7 @@ "install_default": false, // can be an url, list of urls, or list of search terms for osm // RECOMMENDED: url to txt file, one skill_id/skill_url per line - "default_skills": "https://raw.githubusercontent.com/NeonGeckoCom/neon-skills-submodules/master/DEFAULT-SKILLS" + "default_skills": "https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS" }, diff --git a/requirements/client.txt b/requirements/client.txt index effb642a5..cc8535c06 100644 --- a/requirements/client.txt +++ b/requirements/client.txt @@ -1,6 +1,6 @@ neon-transcripts-controller @ git+https://github.com/NeonGeckoCom/transcripts_controller # wake word plugins -chatterbox-wake-word-plugin-dummy -ovos-ww-plugin-pocketsphinx -ovos-ww-plugin-precise +chatterbox-wake-word-plugin-dummy~=0.1 +ovos-ww-plugin-pocketsphinx~=0.1 +ovos-ww-plugin-precise~=0.1 diff --git a/requirements/local_speech_processing.txt b/requirements/local_speech_processing.txt index 6a88ed70b..1fa8259da 100644 --- a/requirements/local_speech_processing.txt +++ b/requirements/local_speech_processing.txt @@ -1,3 +1,2 @@ -neon-stt-plugin-deepspeech-stream-local>=0.1.2 -neon-tts-plugin-mimic>=0.1.3 -neon-lang-plugin-libretranslate>=0.1.2 +neon-stt-plugin-deepspeech-stream-local~=0.1 +neon-tts-plugin-mimic~=0.1 diff --git a/requirements/pi.txt b/requirements/pi.txt index 3758ba2a7..81a304763 100644 --- a/requirements/pi.txt +++ b/requirements/pi.txt @@ -1,5 +1,2 @@ deepspeech @ https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/deepspeech-0.9.3-cp37-cp37m-linux_aarch64.whl vosk @ https://github.com/alphacep/vosk-api/releases/download/v0.3.30/vosk-0.3.30-py3-none-linux_aarch64.whl - -neon-tts-plugin-mimic -neon-stt-plugin-deepspeech-stream-local diff --git a/requirements/remote_speech_processing.txt b/requirements/remote_speech_processing.txt index 01bfd77a4..837162e76 100644 --- a/requirements/remote_speech_processing.txt +++ b/requirements/remote_speech_processing.txt @@ -1,3 +1,3 @@ -neon-stt-plugin-google-cloud-streaming>=0.2.2 -neon-tts-plugin-polly~=0.1.0 -neon-lang-plugin-amazon_translate>=0.1.0 \ No newline at end of file +neon-stt-plugin-google-cloud-streaming~=0.2 +neon-tts-plugin-polly~=0.1 +neon-lang-plugin-amazon_translate~=0.1 \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index eec5793c3..aaa2a794e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,15 +12,16 @@ neon_gui # TODO: Version spec GUI # utils -neon-utils~=0.12 -rapidfuzz -kthread -ovos_utils>=0.0.12 -ovos-skills-manager>=0.0.2 +neon-utils~=0.12,>=0.14.3a5 +# TODO: Update to stable version +ovos_utils~=0.0.12 +ovos-skills-manager~=0.0.8 # plugins -ovos-plugin-manager~=0.0.1 -neon-lang-plugin-libretranslate>=0.1.2 +neon-lang-plugin-libretranslate~=0.1,>=0.1.2 + +# TODO: Patching OPM +ovos-plugin-manager==0.0.2 # text parser modules -RAKEkeywords>=0.2.0 +RAKEkeywords~=0.2 diff --git a/setup.sh b/setup.sh index 700c1fa44..110e20e42 100644 --- a/setup.sh +++ b/setup.sh @@ -334,7 +334,7 @@ doInstall(){ fi echo "${GITHUB_TOKEN}">~/token.txt - pip install --upgrade pip~=21.1.0 + pip install --upgrade pip~=21.3 pip install wheel pip install "${pipStr}" neon-config-import diff --git a/test/setup_dev_local.sh b/test/setup_dev_local.sh index 12632dc0e..d02682317 100644 --- a/test/setup_dev_local.sh +++ b/test/setup_dev_local.sh @@ -96,12 +96,14 @@ if [ "${installGui}" == "true" ]; then fi echo "${GITHUB_TOKEN}">~/token.txt -pip install --upgrade pip==21.2.4 +pip install --upgrade pip~=21.3 pip install wheel cd "${sourceDir}" || exit 10 pip install ".${optStr}" # --use-deprecated=legacy-resolver +# TODO: This is patching an issue with config paths containing `NeonCore/NeonCore`; patch in devMode setup DM +export NEON_CONFIG_PATH="${sourceDir}" neon-config-import # Install Default Skills diff --git a/test/setup_remote.sh b/test/setup_remote.sh index 64c07f015..f2f80af0d 100644 --- a/test/setup_remote.sh +++ b/test/setup_remote.sh @@ -99,12 +99,14 @@ fi echo "${GITHUB_TOKEN}">~/token.txt -pip install --upgrade pip==21.2.4 +pip install --upgrade pip~=21.3 pip install wheel cd "${sourceDir}" || exit 10 pip install ".${optStr}" # --use-deprecated=legacy-resolver +# TODO: This is patching an issue with config paths containing `NeonCore/NeonCore`; patch in devMode setup DM +export NEON_CONFIG_PATH="${sourceDir}" neon-config-import # Install Default Skills diff --git a/test/test_run_neon.py b/test/test_run_neon.py index 08690acfd..45dd0c631 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -26,12 +26,12 @@ import os.path import sys import unittest -import pytest from time import time, sleep from multiprocessing import Process from neon_utils.log_utils import LOG from mycroft_bus_client import MessageBusClient, Message +from neon_utils.configuration_utils import get_neon_local_config sys.path.append(os.path.dirname(os.path.dirname(__file__))) from neon_core.run_neon import start_neon, stop_neon @@ -42,6 +42,14 @@ class TestRunNeon(unittest.TestCase): @classmethod def setUpClass(cls) -> None: + # Blacklist skills to prevent logged errors + local_conf = get_neon_local_config() + local_conf["skills"]["blacklist"] = \ + local_conf["skills"]["blacklist"].extend( + ["skill-ovos-homescreen.openvoiceos", + "skill-balena-wifi-setup.openvoiceos"]) + local_conf.write_changes() + cls.process = Process(target=start_neon, daemon=False) cls.process.start() cls.bus = MessageBusClient() @@ -75,7 +83,14 @@ def test_messagebus_connection(self): bus.close() def test_speech_module(self): + # TODO: Remove this after readiness is better defined DM + i = 0 response = self.bus.wait_for_response(Message('mycroft.speech.is_ready')) + while not response.data['status'] and i < 10: + LOG.warning(f"Speech not ready when core reported ready!") + sleep(5) + response = self.bus.wait_for_response(Message('mycroft.speech.is_ready')) + i += 1 self.assertTrue(response.data['status']) context = {"client": "tester", @@ -131,7 +146,7 @@ def test_audio_module(self): # self.assertIsInstance(data["success"], bool) def test_skills_module(self): - response = self.bus.wait_for_response(Message('mycroft.skills.is_ready'), timeout=10) + response = self.bus.wait_for_response(Message('mycroft.skills.is_ready')) self.assertTrue(response.data['status']) response = self.bus.wait_for_response(Message("skillmanager.list"), "mycroft.skills.list") From c6d0521620cf1c2ca229df21412bc7c97f16c6ad Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 13 Jan 2022 19:32:41 +0000 Subject: [PATCH 15/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index f3403f5e1..ad837328e 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a5" +__version__ = "21.10.2a6" From bbd28792fcfc5af82176faad5e75ef83ee729240 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 24 Jan 2022 18:01:29 -0800 Subject: [PATCH 16/70] Initialize Signal Handlers in skills service (#180) --- neon_core/skills/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index 84049bc1d..4ec886773 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -29,6 +29,7 @@ from neon_utils.net_utils import check_online from neon_utils import LOG from neon_utils.metrics_utils import announce_connection +from neon_utils.signal_utils import init_signal_handlers, init_signal_bus from neon_core.skills.fallback_skill import FallbackSkill from neon_core.skills.intent_service import NeonIntentService @@ -86,6 +87,8 @@ def start(self): set_default_tz() self.bus = self.bus or start_message_bus_client("SKILLS") + init_signal_bus(self.bus) + init_signal_handlers() self._register_intent_services() self.event_scheduler = EventScheduler(self.bus) self.status = ProcessStatus('skills', self.bus, self.callbacks) From 21a7f6ba348f2450e3f38819749f75d5b24580a0 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Tue, 25 Jan 2022 02:01:50 +0000 Subject: [PATCH 17/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index ad837328e..481f0c070 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a6" +__version__ = "21.10.2a7" From d4998d757e013ee7af715f5504cad13ee5347512 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Tue, 25 Jan 2022 15:24:24 -0800 Subject: [PATCH 18/70] QML File Server (#202) * Add skill http server * Cleanup skill file server and fix license * Refactor file server to qml_file_server.py Update file server to handle system and skill QML resources * Cleanup QML File Server implementation --- .github/workflows/unit_tests.yml | 9 ++++ neon_core/skills/service.py | 22 +++++--- neon_core/util/qml_file_server.py | 84 +++++++++++++++++++++++++++++++ requirements/requirements.txt | 2 +- test/test_qml_file_server.py | 49 ++++++++++++++++++ 5 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 neon_core/util/qml_file_server.py create mode 100644 test/test_qml_file_server.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a63ac015f..9043e9de0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -66,6 +66,15 @@ jobs: name: diagnostic-utils-test-results path: tests/diagnostic-utils-test-results.xml + - name: Test QML File Server + run: | + pytest test/test_qml_file_server.py --doctest-modules --junitxml=tests/qml-file-server-test-results.xml + - name: Upload QML File Server test results + uses: actions/upload-artifact@v2 + with: + name: qml-file-server-test-results + path: tests/qml-file-server-test-results.xml + unit_tests: strategy: matrix: diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index 4ec886773..b3b8ae75b 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -22,6 +22,7 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import time from neon_utils.configuration_utils import get_neon_skills_config, \ @@ -30,16 +31,17 @@ from neon_utils import LOG from neon_utils.metrics_utils import announce_connection from neon_utils.signal_utils import init_signal_handlers, init_signal_bus +from neon_utils.messagebus_utils import get_messagebus from neon_core.skills.fallback_skill import FallbackSkill from neon_core.skills.intent_service import NeonIntentService from neon_core.skills.skill_manager import NeonSkillManager from neon_core.util.diagnostic_utils import report_metric +from neon_core.util.qml_file_server import start_qml_http_server from mycroft.skills.api import SkillApi from mycroft.skills.event_scheduler import EventScheduler from mycroft.skills.msm_wrapper import MsmException -from mycroft.util import start_message_bus_client from mycroft.configuration.locale import set_default_lang, set_default_tz from mycroft.util.process_utils import ProcessStatus, StatusCallbackMap @@ -78,6 +80,12 @@ def __init__(self, alive_hook=on_alive, started_hook=on_started, on_ready=ready_hook, on_error=error_hook, on_stopping=stopping_hook) + skill_config = get_neon_skills_config() + if skill_config.get("run_gui_file_server"): + self.http_server = start_qml_http_server( + skill_config["directory"]) + else: + self.http_server = None def start(self): # config = Configuration.get() @@ -86,7 +94,7 @@ def start(self): # Set the default timezone to match the configured one set_default_tz() - self.bus = self.bus or start_message_bus_client("SKILLS") + self.bus = self.bus or get_messagebus() init_signal_bus(self.bus) init_signal_handlers() self._register_intent_services() @@ -122,10 +130,8 @@ def handle_metric(message): LOG.info("Metrics reporting disabled") def _register_intent_services(self): - """Start up the all intent services and connect them as needed. - - Arguments: - bus: messagebus client to register the services on + """ + Start up the all intent services and connect them as needed. """ service = NeonIntentService(self.bus) # Register handler to trigger fallback system @@ -167,6 +173,10 @@ def shutdown(self): self.status.set_stopping() if self.event_scheduler is not None: self.event_scheduler.shutdown() + + if self.http_server is not None: + self.http_server.shutdown() + # Terminate all running threads that update skills if self.skill_manager is not None: self.skill_manager.stop() diff --git a/neon_core/util/qml_file_server.py b/neon_core/util/qml_file_server.py new file mode 100644 index 000000000..bf9035e66 --- /dev/null +++ b/neon_core/util/qml_file_server.py @@ -0,0 +1,84 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import socketserver +import http.server + +from tempfile import gettempdir +from os.path import isdir, join, dirname +from threading import Thread, Event + +_HTTP_SERVER = None + + +class QmlFileHandler(http.server.SimpleHTTPRequestHandler): + def end_headers(self) -> None: + mimetype = self.guess_type(self.path) + is_file = not self.path.endswith('/') + if is_file and any([mimetype.startswith(prefix) for + prefix in ("text/", "application/octet-stream")]): + self.send_header('Content-Type', "text/plain") + self.send_header('Content-Disposition', 'inline') + super().end_headers() + + +def start_qml_http_server(skills_dir: str, port: int = 8000): + if not isdir(skills_dir): + os.makedirs(skills_dir) + system_dir = join(dirname(dirname(__file__)), "res") + + qml_dir = join(gettempdir(), "neon", "qml") + os.makedirs(qml_dir, exist_ok=True) + + served_skills_dir = join(qml_dir, "skills") + served_system_dir = join(qml_dir, "system") + if os.path.exists(served_skills_dir): + os.remove(served_skills_dir) + if os.path.exists(served_system_dir): + os.remove(served_system_dir) + + os.symlink(skills_dir, join(qml_dir, "skills")) + os.symlink(system_dir, join(qml_dir, "system")) + started_event = Event() + http_daemon = Thread(target=_initialize_http_server, + args=(started_event, qml_dir, port), + daemon=True) + http_daemon.start() + started_event.wait(30) + return _HTTP_SERVER + + +def _initialize_http_server(started: Event, directory: str, port: int): + global _HTTP_SERVER + os.chdir(directory) + handler = QmlFileHandler + http_server = socketserver.TCPServer(("", port), handler) + _HTTP_SERVER = http_server + started.set() + http_server.serve_forever() diff --git a/requirements/requirements.txt b/requirements/requirements.txt index aaa2a794e..54cf05b12 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,7 +12,7 @@ neon_gui # TODO: Version spec GUI # utils -neon-utils~=0.12,>=0.14.3a5 +neon-utils~=0.12,>=0.14.3a7 # TODO: Update to stable version ovos_utils~=0.0.12 ovos-skills-manager~=0.0.8 diff --git a/test/test_qml_file_server.py b/test/test_qml_file_server.py new file mode 100644 index 000000000..5937c6f0a --- /dev/null +++ b/test/test_qml_file_server.py @@ -0,0 +1,49 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import sys +import unittest +import requests + +from socketserver import TCPServer + +sys.path.append(os.path.dirname(os.path.dirname(__file__))) +from neon_core.util.qml_file_server import start_qml_http_server + + +class SkillFileServerTests(unittest.TestCase): + + def test_start_file_server(self): + server = start_qml_http_server('/') + self.assertIsInstance(server, TCPServer) + resp = requests.get("http://localhost:8000") + self.assertTrue(resp.ok) + self.assertIn("Directory listing for /", resp.text) + server.shutdown() + + +if __name__ == '__main__': + unittest.main() From 4812c3962f5119e6396d8e6fba5c0c19c7c37d4e Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Tue, 25 Jan 2022 23:24:45 +0000 Subject: [PATCH 19/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 481f0c070..797dac711 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a7" +__version__ = "21.10.2a8" From 59d0909602583d0cb30bdd929c69a47adb595f56 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:33:52 -0800 Subject: [PATCH 20/70] Normalize language in `handle_utterance` (#204) --- neon_core/skills/intent_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index cf7288807..1dc2acac8 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -37,6 +37,7 @@ from neon_utils.log_utils import LOG from neon_utils.configuration_utils import get_neon_device_type from ovos_utils.json_helper import merge_dict +from lingua_franca.parse import get_full_lang_code from mycroft.configuration.locale import set_default_lang from mycroft.skills.intent_service import IntentService @@ -138,7 +139,8 @@ def handle_utterance(self, message): try: # Get language of the utterance - lang = message.data.get('lang', self.language_config["user"]) + lang = get_full_lang_code( + message.data.get('lang', self.language_config["user"])) # Add or init timing data message.context = message.context or {} From ff417518d3d980c169f71e58de5c3ec6e1e28361 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 26 Jan 2022 18:34:14 +0000 Subject: [PATCH 21/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 797dac711..5bc23a67a 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a8" +__version__ = "21.10.2a9" From dbebb3ad0d2e5e383bcb886d88bbea057a76568c Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 31 Jan 2022 13:06:35 -0800 Subject: [PATCH 22/70] Restructure dependencies (#203) * Refactor core module dependencies into an optional spec * Refactor get_messagebus to import from neon_utils * Refactor core module dependencies into an optional spec --- neon_core/messagebus/__init__.py | 2 +- neon_core/skills/skill_store.py | 2 +- requirements/core_modules.txt | 9 +++++++++ requirements/requirements.txt | 18 +++++------------- setup.py | 1 + setup.sh | 2 +- test/setup_dev_local.sh | 2 +- test/setup_remote.sh | 2 +- 8 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 requirements/core_modules.txt diff --git a/neon_core/messagebus/__init__.py b/neon_core/messagebus/__init__.py index 8daedd1bc..b158696d2 100644 --- a/neon_core/messagebus/__init__.py +++ b/neon_core/messagebus/__init__.py @@ -23,7 +23,7 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from neon_messagebus.util.message_utils import get_messagebus +from neon_utils.messagebus_utils import get_messagebus from neon_utils import LOG LOG.warning("This reference is deprecated; import from neon_messagebus directly") # TODO: Deprecate in neon_core 22.04 diff --git a/neon_core/skills/skill_store.py b/neon_core/skills/skill_store.py index fa4a142b2..fd1bae00d 100644 --- a/neon_core/skills/skill_store.py +++ b/neon_core/skills/skill_store.py @@ -31,7 +31,7 @@ from neon_utils.configuration_utils import get_neon_skills_config from datetime import datetime, timedelta -from neon_messagebus.util.message_utils import get_messagebus +from neon_utils.messagebus_utils import get_messagebus from neon_core.util.skill_utils import get_remote_entries from mycroft.skills.event_scheduler import EventSchedulerInterface diff --git a/requirements/core_modules.txt b/requirements/core_modules.txt new file mode 100644 index 000000000..13d8bfa8c --- /dev/null +++ b/requirements/core_modules.txt @@ -0,0 +1,9 @@ +ovos-core[all]~=0.0.1 + +# neon core modules +neon_enclosure~=0.1,>=0.1.2 +neon_messagebus +neon_speech~=0.3 +neon_audio~=0.4 +neon_gui +# TODO: Version spec GUI \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 54cf05b12..a97ad482e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,27 +1,19 @@ # mycroft -# TODO: Reduce ovos-core installed deps here -ovos-core[all]==0.0.1 +ovos-core[skills]==0.0.1 mock_msm -# neon core modules -neon_messagebus -neon_speech~=0.3 -neon_audio~=0.4 -neon_enclosure~=0.1,>=0.1.2 -neon_gui -# TODO: Version spec GUI - # utils neon-utils~=0.12,>=0.14.3a7 # TODO: Update to stable version ovos_utils~=0.0.12 ovos-skills-manager~=0.0.8 -# plugins -neon-lang-plugin-libretranslate~=0.1,>=0.1.2 - # TODO: Patching OPM ovos-plugin-manager==0.0.2 +SpeechRecognition~=3.8 + +# plugins +neon-lang-plugin-libretranslate~=0.1,>=0.1.2 # text parser modules RAKEkeywords~=0.2 diff --git a/setup.py b/setup.py index 134c219a6..6e7f4b3c2 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ def get_requirements(requirements_filename: str): long_description_content_type="text/markdown", install_requires=get_requirements('requirements.txt'), extras_require={ + "core_modules": get_requirements("core_modules.txt"), "client": get_requirements("client.txt"), "server": get_requirements("server.txt"), "dev": get_requirements("dev.txt"), diff --git a/setup.sh b/setup.sh index 110e20e42..6ec63402f 100644 --- a/setup.sh +++ b/setup.sh @@ -265,7 +265,7 @@ doInstall(){ fi # Build optional dependency string for pip installation - options=() + options=("core_modules") if [ "${localDeps}" == "true" ]; then if [ "${arm}" == "true" ]; then echo "Local Dependencies not supported on ARM; remote STT/TTS will be used." diff --git a/test/setup_dev_local.sh b/test/setup_dev_local.sh index d02682317..dd17eceb6 100644 --- a/test/setup_dev_local.sh +++ b/test/setup_dev_local.sh @@ -42,7 +42,7 @@ export ttsModule="neon_tts_mimic" localDeps="true" installGui="false" -options=() +options=("core_modules") options+=("test") if [ "${localDeps}" == "true" ]; then options+=("local") diff --git a/test/setup_remote.sh b/test/setup_remote.sh index f2f80af0d..22072b458 100644 --- a/test/setup_remote.sh +++ b/test/setup_remote.sh @@ -43,7 +43,7 @@ export ttsModule="amazon" localDeps="false" installGui="false" -options=() +options=("core_modules") options+=("test") if [ "${localDeps}" == "true" ]; then options+=("local") From d31e754dfddc21e4c3389643712299c2da19bb57 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Mon, 31 Jan 2022 21:06:53 +0000 Subject: [PATCH 23/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 5bc23a67a..01d110d98 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a9" +__version__ = "21.10.2a10" From 3dc593850341e10cca8e0870ba9e9fdfd9b1d410 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 31 Jan 2022 13:25:42 -0800 Subject: [PATCH 24/70] Add call to initialize config directory in module init (#205) --- neon_core/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neon_core/__init__.py b/neon_core/__init__.py index aa939d7d7..74420f4ba 100644 --- a/neon_core/__init__.py +++ b/neon_core/__init__.py @@ -96,7 +96,8 @@ def setup_ovos_config(): # for now it simply provides correct default values setup_ovos_core_config() -from neon_utils.configuration_utils import write_mycroft_compatible_config +from neon_utils.configuration_utils import write_mycroft_compatible_config, init_config_dir +init_config_dir() # Write and reload Mycroft-compat conf file neon_config_path = join(xdg.BaseDirectory.save_config_path("neon"), From d2b62d54ffcf06e709974acb4008f3f329d051ad Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Mon, 31 Jan 2022 21:26:03 +0000 Subject: [PATCH 25/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 01d110d98..c3c711e10 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a10" +__version__ = "21.10.2a11" From d9e61cefa92b031ae1279f207eb83a0b9b32bdbf Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Tue, 1 Feb 2022 17:04:25 -0800 Subject: [PATCH 26/70] Docker Support (#132) * Reduce base dependencies to skills module only Add Dockerfile, overlay files, and updated README documentation * Refactor core modules to an optional extra * Update requirements spec * Add `git` system dependency to skills module for CC skill compat. * Update docker build stage targets Add docker workflows to automation * Update docker step IDs * Add `env` params to Docker workflow * Refactor container image name to "neon-skills" * Fix name in IMAGE_NAME * Add back version automation and remove `workflow_dispatch` option * Troubleshoot docker alpha versioning * Add back workflow_dispatch for testing * Refactor Docker version automation * Fix logic in docker release automation Co-authored-by: Daniel McKnight --- .github/workflows/publish_release.yml | 56 +++++++++++++++++ .github/workflows/publish_test_build.yml | 63 ++++++++++++++++++- Dockerfile | 39 ++++++++++++ README.md | 16 +++++ docker_overlay/asoundrc | 2 + docker_overlay/mycroft.conf | 5 ++ docker_overlay/ngi_local_conf.yml | 7 +++ .../settings.json | 7 +++ requirements/requirements.txt | 2 +- 9 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 docker_overlay/asoundrc create mode 100644 docker_overlay/mycroft.conf create mode 100644 docker_overlay/ngi_local_conf.yml create mode 100644 docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 9430eb319..cc0bf6ffb 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -19,3 +19,59 @@ jobs: with: token: ${{secrets.GITHUB_TOKEN}} tag: ${{env.VERSION}} + build_and_publish_docker: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Get Version + id: version + run: | + VERSION=$(sed "s/a/-a./" <<< $(python setup.py --version)) + echo ::set-output name=version::${VERSION} + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for base Docker + id: meta + uses: docker/metadata-action@v2 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{ steps.version.outputs.version }} + type=ref,event=branch + - name: Build and push base Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: base + + - name: Extract metadata for default_skills Docker + id: meta + uses: docker/metadata-action@v2 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-default_skills + tags: | + type=semver,pattern={{version}},value=${{ steps.version.outputs.version }} + type=ref,event=branch + - name: Build and push default_skills Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: default_skills \ No newline at end of file diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml index 1d7180599..62776473c 100644 --- a/.github/workflows/publish_test_build.yml +++ b/.github/workflows/publish_test_build.yml @@ -8,8 +8,12 @@ on: paths-ignore: - 'version.py' +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/neon-skills + jobs: - build_and_publish: + increment_version: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -30,3 +34,60 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Increment Version + build_and_publish_docker: + runs-on: ubuntu-latest + needs: increment_version + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Get Version + id: version + run: | + VERSION=$(sed "s/a/-a./" <<< $(python setup.py --version)) + echo ::set-output name=version::${VERSION} + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for base Docker + id: base_meta + uses: docker/metadata-action@v2 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{ steps.version.outputs.version }} + type=ref,event=branch + - name: Build and push base Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.base_meta.outputs.tags }} + labels: ${{ steps.base_meta.outputs.labels }} + target: base + + - name: Extract metadata for default_skills Docker + id: default_skills_meta + uses: docker/metadata-action@v2 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-default_skills + tags: | + type=semver,pattern={{version}},value=${{ steps.version.outputs.version }} + type=ref,event=branch + - name: Build and push default_skills Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.default_skills_meta.outputs.tags }} + labels: ${{ steps.default_skills_meta.outputs.labels }} + target: default_skills \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d362540c3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.8-slim as base + +LABEL vendor=neon.ai \ + ai.neon.name="neon-skills" + +ENV NEON_CONFIG_PATH /config + +RUN apt-get update && \ + apt-get install -y \ + gcc \ + g++ \ + python3-dev \ + swig \ + libssl-dev \ + libfann-dev \ + portaudio19-dev \ + libsndfile1 \ + libpulse-dev \ + ffmpeg \ + git # TODO: git required for getting scripts, skill should be refactored to remove this dependency + +ADD . /neon_core +WORKDIR /neon_core + +COPY docker_overlay/asoundrc /root/.asoundrc + +RUN pip install wheel && \ + pip install . + +CMD ["neon_skills_service"] + +FROM base as default_skills + +RUN mkdir -p /root/.config/neon +RUN mkdir -p /root/.local/share/neon +COPY docker_overlay/skill_settings /root/.config/neon/skills +COPY docker_overlay/ngi_local_conf.yml /config/ +RUN neon-install-default-skills +RUN rm /config/ngi_local_conf.yml \ No newline at end of file diff --git a/README.md b/README.md index b43f5632b..affe2ea4f 100644 --- a/README.md +++ b/README.md @@ -448,3 +448,19 @@ where Neon may have saved files: You may now [re-install Neon](#installing-neon) > *Note:* You may need your [credential files](#optional-service-account-setup) to complete re-installation. + +# Running Docker Modules + +Skills Service +```shell +docker run -d \ +--name=neon_skills \ +--network=host \ +-v ~/.config/pulse/cookie:/root/.config/pulse/cookie:ro \ +-v ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro \ +-v ${NEON_CONFIG_DIR}:/config \ +--device=/dev/snd:/dev/snd \ +-e PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native \ +neon_skills +``` +>*Note:* The above example assumes `NEON_CONFIG_DIR` contains valid configuration diff --git a/docker_overlay/asoundrc b/docker_overlay/asoundrc new file mode 100644 index 000000000..190f790ed --- /dev/null +++ b/docker_overlay/asoundrc @@ -0,0 +1,2 @@ +pcm.default pulse +ctl.default pulse \ No newline at end of file diff --git a/docker_overlay/mycroft.conf b/docker_overlay/mycroft.conf new file mode 100644 index 000000000..95e67d975 --- /dev/null +++ b/docker_overlay/mycroft.conf @@ -0,0 +1,5 @@ +{ + "play_wav_cmdline": "play %1", + "play_mp3_cmdline": "play %1", + "play_ogg_cmdline": "play %1" +} diff --git a/docker_overlay/ngi_local_conf.yml b/docker_overlay/ngi_local_conf.yml new file mode 100644 index 000000000..72fb1ba4b --- /dev/null +++ b/docker_overlay/ngi_local_conf.yml @@ -0,0 +1,7 @@ +skills: + essential_skills: [] + default_skills: https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS-DOCKER + skill_manager: osm + +dirVars: + skillsDir: /root/.local/share/neon/skills \ No newline at end of file diff --git a/docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json b/docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json new file mode 100644 index 000000000..21bbcf5e0 --- /dev/null +++ b/docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json @@ -0,0 +1,7 @@ +{ + "weather_skill": "skill-weather.neongeckocom", + "datetime_skill": "skill-date_time.neongeckocom", + "examples_skill": "skill-about.neongeckocom", + "wallpaper": "default.jpg", + "__mycroft_skill_firstrun": false +} diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a97ad482e..1ffeb0f23 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,7 +3,7 @@ ovos-core[skills]==0.0.1 mock_msm # utils -neon-utils~=0.12,>=0.14.3a7 +neon-utils~=0.12,>=0.14.3a15 # TODO: Update to stable version ovos_utils~=0.0.12 ovos-skills-manager~=0.0.8 From c757a7e2fd54b6410059cd92d58e295ed3c35feb Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 2 Feb 2022 01:04:44 +0000 Subject: [PATCH 27/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index c3c711e10..e7291e5ee 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a11" +__version__ = "21.10.2a12" From 59429bd8fa096708dfece62671236e77f32726e5 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Tue, 1 Feb 2022 17:44:02 -0800 Subject: [PATCH 28/70] Update image name in automation to match other core modules (#206) Co-authored-by: Daniel McKnight --- .github/workflows/publish_release.yml | 4 ++++ .github/workflows/publish_test_build.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index cc0bf6ffb..0f957e2da 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -6,6 +6,10 @@ on: branches: - master +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/neon_skills + jobs: tag_release: runs-on: ubuntu-latest diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml index 62776473c..7586a8055 100644 --- a/.github/workflows/publish_test_build.yml +++ b/.github/workflows/publish_test_build.yml @@ -10,7 +10,7 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository_owner }}/neon-skills + IMAGE_NAME: ${{ github.repository_owner }}/neon_skills jobs: increment_version: From e30aed3003e249643fe45e8d7242012ecbc84297 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 2 Feb 2022 01:44:22 +0000 Subject: [PATCH 29/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index e7291e5ee..60e0efbd8 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a12" +__version__ = "21.10.2a13" From bf430264c298a6dfb59d746877c496b5d65bdfff Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 3 Feb 2022 15:35:46 -0800 Subject: [PATCH 30/70] Remove `neon_gui_service` entrypoint that was moved to `neon_gui` module (#210) Co-authored-by: Daniel McKnight --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 6e7f4b3c2..60f2e5999 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ def get_requirements(requirements_filename: str): entry_points={ 'console_scripts': [ 'neon_skills_service=neon_core.skills.__main__:main', - 'neon_gui_service=neon_core.gui.__main__:main', 'neon-install-default-skills=neon_core.util.skill_utils:install_skills_default', 'neon-upload-diagnostics=neon_core.util.diagnostic_utils:cli_send_diags', 'neon-start=neon_core.run_neon:start_neon', From 0f3da1a2d8a0365f9aaaa8c9cf72c2d3c4060830 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 3 Feb 2022 23:36:03 +0000 Subject: [PATCH 31/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 60e0efbd8..099d6f5f8 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a13" +__version__ = "21.10.2a14" From 9320b16127d8bc4eb8b9c6eab1cb75550ac424c8 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 17 Feb 2022 17:49:04 -0800 Subject: [PATCH 32/70] Refactor SkillManager (#208) * Refactor to deprecate override of `load_priority` method and remove `MsmException` handling * Deprecate _get_skill_directories override and update unit tests to test skills load * Fix logical error in unit test update Co-authored-by: Daniel McKnight --- neon_core/skills/service.py | 20 ++++------- neon_core/skills/skill_manager.py | 58 ++----------------------------- test/test_run_neon.py | 1 + 3 files changed, 10 insertions(+), 69 deletions(-) diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index b3b8ae75b..582d7809f 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -41,7 +41,6 @@ from mycroft.skills.api import SkillApi from mycroft.skills.event_scheduler import EventScheduler -from mycroft.skills.msm_wrapper import MsmException from mycroft.configuration.locale import set_default_lang, set_default_tz from mycroft.util.process_utils import ProcessStatus, StatusCallbackMap @@ -147,18 +146,13 @@ def _initialize_skill_manager(self): Returns: SkillManager instance or None if it couldn't be initialized """ - try: - self.skill_manager = NeonSkillManager(self.bus, self.watchdog) - self.skill_manager.load_priority() - except MsmException: - # skill manager couldn't be created, wait for network connection and - # retry - self.skill_manager = None - LOG.info( - 'MSM is uninitialized and requires network connection to fetch ' - 'skill information\nWill retry after internet connection is ' - 'established.' - ) + self.skill_manager = NeonSkillManager(self.bus, self.watchdog) + + # TODO: This config patching should be handled in neon_utils + self.skill_manager.config["skills"]["priority_skills"] = \ + self.skill_manager.config["skills"].get("priority") or \ + self.skill_manager.config["skills"]["priority_skills"] + self.skill_manager.load_priority() def _wait_for_internet_connection(self): if get_neon_skills_config().get("wait_for_internet", True): diff --git a/neon_core/skills/skill_manager.py b/neon_core/skills/skill_manager.py index 931291bab..e2051b3d0 100644 --- a/neon_core/skills/skill_manager.py +++ b/neon_core/skills/skill_manager.py @@ -23,12 +23,11 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os - -from glob import glob from neon_utils.configuration_utils import get_neon_skills_config from neon_utils.log_utils import LOG + from neon_core.skills.skill_store import SkillsStore + from mycroft.util import connected from mycroft.skills.skill_manager import SkillManager @@ -59,38 +58,6 @@ def download_or_update_defaults(self): # if no internet just skip this update LOG.error("no internet, skipped default skills installation") - def load_priority(self): - # NOTE: mycroft uses the skill name, this is not deterministic! msm - # decides what the name is based on the meta info from selene, - # if missing it uses folder name (skill_id), for backwards compat - # name is still support but skill_id is recommended! the name can be - # changed at any time and mess up the .conf, if the skill_id changes - # lots of other stuff will break so you are assured to notice - # TODO deprecate usage of skill_name once mycroft catches up - skills = {skill.name: skill for skill in self.msm.all_skills} - skill_ids = {os.path.basename(skill.path): skill - for skill in self.msm.all_skills} - priority_skills = self.skill_config.get("priority", []) - for skill_name in priority_skills: - skill = skill_ids.get(skill_name) or skills.get(skill_name) - if skill is not None: - if not skill.is_local: - try: - self.msm.install(skill) - except Exception as e: - LOG.exception(f"Downloading priority skill: {skill_name} failed") - LOG.error(e) - continue - loader = self._load_skill(skill.path) - if loader: - self.upload_queue.put(loader) - else: - LOG.error( - 'Priority skill {} can\'t be found'.format(skill_name) - ) - - self._alive_status = True - def run(self): """Load skills and update periodically from disk and internet.""" self.download_or_update_defaults() @@ -103,24 +70,3 @@ def _emit_converse_error(self, message, skill_id, error_msg): reply = message.reply('skill.converse.error', data=dict(skill_id=skill_id, error=error_msg)) self.bus.emit(reply) - - def _get_skill_directories(self): - """ - Locates all skill directories in the configured skill install path - """ - # TODO: Integrate this with OSM local appstores DM - base_skill_dir = glob(os.path.join(self.skill_config["directory"], "*/")) - skill_directories = [] - for skill_dir in base_skill_dir: - # TODO: all python packages must have __init__.py! Better way? - # check if folder is a skill (must have __init__.py) - if SKILL_MAIN_MODULE in os.listdir(skill_dir): - skill_directories.append(skill_dir.rstrip('/')) - if skill_dir in self.empty_skill_dirs: - self.empty_skill_dirs.discard(skill_dir) - else: - if skill_dir not in self.empty_skill_dirs: - self.empty_skill_dirs.add(skill_dir) - LOG.debug('Found skills directory with no skill: ' + - skill_dir) - return skill_directories diff --git a/test/test_run_neon.py b/test/test_run_neon.py index 45dd0c631..d485be8fb 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -153,6 +153,7 @@ def test_skills_module(self): self.assertIsInstance(response, Message) loaded_skills = response.data self.assertIsInstance(loaded_skills, dict) + self.assertGreater(len(loaded_skills.keys()), 1) # TODO: Test user utterance -> response From 24ac34e10a94ff632ae6253b42f4ece55c446d5b Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 18 Feb 2022 01:49:23 +0000 Subject: [PATCH 33/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 099d6f5f8..d80c26f12 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a14" +__version__ = "21.10.2a15" From acca2163e26e4038e8f26d99cae7d4cb04bf0721 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 24 Feb 2022 18:37:00 -0800 Subject: [PATCH 34/70] Update OVOS dependency versions (#212) * Fix dependency compatibility (#213) * Lock json_database dependency version * Restrict ovos-utils dependency version * Lock json-database dependency version * Lock OPM version for version compat. * Further restrict OPM dependency * Update json_database dependency to fulfill default skill requirements * Increment version Co-authored-by: Daniel McKnight (cherry picked from commit 611ec321de3981d7dff8afb21eef175dcf23850d) * Update ovos package dependency versions * Update ovos-core package dependency version * Troubleshoot ovos-core package dependency version * Troubleshoot ovos-core package dependency version Update unit tests to install wheel * Troubleshoot unit test setup * Refactor xdg reference to ovos_utils * Troubleshooting test errors * Refactor one-time config initialization to config module Formatting and code cleanup Update dependencies to test updated package versions * Fix ovos-core version resolution issue * Fix configuration test case Add json-database dependency for alerts skill compat * remove invalid json-database pinned version * Bump core module versions * Troubleshooting * Troubleshooting unit test failures * Troubleshooting automated tests * Increase Python version coverage in tests * Fix parse errors in unit_tests workflow * Fix parse errors in setup_tests workflow * Cleanup requirements Update unit tests * Troubleshoot ovos package compatible versions * Troubleshooting dependency versions and test failures * Testing updated TTS plugin * Update mimic plugin spec * Troubleshooting requirements versioning * Troubleshooting requirements versioning (OPM) Co-authored-by: Daniel McKnight --- .github/workflows/setup_tests.yml | 2 +- .github/workflows/unit_tests.yml | 12 +-- neon_core/__init__.py | 92 +--------------- neon_core/config.py | 127 +++++++++++++++++++++++ neon_core/configuration/__init__.py | 6 +- neon_core/skills/__main__.py | 3 +- neon_core/util/skill_utils.py | 10 +- requirements/core_modules.txt | 11 +- requirements/local_speech_processing.txt | 2 +- requirements/requirements.txt | 16 ++- test/test_configuration.py | 12 ++- version.py | 2 +- 12 files changed, 172 insertions(+), 123 deletions(-) create mode 100644 neon_core/config.py diff --git a/.github/workflows/setup_tests.yml b/.github/workflows/setup_tests.yml index f1dbc6644..f6e14b10b 100644 --- a/.github/workflows/setup_tests.yml +++ b/.github/workflows/setup_tests.yml @@ -9,7 +9,7 @@ jobs: remote: strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.7, 3.8, 3.9 ] runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 9043e9de0..8e24c9384 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -26,7 +26,7 @@ jobs: util_tests: strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.6, 3.7, 3.8, 3.9 ] runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -39,8 +39,8 @@ jobs: run: | sudo apt update sudo apt install -y gcc libfann-dev swig libssl-dev portaudio19-dev git libpulse-dev - pip install -r requirements/requirements.txt - pip install -r requirements/test.txt + pip install wheel + pip install . -r requirements/test.txt env: GITHUB_TOKEN: ${{secrets.neon_token}} @@ -78,7 +78,7 @@ jobs: unit_tests: strategy: matrix: - python-version: [ 3.6, 3.7, 3.8 ] + python-version: [ 3.6, 3.7, 3.8, 3.9, '3.10' ] runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -91,8 +91,8 @@ jobs: run: | sudo apt update sudo apt install -y gcc libfann-dev swig libssl-dev portaudio19-dev git libpulse-dev - pip install -r requirements/requirements.txt - pip install -r requirements/test.txt + pip install wheel + pip install . -r requirements/test.txt env: GITHUB_TOKEN: ${{secrets.neon_token}} diff --git a/neon_core/__init__.py b/neon_core/__init__.py index 74420f4ba..5026a976f 100644 --- a/neon_core/__init__.py +++ b/neon_core/__init__.py @@ -23,98 +23,12 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from os.path import join, dirname -import xdg.BaseDirectory -import json +from neon_core.config import init_config +from os.path import dirname -from ovos_utils.json_helper import merge_dict -from ovos_utils.system import set_root_path -from ovos_utils.configuration import set_config_name - -from neon_utils import LOG NEON_ROOT_PATH = dirname(dirname(__file__)) - - -def setup_ovos_core_config(): - """ - Runs at module init to ensure base ovos.conf exists to patch ovos-core. Note that this must run before any import - of Configuration class. - """ - OVOS_CONFIG = join(xdg.BaseDirectory.save_config_path("OpenVoiceOS"), - "ovos.conf") - - _NEON_OVOS_CONFIG = { - "module_overrides": { - "neon_core": { - "xdg": True, - "base_folder": "neon", - "config_filename": "neon.conf", - "default_config_path": join(dirname(__file__), - 'configuration', 'neon.conf') - } - }, - # if these services are running standalone (neon_core not in venv) - # config them to use neon_core config from above - "submodule_mappings": { - "neon_speech": "neon_core", - "neon_audio": "neon_core", - "neon_enclosure": "neon_core" - } - } - - cfg = {} - try: - with open(OVOS_CONFIG) as f: - cfg = json.load(f) - except FileNotFoundError: - pass - except Exception as e: - LOG.error(e) - - cfg = merge_dict(cfg, _NEON_OVOS_CONFIG) - with open(OVOS_CONFIG, "w") as f: - json.dump(cfg, f, indent=4, ensure_ascii=True) - - -def setup_ovos_config(): - """ - Configure ovos_utils to read from neon.conf files and set this path as the root. - """ - # TODO: This method will be handled in ovos-core directly in the future - # ensure ovos_utils can find neon_core - set_root_path(NEON_ROOT_PATH) - # make ovos_utils load the proper .conf files - set_config_name("neon.conf", "neon_core") - - -setup_ovos_config() - -# make ovos-core Configuration.get() load neon.conf -# TODO ovos-core does not yet support yaml configs, once it does -# Configuration.get() will be made to load the existing neon config files, -# for now it simply provides correct default values -setup_ovos_core_config() - -from neon_utils.configuration_utils import write_mycroft_compatible_config, init_config_dir -init_config_dir() - -# Write and reload Mycroft-compat conf file -neon_config_path = join(xdg.BaseDirectory.save_config_path("neon"), - "neon.conf") -write_mycroft_compatible_config(neon_config_path) -from neon_core.configuration import Configuration -Configuration.load_config_stack(cache=True, remote=False) - - -# TODO: Consider when this log is valid/config is changed or not already synced with neon_config DM -LOG.info(f"{neon_config_path} will be overwritten with Neon YAML config contents.") - -# patch version string to allow downstream to know where it is running -import mycroft.version -CORE_VERSION_STR = '.'.join(map(str, mycroft.version.CORE_VERSION_TUPLE)) + \ - "(NeonGecko)" -mycroft.version.CORE_VERSION_STR = CORE_VERSION_STR +CORE_VERSION_STR = init_config(NEON_ROOT_PATH) from neon_core.skills import NeonSkill, NeonFallbackSkill diff --git a/neon_core/config.py b/neon_core/config.py new file mode 100644 index 000000000..89db8a04d --- /dev/null +++ b/neon_core/config.py @@ -0,0 +1,127 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import os + +from os.path import join, dirname +from ovos_utils.json_helper import merge_dict +from ovos_utils.system import set_root_path +from ovos_utils.configuration import set_config_name +from ovos_utils.xdg_utils import xdg_config_home +from neon_utils.configuration_utils import write_mycroft_compatible_config,\ + init_config_dir +from neon_utils.logger import LOG + + +def setup_ovos_core_config(): + """ + Runs at module init to ensure base ovos.conf exists to patch ovos-core. + Note that this must run before any import of Configuration class. + """ + ovos_config_path = join(xdg_config_home(), "OpenVoiceOS", "ovos.conf") + + neon_default_config = { + "module_overrides": { + "neon_core": { + "xdg": True, + "base_folder": "neon", + "config_filename": "neon.conf", + "default_config_path": join(dirname(__file__), + 'configuration', 'neon.conf') + } + }, + # if these services are running standalone (neon_core not in venv) + # config them to use neon_core config from above + "submodule_mappings": { + "neon_speech": "neon_core", + "neon_audio": "neon_core", + "neon_enclosure": "neon_core" + } + } + + cfg = {} + try: + with open(ovos_config_path) as f: + cfg = json.load(f) + except FileNotFoundError: + pass + except Exception as e: + LOG.error(e) + + cfg = merge_dict(cfg, neon_default_config) + with open(ovos_config_path, "w") as f: + json.dump(cfg, f, indent=4, ensure_ascii=True) + + +def setup_ovos_config(neon_root_path): + """ + Configure ovos_utils to read from neon.conf files and set this path as the root. + """ + # TODO: This method will be handled in ovos-core directly in the future + # ensure ovos_utils can find neon_core + set_root_path(neon_root_path) + # make ovos_utils load the proper .conf files + set_config_name("neon.conf", "neon_core") + + +def setup_neon_system_config(): + """ + Ensure default neon config file is specified in envvars + """ + config_home = join(xdg_config_home(), "neon") + config_file = join(config_home, "neon.conf") + if not os.path.isdir(config_home): + os.makedirs(config_home) + os.environ["MYCROFT_SYSTEM_CONFIG"] = config_file + + +def init_config(neon_root_path): + setup_ovos_config(neon_root_path) + setup_neon_system_config() + # make ovos-core Configuration.get() load neon.conf + # TODO ovos-core does not yet support yaml configs, once it does + # Configuration.get() will be made to load the existing neon config files, + # for now it simply provides correct default values + setup_ovos_core_config() + + init_config_dir() + + # Write and reload Mycroft-compat conf file + neon_config_path = join(xdg_config_home(), "neon", "neon.conf") + # TODO: Consider when this log is valid/config is changed or not already synced with neon_config DM + LOG.info(f"{neon_config_path} will be overwritten with Neon YAML config") + write_mycroft_compatible_config(neon_config_path) + + from neon_core.configuration import Configuration + Configuration.load_config_stack(cache=True, remote=False) + + # patch version string to allow downstream to know where it is running + import mycroft.version + core_version_str = '.'.join(map(str, + mycroft.version.CORE_VERSION_TUPLE)) + \ + "(NeonGecko)" + mycroft.version.CORE_VERSION_STR = core_version_str + return core_version_str diff --git a/neon_core/configuration/__init__.py b/neon_core/configuration/__init__.py index 37cc663a0..b434a72b9 100644 --- a/neon_core/configuration/__init__.py +++ b/neon_core/configuration/__init__.py @@ -23,10 +23,8 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from mycroft.configuration.config import Configuration, LocalConf +from mycroft.configuration.config import Configuration + def get_private_keys(): return Configuration.get(remote=False).get("keys", {}) - - - diff --git a/neon_core/skills/__main__.py b/neon_core/skills/__main__.py index 4b473adf7..6f1c39795 100644 --- a/neon_core/skills/__main__.py +++ b/neon_core/skills/__main__.py @@ -23,9 +23,10 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from neon_core.skills.service import NeonSkillService + from mycroft.lock import Lock from mycroft.util import reset_sigint_handler, wait_for_exit_signal -from neon_core.skills.service import NeonSkillService def main(*args, **kwargs): diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index 24d7952e1..d2534d580 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -24,6 +24,8 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json +import os.path + import requests from os import listdir @@ -90,9 +92,13 @@ def install_skills_from_list(skills_to_install: list, config: dict = None): else: LOG.warning(f"Requested Skill not in Neon skill store ({url})") entry = osm.skill_entry_from_url(url) - # try: + osm.install_skill(entry, skill_dir) - LOG.info(f"Installed {url} to {skill_dir}") + if not os.path.isdir(os.path.join(skill_dir, entry.uuid)): + LOG.error(f"Failed to install: " + f"{os.path.join(skill_dir, entry.uuid)}") + else: + LOG.info(f"Installed {url} to {skill_dir}") except Exception as e: LOG.error(e) if token_set: diff --git a/requirements/core_modules.txt b/requirements/core_modules.txt index 13d8bfa8c..32dca088e 100644 --- a/requirements/core_modules.txt +++ b/requirements/core_modules.txt @@ -1,9 +1,14 @@ -ovos-core[all]~=0.0.1 +# Patching compat with extra-stt +# ovos-core[all]~=0.0.2a5 +ovos-core[audio-backend,PHAL,tts,skills_lgpl,gui,bus]~=0.0.2a5 +SpeechRecognition~=3.8.1 +PyAudio~=0.2.11 +ovos-ww-plugin-pocketsphinx~=0.1.2 # neon core modules neon_enclosure~=0.1,>=0.1.2 neon_messagebus -neon_speech~=0.3 -neon_audio~=0.4 +neon_speech~=0.3,>=0.3.3a4 +neon_audio~=0.4,>=0.4.3a4 neon_gui # TODO: Version spec GUI \ No newline at end of file diff --git a/requirements/local_speech_processing.txt b/requirements/local_speech_processing.txt index 1fa8259da..b08985275 100644 --- a/requirements/local_speech_processing.txt +++ b/requirements/local_speech_processing.txt @@ -1,2 +1,2 @@ neon-stt-plugin-deepspeech-stream-local~=0.1 -neon-tts-plugin-mimic~=0.1 +neon-tts-plugin-mimic~=0.1,>=0.1.7 \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1ffeb0f23..09d531921 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,16 +1,12 @@ # mycroft -ovos-core[skills]==0.0.1 -mock_msm +ovos-core[skills_lgpl]~=0.0.2a5 # utils -neon-utils~=0.12,>=0.14.3a15 -# TODO: Update to stable version -ovos_utils~=0.0.12 -ovos-skills-manager~=0.0.8 - -# TODO: Patching OPM -ovos-plugin-manager==0.0.2 -SpeechRecognition~=3.8 +neon-utils~=0.12,>=0.15.1a1 +ovos_utils~=0.0.15,>=0.0.17a1 +ovos-skills-manager~=0.0.10a20 +ovos-plugin-manager~=0.0.4,>=0.0.7a0 +# TODO: Update to stable versions # plugins neon-lang-plugin-libretranslate~=0.1,>=0.1.2 diff --git a/test/test_configuration.py b/test/test_configuration.py index c4b788f70..e99adcfa9 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -24,12 +24,9 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import shutil import sys import unittest -from ovos_plugin_manager.templates.language import LanguageDetector, LanguageTranslator - from neon_utils.configuration_utils import get_mycroft_compatible_config sys.path.append(os.path.dirname(os.path.dirname(__file__))) @@ -44,7 +41,8 @@ def test_neon_core_config_init(self): if isinstance(val, dict): for k, v in val.items(): if not isinstance(v, dict): - self.assertEqual(neon_compat_config[key][k], v, neon_compat_config[key]) + self.assertEqual(neon_compat_config[key][k], + v, neon_compat_config[key]) else: self.assertEqual(neon_compat_config[key], val) @@ -56,11 +54,15 @@ def test_ovos_core_config_init(self): if isinstance(val, dict): for k, v in val.items(): if not isinstance(v, dict): - self.assertEqual(mycroft_config[key][k], v, mycroft_config[key]) + self.assertEqual(mycroft_config[key][k], + v, mycroft_config[key]) else: self.assertEqual(mycroft_config[key], val) def test_signal_dir(self): + import neon_core + + self.assertIsNotNone(os.environ.get("MYCROFT_SYSTEM_CONFIG")) from neon_utils.skill_override_functions import IPC_DIR as neon_ipc_dir from ovos_utils.signal import get_ipc_directory as ovos_ipc_dir from mycroft.util.signal import get_ipc_directory as mycroft_ipc_dir diff --git a/version.py b/version.py index d80c26f12..5bfa2f83a 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2a15" +__version__ = "21.10.2" From 4385353285d9dbcab912979ec553b63a0e5b1fee Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 25 Feb 2022 02:37:24 +0000 Subject: [PATCH 35/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 5bfa2f83a..4ed2d43cd 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.2" +__version__ = "21.10.3a0" From 0f6ce990f2ef23678bdf7211306f5441c92215ab Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 25 Feb 2022 15:25:22 -0800 Subject: [PATCH 36/70] Fix configuration test failures (#215) * Troubleshooting unit test failure * Refactor config.py and fix first-run config errors * Update method name and docstring for clarity * Debug GH test failures * Ensure OVOS config path exists before writing config * Fix typo in config dir creation * Troubleshooting config unit test failures * Refactor configuration tests to better match real-world execution * Cleanup imports Ensure latest ovos_utils * Add ovos_config_assistant log outputs to debug test failures * More debugging of config read * Troubleshooting module detection * Cleanup config initialization * Troubleshoot setup test failures * Remove log added to function in neon-utils Co-authored-by: Daniel McKnight --- .github/workflows/setup_tests.yml | 2 +- .github/workflows/unit_tests.yml | 6 ++++ neon_core/__init__.py | 10 ++++-- neon_core/config.py | 59 ++++++++++++++++++------------- requirements/requirements.txt | 4 +-- requirements/test.txt | 3 +- test/test_configuration.py | 27 +++++++++++--- 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/.github/workflows/setup_tests.yml b/.github/workflows/setup_tests.yml index f6e14b10b..1a620ebe2 100644 --- a/.github/workflows/setup_tests.yml +++ b/.github/workflows/setup_tests.yml @@ -9,7 +9,7 @@ jobs: remote: strategy: matrix: - python-version: [ 3.7, 3.8, 3.9 ] + python-version: [ 3.7, 3.9 ] runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8e24c9384..0a707619e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -104,6 +104,12 @@ jobs: with: name: configuration-test-results path: tests/configuration-test-results.xml + - name: Upload config file + uses: actions/upload-artifact@v2 + with: + name: ovos.conf + path: ~/.config/OpenVoiceOS/ovos.conf + if: always() - name: Test Language run: | diff --git a/neon_core/__init__.py b/neon_core/__init__.py index 5026a976f..62d942b91 100644 --- a/neon_core/__init__.py +++ b/neon_core/__init__.py @@ -23,12 +23,16 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from neon_core.config import init_config +import sys + +from neon_core.config import init_config, get_core_version from os.path import dirname -NEON_ROOT_PATH = dirname(dirname(__file__)) -CORE_VERSION_STR = init_config(NEON_ROOT_PATH) +NEON_ROOT_PATH = dirname(__file__) +sys.path.append(NEON_ROOT_PATH) +init_config() +CORE_VERSION_STR = get_core_version() from neon_core.skills import NeonSkill, NeonFallbackSkill diff --git a/neon_core/config.py b/neon_core/config.py index 89db8a04d..f61e1ce6f 100644 --- a/neon_core/config.py +++ b/neon_core/config.py @@ -29,10 +29,8 @@ from os.path import join, dirname from ovos_utils.json_helper import merge_dict from ovos_utils.system import set_root_path -from ovos_utils.configuration import set_config_name from ovos_utils.xdg_utils import xdg_config_home -from neon_utils.configuration_utils import write_mycroft_compatible_config,\ - init_config_dir + from neon_utils.logger import LOG @@ -71,22 +69,21 @@ def setup_ovos_core_config(): except Exception as e: LOG.error(e) + if cfg == neon_default_config: + # Skip merge/write config if it's already equivalent + return + disk_cfg = dict(cfg) cfg = merge_dict(cfg, neon_default_config) - with open(ovos_config_path, "w") as f: + if disk_cfg == cfg: + # Skip write config if it's already equivalent + return + if not os.path.isdir(dirname(ovos_config_path)): + os.makedirs(dirname(ovos_config_path)) + LOG.info(f"Writing config file: {ovos_config_path}") + with open(ovos_config_path, "w+") as f: json.dump(cfg, f, indent=4, ensure_ascii=True) -def setup_ovos_config(neon_root_path): - """ - Configure ovos_utils to read from neon.conf files and set this path as the root. - """ - # TODO: This method will be handled in ovos-core directly in the future - # ensure ovos_utils can find neon_core - set_root_path(neon_root_path) - # make ovos_utils load the proper .conf files - set_config_name("neon.conf", "neon_core") - - def setup_neon_system_config(): """ Ensure default neon config file is specified in envvars @@ -98,23 +95,37 @@ def setup_neon_system_config(): os.environ["MYCROFT_SYSTEM_CONFIG"] = config_file -def init_config(neon_root_path): - setup_ovos_config(neon_root_path) +def overwrite_neon_conf(): + """ + Write over neon.conf file with Neon configuration + """ + from neon_utils.configuration_utils import \ + write_mycroft_compatible_config, init_config_dir + init_config_dir() + + # Write Mycroft-compat conf file + neon_config_path = join(xdg_config_home(), "neon", "neon.conf") + write_mycroft_compatible_config(neon_config_path) + + +def init_config(): + """ + Initialize all configuration methods to read from the same config + """ setup_neon_system_config() # make ovos-core Configuration.get() load neon.conf # TODO ovos-core does not yet support yaml configs, once it does # Configuration.get() will be made to load the existing neon config files, # for now it simply provides correct default values setup_ovos_core_config() + overwrite_neon_conf() - init_config_dir() - - # Write and reload Mycroft-compat conf file - neon_config_path = join(xdg_config_home(), "neon", "neon.conf") - # TODO: Consider when this log is valid/config is changed or not already synced with neon_config DM - LOG.info(f"{neon_config_path} will be overwritten with Neon YAML config") - write_mycroft_compatible_config(neon_config_path) +def get_core_version() -> str: + """ + Get the core version string. + NOTE: `init_config` should be called before this method + """ from neon_core.configuration import Configuration Configuration.load_config_stack(cache=True, remote=False) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 09d531921..920f45caf 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,8 +2,8 @@ ovos-core[skills_lgpl]~=0.0.2a5 # utils -neon-utils~=0.12,>=0.15.1a1 -ovos_utils~=0.0.15,>=0.0.17a1 +neon-utils~=0.12,>=0.15.1a7 +ovos_utils~=0.0.18 ovos-skills-manager~=0.0.10a20 ovos-plugin-manager~=0.0.4,>=0.0.7a0 # TODO: Update to stable versions diff --git a/requirements/test.txt b/requirements/test.txt index 6ec7a7632..25769044a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,5 @@ pytest pytest-cov mock==4.0.2 -neon-lang-plugin-libretranslate>=0.1.2 \ No newline at end of file +neon-lang-plugin-libretranslate>=0.1.2 +ovos-config-assistant \ No newline at end of file diff --git a/test/test_configuration.py b/test/test_configuration.py index e99adcfa9..fa9a6c1cb 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -26,14 +26,33 @@ import os import sys import unittest +from pprint import pformat -from neon_utils.configuration_utils import get_mycroft_compatible_config - +from neon_utils.logger import LOG sys.path.append(os.path.dirname(os.path.dirname(__file__))) class ConfigurationTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + from ovos_config_assistant import config_helpers + + ovos_config = os.path.expanduser("~/.config/OpenVoiceOS/ovos.conf") + if os.path.isfile(ovos_config): + os.remove(ovos_config) + assert config_helpers.get_ovos_default_config_paths() == [] + + import neon_core + assert isinstance(neon_core.CORE_VERSION_STR, str) + assert len(config_helpers.get_ovos_default_config_paths()) == 1 + LOG.info(config_helpers.get_ovos_default_config_paths()) + ovos_config = config_helpers.get_ovos_config() + LOG.info(pformat(ovos_config)) + assert ovos_config['config_filename'] == 'neon.conf' + def test_neon_core_config_init(self): + from neon_utils.configuration_utils import \ + get_mycroft_compatible_config from neon_core.configuration import Configuration neon_compat_config = Configuration.get() neon_config = get_mycroft_compatible_config() @@ -47,6 +66,8 @@ def test_neon_core_config_init(self): self.assertEqual(neon_compat_config[key], val) def test_ovos_core_config_init(self): + from neon_utils.configuration_utils import \ + get_mycroft_compatible_config from mycroft.configuration import Configuration as MycroftConfig mycroft_config = MycroftConfig.get() neon_config = get_mycroft_compatible_config() @@ -60,8 +81,6 @@ def test_ovos_core_config_init(self): self.assertEqual(mycroft_config[key], val) def test_signal_dir(self): - import neon_core - self.assertIsNotNone(os.environ.get("MYCROFT_SYSTEM_CONFIG")) from neon_utils.skill_override_functions import IPC_DIR as neon_ipc_dir from ovos_utils.signal import get_ipc_directory as ovos_ipc_dir From 6cdb073ade4cb44b9817c05811b360d61978b0b0 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 25 Feb 2022 23:25:40 +0000 Subject: [PATCH 37/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 4ed2d43cd..e1369a9b8 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a0" +__version__ = "21.10.3a1" From b1fc4ff3f569c99e0489ac209dbb732ca87e0314 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 25 Feb 2022 16:22:33 -0800 Subject: [PATCH 38/70] Update docker overlay files for ovos compat (#216) * Update docker overlay files for ovos compat * bump ovos-core dependency spec Co-authored-by: Daniel McKnight --- Dockerfile | 4 +++- docker_overlay/mycroft.conf | 5 ----- docker_overlay/neon.conf | 8 ++++++++ requirements/requirements.txt | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) delete mode 100644 docker_overlay/mycroft.conf create mode 100644 docker_overlay/neon.conf diff --git a/Dockerfile b/Dockerfile index d362540c3..8b3583142 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,9 @@ WORKDIR /neon_core COPY docker_overlay/asoundrc /root/.asoundrc +RUN mkdir -p /root/.config/neon +COPY docker_overlay/neon.conf /root/.config/neon/neon.conf + RUN pip install wheel && \ pip install . @@ -31,7 +34,6 @@ CMD ["neon_skills_service"] FROM base as default_skills -RUN mkdir -p /root/.config/neon RUN mkdir -p /root/.local/share/neon COPY docker_overlay/skill_settings /root/.config/neon/skills COPY docker_overlay/ngi_local_conf.yml /config/ diff --git a/docker_overlay/mycroft.conf b/docker_overlay/mycroft.conf deleted file mode 100644 index 95e67d975..000000000 --- a/docker_overlay/mycroft.conf +++ /dev/null @@ -1,5 +0,0 @@ -{ - "play_wav_cmdline": "play %1", - "play_mp3_cmdline": "play %1", - "play_ogg_cmdline": "play %1" -} diff --git a/docker_overlay/neon.conf b/docker_overlay/neon.conf new file mode 100644 index 000000000..9e0dd4a79 --- /dev/null +++ b/docker_overlay/neon.conf @@ -0,0 +1,8 @@ +{ + "play_wav_cmdline": "play %1", + "play_mp3_cmdline": "play %1", + "play_ogg_cmdline": "play %1", + "skills": { + "extra_directories": ["/skills"] + } +} diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 920f45caf..ce2e8551e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ # mycroft -ovos-core[skills_lgpl]~=0.0.2a5 +ovos-core[skills_lgpl]~=0.0.2a10 # utils neon-utils~=0.12,>=0.15.1a7 From c3b95bf6cc88cf1b98d8a5f917bdab4246ae5e1a Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Sat, 26 Feb 2022 00:22:50 +0000 Subject: [PATCH 39/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index e1369a9b8..886d12e1e 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a1" +__version__ = "21.10.3a2" From 6d4b582fb518ea34ffb9c1e1c19c0b790ea661c9 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 28 Feb 2022 10:20:28 -0800 Subject: [PATCH 40/70] Add function to install local skill dependencies with unit tests (#217) * Add util to install local skill dependencies with unit tests * Refactor dependencies and imports * Cleanup test method overrides * Handle reload of mocked modules in tests * Rollback unrelated language module change Co-authored-by: Daniel McKnight --- neon_core/util/skill_utils.py | 48 ++- requirements/requirements.txt | 2 +- test/local_skills/skill-about/LICENSE.md | 21 ++ test/local_skills/skill-about/README.md | 53 +++ test/local_skills/skill-about/__init__.py | 107 ++++++ .../local_skills/skill-about/requirements.txt | 1 + test/local_skills/skill-about/skill.json | 48 +++ test/local_skills/skill-osm_parsing/LICENSE | 29 ++ test/local_skills/skill-osm_parsing/README.md | 34 ++ .../skill-osm_parsing/__init__.py | 8 + .../skill-osm_parsing/manifest.yml | 7 + .../skill-osm_parsing/requirements.txt | 1 + .../local_skills/skill-osm_parsing/skill.json | 51 +++ .../skill-ovos-homescreen/LICENSE | 201 +++++++++++ .../skill-ovos-homescreen/MANIFEST.in | 5 + .../skill-ovos-homescreen/README.md | 3 + .../skill-ovos-homescreen/__init__.py | 337 ++++++++++++++++++ .../skill-ovos-homescreen/requirements.txt | 3 + .../skill-ovos-homescreen/settingsmeta.yml | 22 ++ .../skill-ovos-homescreen/setup.py | 26 ++ test/test_skill_utils.py | 73 +++- 21 files changed, 1072 insertions(+), 8 deletions(-) create mode 100755 test/local_skills/skill-about/LICENSE.md create mode 100755 test/local_skills/skill-about/README.md create mode 100755 test/local_skills/skill-about/__init__.py create mode 100755 test/local_skills/skill-about/requirements.txt create mode 100755 test/local_skills/skill-about/skill.json create mode 100644 test/local_skills/skill-osm_parsing/LICENSE create mode 100644 test/local_skills/skill-osm_parsing/README.md create mode 100644 test/local_skills/skill-osm_parsing/__init__.py create mode 100644 test/local_skills/skill-osm_parsing/manifest.yml create mode 100644 test/local_skills/skill-osm_parsing/requirements.txt create mode 100644 test/local_skills/skill-osm_parsing/skill.json create mode 100755 test/local_skills/skill-ovos-homescreen/LICENSE create mode 100755 test/local_skills/skill-ovos-homescreen/MANIFEST.in create mode 100755 test/local_skills/skill-ovos-homescreen/README.md create mode 100755 test/local_skills/skill-ovos-homescreen/__init__.py create mode 100755 test/local_skills/skill-ovos-homescreen/requirements.txt create mode 100755 test/local_skills/skill-ovos-homescreen/settingsmeta.yml create mode 100755 test/local_skills/skill-ovos-homescreen/setup.py diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index d2534d580..70a92a620 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -31,7 +31,9 @@ from os import listdir from tempfile import mkdtemp from shutil import rmtree -from os.path import expanduser, join +from os.path import expanduser, join, isdir + +from ovos_skills_manager.requirements import install_system_deps, pip_install from ovos_skills_manager.skill_entry import SkillEntry from ovos_skills_manager.osm import OVOSSkillsManager from ovos_skills_manager.session import SESSION, set_github_token, clear_github_token @@ -132,3 +134,47 @@ def get_remote_entries(url): else: LOG.error(f"{url} request failed with code: {r.status_code}") return [] + + +def _install_skill_dependencies(skill: SkillEntry): + """ + Install any system and Python dependencies for the specified skill + :param skill: Skill to install dependencies for + """ + sys_deps = skill.requirements.get("system") + requirements = skill.requirements.get("python") + if sys_deps: + install_system_deps(sys_deps) + if requirements: + pip_install(requirements) + LOG.info(f"Installed dependencies for {skill.skill_folder}") + + +def install_local_skills(local_skills_dir: str = "/skills") -> list: + """ + Install skill dependencies for skills in the specified directory and ensure + the directory is loaded. + NOTE: dependence on other skills is not handled here. + Only Python and System dependencies are handled + :param local_skills_dir: Directory to install skills from + :returns: list of installed skill directories + """ + local_skills_dir = expanduser(local_skills_dir) + if not isdir(local_skills_dir): + raise ValueError(f"{local_skills_dir} is not a valid directory") + installed_skills = list() + for skill in listdir(local_skills_dir): + if not isdir(skill): + pass + LOG.debug(f"Attempting installation of {skill}") + try: + entry = SkillEntry.from_directory(join(local_skills_dir, skill)) + _install_skill_dependencies(entry) + installed_skills.append(skill) + except Exception as e: + LOG.error(f"Exception while installing {skill}") + LOG.error(e) + if local_skills_dir not in \ + get_neon_skills_config().get("extra_directories", []): + LOG.error(f"{local_skills_dir} not found in configuration") + return installed_skills diff --git a/requirements/requirements.txt b/requirements/requirements.txt index ce2e8551e..21a429498 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -8,7 +8,7 @@ ovos-skills-manager~=0.0.10a20 ovos-plugin-manager~=0.0.4,>=0.0.7a0 # TODO: Update to stable versions -# plugins +# default plugins neon-lang-plugin-libretranslate~=0.1,>=0.1.2 # text parser modules diff --git a/test/local_skills/skill-about/LICENSE.md b/test/local_skills/skill-about/LICENSE.md new file mode 100755 index 000000000..525bb37e5 --- /dev/null +++ b/test/local_skills/skill-about/LICENSE.md @@ -0,0 +1,21 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 License + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/test/local_skills/skill-about/README.md b/test/local_skills/skill-about/README.md new file mode 100755 index 000000000..cc41cf775 --- /dev/null +++ b/test/local_skills/skill-about/README.md @@ -0,0 +1,53 @@ +# About + +## Summary + +Skill used to provide information about Neon + +## Requirements + +No special required packages for this skill. + +## Description + +This skill provides information about Neon. You can ask about licensing and your Neon installation. + +## Examples +- "neon tell me my license" +- "neon tell me my skills" + +## Location + + ${skills}/about.neon + +## Details + +### Text + + neon tell me my skills + >> You have the following skills installed: about, alerts, audio record, avmusic, caffeinewiz, + controls, custom conversation, date time, device control center, eliza 1965 chatbot, fallback duck + duck go, fallback unknown, fallback wolfram alpha, i like brands, i like coupons, ip address, + joke, launcher, messaging, mycroft pairing, openhab, personal, speak, speed test, spelling, stock, + stop, support helper, synonyms, translation, usb cam, volume, weather, wifi setup, wiki + + neon tell me my license + >> Copyright 2019 Neongecko Inc. All Rights Reserved. +### Picture + +### Video + + + +## Contact Support + +Use the [link](https://neongecko.com/ContactUs) or [submit an issue on GitHub](https://help.github.com/en/articles/creating-an-issue) + +## Credits +[NeonDaniel](https://github.com/NeonDaniel) +[reginaneon](https://github.com/reginaneon) +[NeonGeckoCom](https://github.com/NeonGeckoCom) + +## Tags +#NeonGecko Original +#NeonAI diff --git a/test/local_skills/skill-about/__init__.py b/test/local_skills/skill-about/__init__.py new file mode 100755 index 000000000..4a047a424 --- /dev/null +++ b/test/local_skills/skill-about/__init__.py @@ -0,0 +1,107 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json + +from neon_utils.skills.neon_skill import NeonSkill +from adapt.intent import IntentBuilder +from os import listdir, path + +from mycroft.skills import skill_api_method + + +class AboutSkill(NeonSkill): + def __init__(self): + super(AboutSkill, self).__init__(name="AboutSkill") + self.skill_info = None + self._update_skills_data() + + def initialize(self): + license_intent = IntentBuilder("license_intent").\ + optionally("Neon").optionally("Long").require("Tell").require("License").build() + self.register_intent(license_intent, self.read_license) + + list_skills_intent = IntentBuilder("list_skills_intent").optionally("Neon").optionally("Tell").\ + require("Skills").build() + self.register_intent(list_skills_intent, self.list_skills) + # TODO: Reload skills list when skills are added/removed DM + + def read_license(self, message): + """ + Reads back the NeonAI license from skill dialog + :param message: Message associated with request + """ + if self.neon_in_request(message): + if message.data.get("Long"): + self.speak_dialog("license_long") + else: + self.speak_dialog("license_short") + + def list_skills(self, message): + """ + Lists all installed skills by name. + :param message: Message associated with request + """ + if self.neon_in_request(message): + skills_list = [s['title'] for s in self.skill_info if s.get('title')] + skills_list.sort() + skills_to_speak = ", ".join(skills_list) + self.speak_dialog("skills_list", data={"list": skills_to_speak}) + + @skill_api_method + def skill_info_examples(self): + """ + API Method to build a list of examples as listed in skill metadata. + """ + examples = [d.get('examples') or list() for d in self.skill_info] + flat_list = [item for sublist in examples for item in sublist] + return flat_list + + def _update_skills_data(self): + """ + Loads skill metadata for all installed skills. + """ + skills = list() + skills_dir = path.dirname(path.dirname(__file__)) + for skill in listdir(skills_dir): + if path.isdir(path.join(skills_dir, skill)) and path.isfile(path.join(skills_dir, skill, "__init__.py")): + if path.isfile(path.join(skills_dir, skill, "skill.json")): + with open(path.join(skills_dir, skill, "skill.json")) as f: + skill_data = json.load(f) + else: + skill_name = str(path.basename(skill).split('.')[0]).replace('-', ' ').lower() + skill_data = {"title": skill_name} + skills.append(skill_data) + self.skill_info = skills + + def stop(self): + pass + + +def create_skill(): + return AboutSkill() diff --git a/test/local_skills/skill-about/requirements.txt b/test/local_skills/skill-about/requirements.txt new file mode 100755 index 000000000..2b664bb04 --- /dev/null +++ b/test/local_skills/skill-about/requirements.txt @@ -0,0 +1 @@ +neon-utils>=0.5.7 \ No newline at end of file diff --git a/test/local_skills/skill-about/skill.json b/test/local_skills/skill-about/skill.json new file mode 100755 index 000000000..9dc346837 --- /dev/null +++ b/test/local_skills/skill-about/skill.json @@ -0,0 +1,48 @@ +{ + "title": "About", + "url": "https://github.com/neongeckocom/skill-about", + "summary": "Skill used to provide information about Neon", + "short_description": "Skill used to provide information about Neon", + "description": "This skill provides information about Neon. You can ask about licensing and your Neon installation.", + "examples": [ + "tell me my license", + "tell me my skills" + ], + "desktopFile": false, + "warning": "", + "systemDeps": false, + "requirements": { + "python": [ + "neon-utils>=0.5.7" + ], + "system": {}, + "skill": [] + }, + "incompatible_skills": [], + "platforms": [ + "i386", + "x86_64", + "ia64", + "arm64", + "arm" + ], + "branch": "master", + "license": "BSD-3-Clause", + "icon": "https://0000.us/klatchat/app/files/neon_images/icons/neon_skill.png", + "category": "", + "categories": [ + "" + ], + "tags": [ + "NeonGecko Original", + "NeonAI" + ], + "credits": [ + "NeonDaniel", + "reginaneon", + "NeonGeckoCom" + ], + "skillname": "About", + "authorname": "neongeckocom", + "foldername": null +} \ No newline at end of file diff --git a/test/local_skills/skill-osm_parsing/LICENSE b/test/local_skills/skill-osm_parsing/LICENSE new file mode 100644 index 000000000..ee7652301 --- /dev/null +++ b/test/local_skills/skill-osm_parsing/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Daniel McKnight +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/test/local_skills/skill-osm_parsing/README.md b/test/local_skills/skill-osm_parsing/README.md new file mode 100644 index 000000000..a4daa1315 --- /dev/null +++ b/test/local_skills/skill-osm_parsing/README.md @@ -0,0 +1,34 @@ +# OSM Test Skill + +## Summary + +Skill used to test OSM Parsing + +## Requirements + +datetime + +## Description + +This is a skill description + +## Examples + +Here are some examples + +- "Do something cool." + +## Category +**Daily** +Productivity + +## Credits +@neongeckocom +@neondaniel +@reginaneon + +## Tags +#NeonGecko +#OVOS +#Test +#NotARealSkill \ No newline at end of file diff --git a/test/local_skills/skill-osm_parsing/__init__.py b/test/local_skills/skill-osm_parsing/__init__.py new file mode 100644 index 000000000..796c5735e --- /dev/null +++ b/test/local_skills/skill-osm_parsing/__init__.py @@ -0,0 +1,8 @@ +class OVOSTestSkill(OVOSSkill): + def __init__(self): + super(OVOSTestSkill, self).__init__(name="OVOSTestSkill") + self.is_a_skill = False + + +def create_skill(): + return OVOSTestSkill() diff --git a/test/local_skills/skill-osm_parsing/manifest.yml b/test/local_skills/skill-osm_parsing/manifest.yml new file mode 100644 index 000000000..fc7c64813 --- /dev/null +++ b/test/local_skills/skill-osm_parsing/manifest.yml @@ -0,0 +1,7 @@ +dependencies: + python: + - manifest_requirement + system: + all: system-manifest-pkg + skill: + - manifest-skill \ No newline at end of file diff --git a/test/local_skills/skill-osm_parsing/requirements.txt b/test/local_skills/skill-osm_parsing/requirements.txt new file mode 100644 index 000000000..985451e0d --- /dev/null +++ b/test/local_skills/skill-osm_parsing/requirements.txt @@ -0,0 +1 @@ +text_requirements \ No newline at end of file diff --git a/test/local_skills/skill-osm_parsing/skill.json b/test/local_skills/skill-osm_parsing/skill.json new file mode 100644 index 000000000..c837dc136 --- /dev/null +++ b/test/local_skills/skill-osm_parsing/skill.json @@ -0,0 +1,51 @@ +{ + "title": "OSM Test Skill", + "url": "https://github.com/OpenVoiceOS/tskill-osm_parsing", + "summary": "Skill used to test OSM Parsing", + "short_description": "Skill used to test OSM Parsing", + "description": "This is a skill description", + "examples": [ + "do something cool" + ], + "desktopFile": false, + "warning": "", + "systemDeps": false, + "requirements": { + "python": [ + "json-requirements" + ], + "system": { + "all": [ + "json-pkg" + ] + }, + "skill": [ + "json-skill" + ] + }, + "incompatible_skills": [], + "platforms": [ + "i386", + "x86_64", + "ia64", + "arm64", + "arm" + ], + "branch": "v0.2.1", + "license": "BSD", + "icon": "https://0000.us/klatchat/app/files/neon_images/icons/neon_skill.png", + "category": "Daily", + "categories": [ + "Daily", + "Productivity" + ], + "tags": [ + "NeonGecko", + "OVOS", + "Test", + "NotARealSkill" + ], + "skillname": "OSM Test Skill", + "authorname": "OpenVoiceOS", + "foldername": "not-a-skill" +} \ No newline at end of file diff --git a/test/local_skills/skill-ovos-homescreen/LICENSE b/test/local_skills/skill-ovos-homescreen/LICENSE new file mode 100755 index 000000000..261eeb9e9 --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/local_skills/skill-ovos-homescreen/MANIFEST.in b/test/local_skills/skill-ovos-homescreen/MANIFEST.in new file mode 100755 index 000000000..9c3f56672 --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/MANIFEST.in @@ -0,0 +1,5 @@ +recursive-include dialog * +recursive-include vocab * +recursive-include locale * +recursive-include res * +recursive-include ui * \ No newline at end of file diff --git a/test/local_skills/skill-ovos-homescreen/README.md b/test/local_skills/skill-ovos-homescreen/README.md new file mode 100755 index 000000000..a64ee89ca --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/README.md @@ -0,0 +1,3 @@ +OVOS Homescreen Skill for Mycroft GUI +------------------------------------------------------------------------------ +###### Provides Custom Resting Face for OVOS diff --git a/test/local_skills/skill-ovos-homescreen/__init__.py b/test/local_skills/skill-ovos-homescreen/__init__.py new file mode 100755 index 000000000..bcb82518f --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/__init__.py @@ -0,0 +1,337 @@ +import datetime +import os +import time +import requests +import json + +from os import path, listdir +from mycroft_bus_client import Message +from ovos_utils.log import LOG +from ovos_utils.skills import get_skills_folder + +from mycroft.skills.core import resting_screen_handler, intent_file_handler, MycroftSkill +from mycroft.skills.skill_loader import load_skill_module +from mycroft.skills.skill_manager import SkillManager +from mycroft.skills.api import SkillApi + + +class OVOSHomescreenSkill(MycroftSkill): + # The constructor of the skill, which calls MycroftSkill's constructor + def __init__(self): + super(OVOSHomescreenSkill, self).__init__(name="OVOSHomescreen") + self.skill_manager = None + self.notifications_storage_model = [] + self.def_wallpaper_folder = "" # path.dirname(__file__) + '/ui/wallpapers/' + self.loc_wallpaper_folder = None + self.selected_wallpaper = self.settings.get("wallpaper") or "default.jpg" + self.wallpaper_collection = [] + self.rtlMode = 1 if self.config_core.get("rtl", False) else 0 + + # Populate skill IDs to use for data sources + self.weather_skill = self.settings.get("weather_skill") or "skill-weather.openvoiceos" + self.datetime_skill = self.settings.get("datetime_skill") or "skill-date-time.mycroftai" + self.skill_info_skill = self.settings.get("examples_skill") or "ovos-skills-info.openvoiceos" + self.weather_api = None + self.datetime_api = None + self.skill_info_api = None + + def initialize(self): + self.loc_wallpaper_folder = self.file_system.path + '/wallpapers/' + now = datetime.datetime.now() + callback_time = datetime.datetime( + now.year, now.month, now.day, now.hour, now.minute + ) + datetime.timedelta(seconds=60) + self.schedule_repeating_event(self.update_dt, callback_time, 10) + self.skill_manager = SkillManager(self.bus) + + # Handler Registration For Notifications + self.add_event("homescreen.wallpaper.set", + self.handle_set_wallpaper) + self.add_event("ovos.notification.update_counter", + self.handle_notification_widget_update) + self.add_event("ovos.notification.update_storage_model", + self.handle_notification_storage_model_update) + self.gui.register_handler("homescreen.swipe.change.wallpaper", + self.change_wallpaper) + self.add_event("mycroft.ready", self.handle_mycroft_ready) + + if not self.file_system.exists("wallpapers"): + os.mkdir(path.join(self.file_system.path, "wallpapers")) + + self.collect_wallpapers() + self._load_skill_apis() + + self.bus.emit(Message("mycroft.device.show.idle")) + + ##################################################################### + # Homescreen Registration & Handling + + @resting_screen_handler("OVOSHomescreen") + def handle_idle(self, _): + LOG.debug('Activating Time/Date resting page') + # self.gui['wallpaper_path'] = self.check_wallpaper_path(self.selected_wallpaper) + # self.gui['selected_wallpaper'] = self.selected_wallpaper + self.gui['wallpaper_path'] = "http://localhost:8000/skill-ovos-homescreen/ui/wallpapers/" + self.gui['selected_wallpaper'] = "default.jpg" + self.gui['notification'] = {} + self.gui["notification_model"] = { + "storedmodel": self.notifications_storage_model, + "count": len(self.notifications_storage_model), + } + self.gui["applications_model"] = self.build_voice_applications_model() + + try: + self.update_dt() + self.update_weather() + self.update_examples() + except Exception as e: + LOG.error(e) + + self.gui['rtl_mode'] = self.rtlMode + self.gui['dateFormat'] = self.config_core.get("date_format") or "DMY" + self.gui.show_page("idle.qml") + + def update_examples(self): + """ + Loads or updates skill examples via the skill_info_api. + """ + if not self.skill_info_api: + LOG.warning("Requested update before skill_info API loaded") + self._load_skill_apis() + if self.skill_info_api: + self.gui['skill_examples'] = {"examples": self.skill_info_api.skill_info_examples()} + else: + LOG.warning("No skill_info_api, skipping update") + + def update_dt(self): + """ + Loads or updates date/time via the datetime_api. + """ + if not self.datetime_api: + LOG.warning("Requested update before datetime API loaded") + self._load_skill_apis() + if self.datetime_api: + self.gui["time_string"] = self.datetime_api.get_display_current_time() + self.gui["date_string"] = self.datetime_api.get_display_date() + self.gui["weekday_string"] = self.datetime_api.get_weekday() + self.gui['day_string'], self.gui["month_string"] = self._split_month_string(self.datetime_api.get_month_date()) + self.gui["year_string"] = self.datetime_api.get_year() + else: + LOG.warning("No datetime_api, skipping update") + + def update_weather(self): + """ + Loads or updates weather via the weather_api. + """ + if not self.weather_api: + LOG.warning("Requested update before weather API loaded") + self._load_skill_apis() + if self.weather_api: + current_weather_report = self.weather_api.get_current_weather_homescreen() + self.gui["weather_api_enabled"] = True + self.gui["weather_code"] = current_weather_report.get("weather_code") + self.gui["weather_temp"] = current_weather_report.get("weather_temp") + else: + self.gui["weather_api_enabled"] = False + LOG.warning("No weather_api, skipping update") + + ##################################################################### + # Wallpaper Manager + + def collect_wallpapers(self): + def_wallpaper_collection, loc_wallpaper_collection = [], [] + # for dirname, dirnames, filenames in os.walk(self.def_wallpaper_folder): + # def_wallpaper_collection = filenames + + for dirname, dirnames, filenames in os.walk(self.loc_wallpaper_folder): + loc_wallpaper_collection = filenames + + self.wallpaper_collection = def_wallpaper_collection + loc_wallpaper_collection + + @intent_file_handler("change.wallpaper.intent") + def change_wallpaper(self, _): + # Get Current Wallpaper idx + current_idx = self.get_wallpaper_idx(self.selected_wallpaper) + collection_length = len(self.wallpaper_collection) - 1 + if not current_idx == collection_length: + fidx = current_idx + 1 + self.selected_wallpaper = self.wallpaper_collection[fidx] + self.settings["wallpaper"] = self.wallpaper_collection[fidx] + + else: + self.selected_wallpaper = self.wallpaper_collection[0] + self.settings["wallpaper"] = self.wallpaper_collection[0] + + self.gui['wallpaper_path'] = self.check_wallpaper_path(self.selected_wallpaper) + self.gui['selected_wallpaper'] = self.selected_wallpaper + + def get_wallpaper_idx(self, filename): + try: + index_element = self.wallpaper_collection.index(filename) + return index_element + except ValueError: + return None + + def handle_set_wallpaper(self, message): + image_url = message.data.get("url", "") + now = datetime.datetime.now() + setname = "wallpaper-" + now.strftime("%H%M%S") + ".jpg" + if image_url: + print(image_url) + response = requests.get(image_url) + with self.file_system.open(path.join("wallpapers", setname), "wb") as my_file: + my_file.write(response.content) + my_file.close() + self.collect_wallpapers() + cidx = self.get_wallpaper_idx(setname) + self.selected_wallpaper = self.wallpaper_collection[cidx] + self.settings["wallpaper"] = self.wallpaper_collection[cidx] + + self.gui['wallpaper_path'] = self.check_wallpaper_path(setname) + self.gui['selected_wallpaper'] = self.selected_wallpaper + + def check_wallpaper_path(self, wallpaper): + file_def_check = self.def_wallpaper_folder + wallpaper + file_loc_check = self.loc_wallpaper_folder + wallpaper + if path.exists(file_def_check): + return self.def_wallpaper_folder + elif path.exists(file_loc_check): + return self.loc_wallpaper_folder + + ##################################################################### + # Manage notifications widget + + def handle_notification_widget_update(self, message): + # Receives notification counter update + # Emits request to update storage model on counter update + notifcation_count = message.data.get("notification_counter", "") + self.gui["notifcation_counter"] = notifcation_count + self.bus.emit(Message("ovos.notification.api.request.storage.model")) + + def handle_notification_storage_model_update(self, message): + # Receives updated storage model and forwards it to widget + notification_model = message.data.get("notification_model", "") + self.gui["notification_model"] = notification_model + + ##################################################################### + # Misc + + def stop(self): + pass + + def shutdown(self): + self.cancel_all_repeating_events() + + def handle_mycroft_ready(self, _): + self._load_skill_apis() + + def _load_skill_apis(self): + """ + Loads weather, date/time, and examples skill APIs + """ + try: + if not self.weather_api: + self.weather_api = SkillApi.get(self.weather_skill) + except Exception as e: + LOG.error(f"Failed To Import Weather Skill: {e}") + + try: + if not self.skill_info_api: + self.skill_info_api = SkillApi.get(self.skill_info_skill) + except Exception as e: + LOG.error(f"Failed To Import OVOS Info Skill: {e}") + + # Import Date Time Skill As Date Time Provider + try: + if not self.datetime_api: + self.datetime_api = SkillApi.get(self.datetime_skill) + except Exception as e: + LOG.error(f"Failed to import DateTime Skill: {e}") + + # TODO: Depreciate this + if not self.datetime_api: + try: + root_dir = self.root_dir.rsplit("/", 1)[0] + time_date_path = str(root_dir) + f"/{self.datetime_skill}/__init__.py" + time_date_id = "datetimeskill" + datetimeskill = load_skill_module(time_date_path, time_date_id) + from datetimeskill import TimeSkill + + self.datetime_api = TimeSkill() + except Exception as e: + LOG.error(f"Failed To Import DateTime Skill: {e}") + + def _split_month_string(self, month_date: str) -> list: + """ + Splits a month+date string into month and date (i.e. "August 06" -> ["August", "06"]) + :param month_date: formatted month and day of month ("August 06" or "06 August") + :return: [day, month] + """ + month_string = month_date.split(" ") + if self.config_core.get('date_format') == 'MDY': + day_string = month_string[1] + month_string = month_string[0] + else: + day_string = month_string[0] + month_string = month_string[1] + + return [day_string, month_string] + + ##################################################################### + # Build Voice Applications Model + + def build_voice_applications_model(self): + voiceApplicationsList = [] + + if path.exists("/opt/mycroft/skills/"): + skill_folders = listdir("/opt/mycroft/skills/") + folder_prefix = "/opt/mycroft/skills" + else: + skill_folders = listdir(get_skills_folder()) + folder_prefix = get_skills_folder() + + resource_app = "app.json" + resource_mobile = "android.json" + + for folder in skill_folders: + absolute_folder_path = path.join(folder_prefix, folder) + + if path.exists(path.join(absolute_folder_path, resource_app)) and path.isfile( + path.join(absolute_folder_path, resource_app)) : + with open(path.join(absolute_folder_path, resource_app)) as f: + expand_file = json.load(f) + folder_path = folder + if not any(d.get('folder', None) == folder_path + for d in voiceApplicationsList): + thumb = absolute_folder_path + expand_file["icon"] + voiceApplicationsList.append({"thumbnail": thumb, + "name": expand_file["name"], + "action": expand_file["action"], + "folder": folder_path}) + + elif path.exists(path.join(absolute_folder_path, resource_mobile)) and path.isfile( + path.join(absolute_folder_path, resource_mobile)) : + with open(path.join(absolute_folder_path, resource_mobile)) as f: + expand_file = json.load(f) + folder_path = folder + if not any(d.get('folder', None) == folder_path + for d in voiceApplicationsList): + thumb = absolute_folder_path + expand_file["android_icon"] + voiceApplicationsList.append({"thumbnail": thumb, + "name": expand_file["android_name"], + "action": expand_file["android_handler"], + "folder": folder_path}) + + try: + sort_on = "name" + decorated = [(dict_[sort_on], dict_) + for dict_ in voiceApplicationsList] + decorated.sort() + return [dict_ for (key, dict_) in decorated] + + except Exception: + return voiceApplicationsList + + +def create_skill(): + return OVOSHomescreenSkill() diff --git a/test/local_skills/skill-ovos-homescreen/requirements.txt b/test/local_skills/skill-ovos-homescreen/requirements.txt new file mode 100755 index 000000000..61587cdb5 --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.20.0,<2.26.0 +pillow==7.1.2 +ovos_utils>=0.0.6 \ No newline at end of file diff --git a/test/local_skills/skill-ovos-homescreen/settingsmeta.yml b/test/local_skills/skill-ovos-homescreen/settingsmeta.yml new file mode 100755 index 000000000..5f10b1176 --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/settingsmeta.yml @@ -0,0 +1,22 @@ +skillMetadata: + sections: + - name: Skill Data Sources + fields: + - name: weather_skill + type: text + label: Weather + value: skill-weather.openvoiceos + - name: datetime_skill + type: text + label: Date and Time + value: skill-date-time.mycroftai + - name: examples_skill + type: text + label: Examples + value: ovos-skills-info.openvoiceos + - name: Personalization + fields: + - name: wallpaper + type: text + label: Wallpaper + value: default.jpg \ No newline at end of file diff --git a/test/local_skills/skill-ovos-homescreen/setup.py b/test/local_skills/skill-ovos-homescreen/setup.py new file mode 100755 index 000000000..f4dcf2d4c --- /dev/null +++ b/test/local_skills/skill-ovos-homescreen/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +from setuptools import setup + +# skill_id=package_name:SkillClass +PLUGIN_ENTRY_POINT = 'mycroft-homescreen.mycroftai=ovos_skill_homescreen:OVOSHomescreenSkill' +# in this case the skill_id is defined to purposefully replace the mycroft version of the skill, +# or rather to be replaced by it in case it is present. all skill directories take precedence over plugin skills + + +setup( + # this is the package name that goes on pip + name='ovos-skill-homescreen', + version='0.0.1', + description='OVOS homescreen skill plugin', + url='https://github.com/OpenVoiceOS/skill-ovos-homescreen', + author='Aix', + author_email='aix.m@outlook.com', + license='Apache-2.0', + package_dir={"ovos_skill_homescreen": ""}, + package_data={'ovos_skill_homescreen': ["vocab/*", "ui/*"]}, + packages=['ovos_skill_homescreen'], + include_package_data=True, + install_requires=["ovos-plugin-manager>=0.0.2", "astral==1.4", "arrow==0.12.0"], + keywords='ovos skill plugin', + entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT} +) diff --git a/test/test_skill_utils.py b/test/test_skill_utils.py index 1ac2432d1..c14fa16cf 100644 --- a/test/test_skill_utils.py +++ b/test/test_skill_utils.py @@ -22,14 +22,18 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +import importlib +import json import os import shutil import sys import unittest +from copy import deepcopy + +from importlib import reload +from mock.mock import Mock sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from neon_core.util.skill_utils import * TEST_SKILLS_NO_AUTH = [ "https://github.com/NeonGeckoCom/alerts.neon/tree/dev", @@ -60,6 +64,7 @@ def tearDown(self) -> None: shutil.rmtree(SKILL_DIR) def test_get_remote_entries(self): + from neon_core.util.skill_utils import get_remote_entries from ovos_skills_manager.session import set_github_token, clear_github_token set_github_token(SKILL_CONFIG["neon_token"]) skills_list = get_remote_entries(SKILL_CONFIG["default_skills"]) @@ -69,29 +74,85 @@ def test_get_remote_entries(self): self.assertTrue(all(skill.startswith("https://github.com") for skill in skills_list)) def test_install_skills_from_list_no_auth(self): + from neon_core.util.skill_utils import install_skills_from_list install_skills_from_list(TEST_SKILLS_NO_AUTH, SKILL_CONFIG) skill_dirs = [d for d in os.listdir(SKILL_DIR) if os.path.isdir(os.path.join(SKILL_DIR, d))] self.assertEqual(len(skill_dirs), len(TEST_SKILLS_NO_AUTH)) self.assertIn("alerts.neon.neongeckocom", skill_dirs) def test_install_skills_from_list_with_auth(self): + from neon_core.util.skill_utils import install_skills_from_list install_skills_from_list(TEST_SKILLS_WITH_AUTH, SKILL_CONFIG) skill_dirs = [d for d in os.listdir(SKILL_DIR) if os.path.isdir(os.path.join(SKILL_DIR, d))] self.assertEqual(len(skill_dirs), len(TEST_SKILLS_WITH_AUTH)) self.assertIn("i-like-brands.neon.neongeckocom", skill_dirs) def test_install_skills_default(self): + from neon_core.util.skill_utils import install_skills_default,\ + get_remote_entries install_skills_default(SKILL_CONFIG) - skill_dirs = [d for d in os.listdir(SKILL_DIR) if os.path.isdir(os.path.join(SKILL_DIR, d))] - self.assertEqual(len(skill_dirs), len(get_remote_entries(SKILL_CONFIG["default_skills"])), - f"{skill_dirs}\n\n{get_remote_entries(SKILL_CONFIG['default_skills'])}") + skill_dirs = [d for d in os.listdir(SKILL_DIR) if + os.path.isdir(os.path.join(SKILL_DIR, d))] + self.assertEqual( + len(skill_dirs), + len(get_remote_entries(SKILL_CONFIG["default_skills"])), + f"{skill_dirs}\n\n" + f"{get_remote_entries(SKILL_CONFIG['default_skills'])}") def test_get_neon_skills_data(self): + from neon_core.util.skill_utils import get_neon_skills_data + from ovos_skills_manager.github.utils import normalize_github_url neon_skills = get_neon_skills_data() self.assertIsInstance(neon_skills, dict) for skill in neon_skills: self.assertIsInstance(neon_skills[skill], dict) - self.assertEqual(skill, normalize_github_url(neon_skills[skill]["url"])) + self.assertEqual(skill, + normalize_github_url(neon_skills[skill]["url"])) + + def test_install_local_skills(self): + import neon_core.util.skill_utils + importlib.reload(neon_core.util.skill_utils) + install_deps = Mock() + neon_core.util.skill_utils._install_skill_dependencies = install_deps + install_local_skills = neon_core.util.skill_utils.install_local_skills + + local_skills_dir = os.path.join(os.path.dirname(__file__), + "local_skills") + + installed = install_local_skills(local_skills_dir) + num_installed = len(installed) + self.assertEqual(installed, os.listdir(local_skills_dir)) + self.assertEqual(num_installed, install_deps.call_count) + + + def test_install_skill_dependencies(self): + # Patch dependency installation + import ovos_skills_manager.requirements + importlib.reload(ovos_skills_manager.requirements) + pip_install = Mock() + install_system_deps = Mock() + ovos_skills_manager.requirements.install_system_deps = \ + install_system_deps + ovos_skills_manager.requirements.pip_install = pip_install + from ovos_skills_manager.skill_entry import SkillEntry + import neon_core.util.skill_utils + importlib.reload(neon_core.util.skill_utils) + from neon_core.util.skill_utils import _install_skill_dependencies + local_skills_dir = os.path.join(os.path.dirname(__file__), + "local_skills") + with open(os.path.join(local_skills_dir, + "skill-osm_parsing", "skill.json")) as f: + skill_json = json.load(f) + entry = SkillEntry.from_json(skill_json, False) + self.assertEqual(entry.json["requirements"], + skill_json["requirements"]) + + _install_skill_dependencies(entry) + pip_install.assert_called_once() + pip_install.assert_called_with(entry.json["requirements"]["python"]) + install_system_deps.assert_called_once() + install_system_deps.assert_called_with( + entry.json["requirements"]["system"]) if __name__ == '__main__': From 099c76fa42fe681e64551b5af859261b96f34ff4 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Mon, 28 Feb 2022 18:20:49 +0000 Subject: [PATCH 41/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 886d12e1e..f53d31c3e 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a2" +__version__ = "21.10.3a3" From 470d01c2d2956282ef289a11fdfc75a4f1398570 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Tue, 8 Mar 2022 15:29:03 -0800 Subject: [PATCH 42/70] ovos_utils config update compat (#218) * Temporarily remove failing ovos_config_assistant call * Revert configuration test changes Update ovos_utils dependency spec * Update config tests for ovos-utils changes * Add wrapper for ovos-utils stack checking compat. * Remove wrapper method and update references Deprecate config wrappers in configuration module * Bump neon_speech dependency spec * Update use_neon_core docstring based on PR feedback Co-authored-by: Daniel McKnight --- neon_core/config.py | 1 - neon_core/util/runtime_utils.py | 36 +++++++++++++++++++++++++++++++++ requirements/core_modules.txt | 2 +- requirements/requirements.txt | 4 ++-- test/test_configuration.py | 29 +++++++++++++++++--------- 5 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 neon_core/util/runtime_utils.py diff --git a/neon_core/config.py b/neon_core/config.py index f61e1ce6f..26751ed47 100644 --- a/neon_core/config.py +++ b/neon_core/config.py @@ -28,7 +28,6 @@ from os.path import join, dirname from ovos_utils.json_helper import merge_dict -from ovos_utils.system import set_root_path from ovos_utils.xdg_utils import xdg_config_home from neon_utils.logger import LOG diff --git a/neon_core/util/runtime_utils.py b/neon_core/util/runtime_utils.py new file mode 100644 index 000000000..291c91afe --- /dev/null +++ b/neon_core/util/runtime_utils.py @@ -0,0 +1,36 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +def use_neon_core(func): + """ + Wrapper to ensure call originates from neon_core for stack checks. + This is used for ovos-utils config platform detection which uses the stack + to determine which module config to return. + """ + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + diff --git a/requirements/core_modules.txt b/requirements/core_modules.txt index 32dca088e..7b0442282 100644 --- a/requirements/core_modules.txt +++ b/requirements/core_modules.txt @@ -8,7 +8,7 @@ ovos-ww-plugin-pocketsphinx~=0.1.2 # neon core modules neon_enclosure~=0.1,>=0.1.2 neon_messagebus -neon_speech~=0.3,>=0.3.3a4 +neon_speech~=0.3,>=0.3.3a7 neon_audio~=0.4,>=0.4.3a4 neon_gui # TODO: Version spec GUI \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 21a429498..99472c588 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,8 +2,8 @@ ovos-core[skills_lgpl]~=0.0.2a10 # utils -neon-utils~=0.12,>=0.15.1a7 -ovos_utils~=0.0.18 +neon-utils~=0.12,>=0.15.1a10 +ovos_utils~=0.0.19,>=0.0.19a3 ovos-skills-manager~=0.0.10a20 ovos-plugin-manager~=0.0.4,>=0.0.7a0 # TODO: Update to stable versions diff --git a/test/test_configuration.py b/test/test_configuration.py index fa9a6c1cb..514859ba7 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -35,18 +35,20 @@ class ConfigurationTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - from ovos_config_assistant import config_helpers - + from ovos_config_assistant.config_helpers import \ + get_ovos_config, get_ovos_default_config_paths ovos_config = os.path.expanduser("~/.config/OpenVoiceOS/ovos.conf") if os.path.isfile(ovos_config): os.remove(ovos_config) - assert config_helpers.get_ovos_default_config_paths() == [] + assert get_ovos_default_config_paths() == [] import neon_core + from neon_core.util.runtime_utils import use_neon_core + assert isinstance(neon_core.CORE_VERSION_STR, str) - assert len(config_helpers.get_ovos_default_config_paths()) == 1 - LOG.info(config_helpers.get_ovos_default_config_paths()) - ovos_config = config_helpers.get_ovos_config() + assert len(use_neon_core(get_ovos_default_config_paths)()) == 1 + LOG.info(use_neon_core(get_ovos_default_config_paths)()) + ovos_config = use_neon_core(get_ovos_config)() LOG.info(pformat(ovos_config)) assert ovos_config['config_filename'] == 'neon.conf' @@ -54,8 +56,10 @@ def test_neon_core_config_init(self): from neon_utils.configuration_utils import \ get_mycroft_compatible_config from neon_core.configuration import Configuration + from neon_core.util.runtime_utils import use_neon_core + neon_compat_config = Configuration.get() - neon_config = get_mycroft_compatible_config() + neon_config = use_neon_core(get_mycroft_compatible_config)() for key, val in neon_config.items(): if isinstance(val, dict): for k, v in val.items(): @@ -69,8 +73,10 @@ def test_ovos_core_config_init(self): from neon_utils.configuration_utils import \ get_mycroft_compatible_config from mycroft.configuration import Configuration as MycroftConfig + from neon_core.util.runtime_utils import use_neon_core + mycroft_config = MycroftConfig.get() - neon_config = get_mycroft_compatible_config() + neon_config = use_neon_core(get_mycroft_compatible_config)() for key, val in neon_config.items(): if isinstance(val, dict): for k, v in val.items(): @@ -86,8 +92,11 @@ def test_signal_dir(self): from ovos_utils.signal import get_ipc_directory as ovos_ipc_dir from mycroft.util.signal import get_ipc_directory as mycroft_ipc_dir - self.assertEqual(neon_ipc_dir, ovos_ipc_dir()) - self.assertEqual(neon_ipc_dir, mycroft_ipc_dir()) + from neon_core.util.runtime_utils import use_neon_core + + self.assertEqual(neon_ipc_dir, use_neon_core(ovos_ipc_dir)()) + self.assertEqual(neon_ipc_dir, + use_neon_core(mycroft_ipc_dir)()) if __name__ == '__main__': From 1323089098a44aa2aa1b6716e546cb89bf2a087e Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Tue, 8 Mar 2022 23:29:27 +0000 Subject: [PATCH 43/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index f53d31c3e..8efa7c3ad 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a3" +__version__ = "21.10.3a4" From 3bb5ab77d4324596f7ab09010488f1a98fb0fe7c Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 9 Mar 2022 15:36:04 -0800 Subject: [PATCH 44/70] Refactor entrypoints and update Docker to handle skill dependency installation (#209) * Add `cli` module to handle entrypoints Update Dockerfile to handle external skills * Update config tests for ovos-utils changes * Remove added configuration methods * Pin higher ovos-ww-plugin-pocketsphinx version for ovos-utils compat * Increment neon-utils version * Update skill settings path in dockerfile Co-authored-by: Daniel McKnight --- Dockerfile | 5 +- neon_core/cli.py | 108 +++++++++++++++++++++++++++++ neon_core/util/diagnostic_utils.py | 1 + neon_core/util/skill_utils.py | 4 +- requirements/client.txt | 2 +- requirements/requirements.txt | 2 +- setup.py | 1 + 7 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 neon_core/cli.py diff --git a/Dockerfile b/Dockerfile index 8b3583142..a4a904fe0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,12 +30,13 @@ COPY docker_overlay/neon.conf /root/.config/neon/neon.conf RUN pip install wheel && \ pip install . -CMD ["neon_skills_service"] +CMD ["neon", "run-skills", "-i", "/skills"] FROM base as default_skills RUN mkdir -p /root/.local/share/neon -COPY docker_overlay/skill_settings /root/.config/neon/skills +# TODO: This path should move to config in the future +COPY docker_overlay/skill_settings /root/.local/share/neon/skills COPY docker_overlay/ngi_local_conf.yml /config/ RUN neon-install-default-skills RUN rm /config/ngi_local_conf.yml \ No newline at end of file diff --git a/neon_core/cli.py b/neon_core/cli.py new file mode 100644 index 000000000..669bf9168 --- /dev/null +++ b/neon_core/cli.py @@ -0,0 +1,108 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from threading import Thread + +import click + +from click_default_group import DefaultGroup +from neon_utils.packaging_utils import get_neon_core_version + + +@click.group("neon", cls=DefaultGroup, + no_args_is_help=True, invoke_without_command=True, + help="Neon Core Commands\n\n" + "See also: neon COMMAND --help") +@click.option("--version", "-v", is_flag=True, required=False, + help="Print the current version") +def neon_core_cli(version: bool = False): + if version: + click.echo(f"Neon version {get_neon_core_version()}") + + +@neon_core_cli.command(help="Start Neon Core") +def start_neon(): + from neon_core.run_neon import start_neon + click.echo("Starting Neon") + Thread(target=start_neon, daemon=False).start() + click.echo("Neon Started") + + +@neon_core_cli.command(help="Stop Neon Core") +def stop_neon(): + from neon_core.run_neon import stop_neon + click.echo("Stopping Neon") + stop_neon() + click.echo("Neon Stopped") + + +@neon_core_cli.command(help="Send Diagnostics") +@click.option("--no-transcripts", is_flag=True, default=False, + help="Skip upload of transcript files") +@click.option("--no-logs", is_flag=True, default=False, + help="Skip upload of log files") +@click.option("--no-config", is_flag=True, default=False, + help="Skip upload of configuration files") +def upload_diagnostics(no_transcripts, no_logs, no_config): + from neon_core.util.diagnostic_utils import send_diagnostics + click.echo("Uploading Diagnostics") + send_diagnostics(not no_logs, not no_transcripts, not no_config) + click.echo("Diagnostic Upload Complete") + + +@neon_core_cli.command(help="Install Default Skills") +def install_default_skills(): + from neon_core.util.skill_utils import install_skills_default + click.echo("Installing Default Skills") + install_skills_default() + click.echo("Default Skills Installed") + + +@neon_core_cli.command(help= + "Install skill requirements for a specified directory") +@click.argument("skill_dir") +def install_skill_requirements(skill_dir): + from neon_core.util.skill_utils import install_local_skills + try: + installed = install_local_skills(skill_dir) + click.echo(f"Installed {len(installed)} skills from {skill_dir}") + except ValueError as e: + click.echo(e) + + +@neon_core_cli.command(help="Start Neon Skills module") +@click.option("--install-skills", "-i", default=None, + help="Path to local skills for which to install dependencies") +def run_skills(install_skills): + from neon_core.util.skill_utils import install_local_skills + from neon_core.skills.__main__ import main + if install_skills: + click.echo(f"Handling installation of skills in: {install_skills}") + install_local_skills(install_skills) + click.echo("Starting Skills Service") + main() + click.echo("Skills Service Shutdown") diff --git a/neon_core/util/diagnostic_utils.py b/neon_core/util/diagnostic_utils.py index 396e2a936..f5fb50b14 100644 --- a/neon_core/util/diagnostic_utils.py +++ b/neon_core/util/diagnostic_utils.py @@ -104,6 +104,7 @@ def cli_send_diags(): """ CLI Entry Point to Send Diagnostics """ + LOG.warning(f"This function is deprecated. Use `neon upload-diagnostics`") import argparse parser = argparse.ArgumentParser(description="Upload Neon Diagnostics Files", add_help=True) parser.add_argument("--no-transcripts", dest="transcripts", default=True, action='store_false', diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index 70a92a620..1fe72e1fe 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -159,6 +159,7 @@ def install_local_skills(local_skills_dir: str = "/skills") -> list: :param local_skills_dir: Directory to install skills from :returns: list of installed skill directories """ + github_token = get_neon_skills_config()["neon_token"] local_skills_dir = expanduser(local_skills_dir) if not isdir(local_skills_dir): raise ValueError(f"{local_skills_dir} is not a valid directory") @@ -168,7 +169,8 @@ def install_local_skills(local_skills_dir: str = "/skills") -> list: pass LOG.debug(f"Attempting installation of {skill}") try: - entry = SkillEntry.from_directory(join(local_skills_dir, skill)) + entry = SkillEntry.from_directory(join(local_skills_dir, skill), + github_token) _install_skill_dependencies(entry) installed_skills.append(skill) except Exception as e: diff --git a/requirements/client.txt b/requirements/client.txt index cc8535c06..958ef7054 100644 --- a/requirements/client.txt +++ b/requirements/client.txt @@ -2,5 +2,5 @@ neon-transcripts-controller @ git+https://github.com/NeonGeckoCom/transcripts_co # wake word plugins chatterbox-wake-word-plugin-dummy~=0.1 -ovos-ww-plugin-pocketsphinx~=0.1 +ovos-ww-plugin-pocketsphinx~=0.1,>=0.1.1 ovos-ww-plugin-precise~=0.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 99472c588..1806ab5b2 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,7 +2,7 @@ ovos-core[skills_lgpl]~=0.0.2a10 # utils -neon-utils~=0.12,>=0.15.1a10 +neon-utils~=0.12,>=0.15.1a11 ovos_utils~=0.0.19,>=0.0.19a3 ovos-skills-manager~=0.0.10a20 ovos-plugin-manager~=0.0.4,>=0.0.7a0 diff --git a/setup.py b/setup.py index 60f2e5999..06af6b32d 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ def get_requirements(requirements_filename: str): include_package_data=True, entry_points={ 'console_scripts': [ + 'neon=neon_core.cli:neon_core_cli', 'neon_skills_service=neon_core.skills.__main__:main', 'neon-install-default-skills=neon_core.util.skill_utils:install_skills_default', 'neon-upload-diagnostics=neon_core.util.diagnostic_utils:cli_send_diags', From c4036902aecf3cc9a25b556ada41d079e8fb6a2b Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 9 Mar 2022 23:36:21 +0000 Subject: [PATCH 45/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 8efa7c3ad..fb756f9d7 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a4" +__version__ = "21.10.3a5" From e9f0146b55ea620cc35d868994d2f3d501e6e9fc Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 11 Mar 2022 10:33:42 -0800 Subject: [PATCH 46/70] Refactor skill management code (#219) * Patch ovos-core import error Troubleshoot skills manager changes * Resolve imports to handle patched configuration * Prevent `super` call to deprecated `_emit_converse_error` method * Increment neon-utils version Co-authored-by: Daniel McKnight --- neon_core/skills/__init__.py | 4 ++++ neon_core/skills/service.py | 38 ++++++++++++++++--------------- neon_core/skills/skill_manager.py | 3 ++- requirements/requirements.txt | 2 +- test/test_run_neon.py | 5 +++- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/neon_core/skills/__init__.py b/neon_core/skills/__init__.py index da5684389..525d5c077 100644 --- a/neon_core/skills/__init__.py +++ b/neon_core/skills/__init__.py @@ -29,12 +29,16 @@ from neon_core.skills.decorators import intent_handler, intent_file_handler, \ resting_screen_handler, conversational_intent +from mycroft.skills.intent_services.adapt_service import AdaptIntent + import mycroft.skills.core mycroft.MycroftSkill = PatchedMycroftSkill mycroft.skills.MycroftSkill = PatchedMycroftSkill mycroft.skills.core.MycroftSkill = PatchedMycroftSkill mycroft.skills.mycroft_skill.MycroftSkill = PatchedMycroftSkill +mycroft.skills.intent_service.AdaptIntent = AdaptIntent + __all__ = ['NeonSkill', 'intent_handler', diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index 582d7809f..bac0a5caa 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -100,12 +100,12 @@ def start(self): self.event_scheduler = EventScheduler(self.bus) self.status = ProcessStatus('skills', self.bus, self.callbacks) SkillApi.connect_bus(self.bus) - self._initialize_skill_manager() + self.skill_manager = NeonSkillManager(self.bus, self.watchdog) self.status.set_started() - self._wait_for_internet_connection() - # TODO can this be removed? its a hack around msm requiring internet... - if self.skill_manager is None: - self._initialize_skill_manager() + # self._wait_for_internet_connection() + # # TODO can this be removed? its a hack around msm requiring internet... + # if self.skill_manager is None: + # self._initialize_skill_manager() self.skill_manager.start() while not self.skill_manager.is_alive(): time.sleep(0.1) @@ -140,19 +140,21 @@ def _register_intent_services(self): ) return service - def _initialize_skill_manager(self): - """Create a thread that monitors the loaded skills, looking for updates - - Returns: - SkillManager instance or None if it couldn't be initialized - """ - self.skill_manager = NeonSkillManager(self.bus, self.watchdog) - - # TODO: This config patching should be handled in neon_utils - self.skill_manager.config["skills"]["priority_skills"] = \ - self.skill_manager.config["skills"].get("priority") or \ - self.skill_manager.config["skills"]["priority_skills"] - self.skill_manager.load_priority() + # def _initialize_skill_manager(self): + # """Create a thread that monitors the loaded skills, looking for updates + # + # Returns: + # SkillManager instance or None if it couldn't be initialized + # """ + # self.skill_manager = NeonSkillManager(self.bus, self.watchdog) + # + # # # TODO: This config patching should be handled in neon_utils + # # self.skill_manager.config["skills"]["priority_skills"] = \ + # # self.skill_manager.config["skills"].get("priority") or \ + # # self.skill_manager.config["skills"]["priority_skills"] + # LOG.info(f"Priority={self.skill_manager.config['skills']['priority_skills']}") + # LOG.info(f"Blacklisted={self.skill_manager.config['skills']['blacklisted_skills']}") + # # self.skill_manager.load_priority() def _wait_for_internet_connection(self): if get_neon_skills_config().get("wait_for_internet", True): diff --git a/neon_core/skills/skill_manager.py b/neon_core/skills/skill_manager.py index e2051b3d0..112a62e02 100644 --- a/neon_core/skills/skill_manager.py +++ b/neon_core/skills/skill_manager.py @@ -64,7 +64,8 @@ def run(self): super().run() def _emit_converse_error(self, message, skill_id, error_msg): - super()._emit_converse_error(message, skill_id, error_msg) + if hasattr(super(), "_emit_converse_error"): + super()._emit_converse_error(message, skill_id, error_msg) # Also emit the old error message to keep compatibility and for any # listener on the bus reply = message.reply('skill.converse.error', diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1806ab5b2..a78982663 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,7 +2,7 @@ ovos-core[skills_lgpl]~=0.0.2a10 # utils -neon-utils~=0.12,>=0.15.1a11 +neon-utils~=0.12,>=0.15.1a16 ovos_utils~=0.0.19,>=0.0.19a3 ovos-skills-manager~=0.0.10a20 ovos-plugin-manager~=0.0.4,>=0.0.7a0 diff --git a/test/test_run_neon.py b/test/test_run_neon.py index d485be8fb..622c6e4cd 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -34,7 +34,6 @@ from neon_utils.configuration_utils import get_neon_local_config sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from neon_core.run_neon import start_neon, stop_neon AUDIO_FILE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "audio_files") @@ -50,6 +49,8 @@ def setUpClass(cls) -> None: "skill-balena-wifi-setup.openvoiceos"]) local_conf.write_changes() + from neon_core.run_neon import start_neon + cls.process = Process(target=start_neon, daemon=False) cls.process.start() cls.bus = MessageBusClient() @@ -59,6 +60,8 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: + from neon_core.run_neon import stop_neon + try: cls.bus.emit(Message("neon.shutdown")) cls.bus.close() From 5e99546066c306c44ca788b67f400667973a5a6a Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 11 Mar 2022 18:34:09 +0000 Subject: [PATCH 47/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index fb756f9d7..5c79e4902 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a5" +__version__ = "21.10.3a6" From ca01d579bb3acffc1df1ca1c4aedc3f061a359dc Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 11 Mar 2022 10:50:30 -0800 Subject: [PATCH 48/70] Add user profile data to messages in intent_service (#198) * Add user profile data to messages in intent_service * Refactor 'profiles' context to 'user_profiles' per OVOS feedback --- neon_core/skills/intent_service.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index 1dc2acac8..eb0b9df13 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -35,7 +35,8 @@ from neon_utils.message_utils import get_message_user from neon_utils.metrics_utils import Stopwatch from neon_utils.log_utils import LOG -from neon_utils.configuration_utils import get_neon_device_type +from neon_utils.configuration_utils import get_neon_device_type,\ + get_neon_user_config from ovos_utils.json_helper import merge_dict from lingua_franca.parse import get_full_lang_code @@ -45,9 +46,11 @@ try: if get_neon_device_type() == "server": - from neon_transcripts_controller.transcript_db_manager import TranscriptDBManager as Transcribe + from neon_transcripts_controller.transcript_db_manager import\ + TranscriptDBManager as Transcribe else: - from neon_transcripts_controller.transcript_file_manager import TranscriptFileManager as Transcribe + from neon_transcripts_controller.transcript_file_manager import\ + TranscriptFileManager as Transcribe except ImportError: Transcribe = None @@ -58,6 +61,8 @@ def __init__(self, bus): self.config = Configuration.get().get('context', {}) self.language_config = get_lang_config() + self.default_user = get_neon_user_config() + set_default_lang(self.language_config["internal"]) self._setup_converse_handlers() @@ -149,6 +154,11 @@ def handle_utterance(self, message): message.context["timing"] = {} message.context["timing"]["handle_utterance"] = time.time() + # Ensure user profile data is present + if "user_profiles" not in message.context: + message.context["user_profiles"] = [self.default_user.content] + message.context["username"] = self.default_user.content["username"] + # Make sure there is a `transcribed` timestamp (should have been added in speech module) if not message.context["timing"].get("transcribed"): message.context["timing"]["transcribed"] = message.context["timing"]["handle_utterance"] From 55a129ae251e7442a8a63e0a080da12635297d81 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 11 Mar 2022 18:50:53 +0000 Subject: [PATCH 49/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 5c79e4902..44c28d86a 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a6" +__version__ = "21.10.3a7" From 6e5e2cf64b6d23a0f43acf3bd3879bcef1e21d04 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 16 Mar 2022 10:08:32 -0700 Subject: [PATCH 50/70] Skills module tests (#220) * Cleanup deprecated intent_service code Add intent_service unit tests * Cleanup dead code in intent_service.py Update test coverage for NeonIntentService * Add test coverage for SkillStore and SkillService with updated docs and minor bugfixes Updated annotation of skills module * Add test case for skills service * Update tests to resolve import order errors * Handle skill directory creation in NeonSkillManager Rename unit tests for clarity * Troubleshoot configured skill directories in skill_store.py * Troubleshooting skills service * Deprecate skill installation by ID with annotation to fix in OSM * Troubleshoot OSM parsing errors * Add branch spec to test URL Co-authored-by: Daniel McKnight --- .github/workflows/unit_tests.yml | 13 +- neon_core/skills/display_service.py | 2 + neon_core/skills/intent_service.py | 101 +++---- neon_core/skills/neon_skill.py | 2 + neon_core/skills/service.py | 42 +-- neon_core/skills/skill_manager.py | 27 +- neon_core/skills/skill_store.py | 125 ++++++--- neon_core/util/skill_utils.py | 9 +- test/test_skills_module.py | 398 ++++++++++++++++++++++++++++ 9 files changed, 585 insertions(+), 134 deletions(-) create mode 100644 test/test_skills_module.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 0a707619e..a81b18bab 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -1,6 +1,6 @@ # This workflow will run unit tests -name: Test Utilities +name: Test Core Modules on: push: workflow_dispatch: @@ -118,4 +118,13 @@ jobs: uses: actions/upload-artifact@v2 with: name: language-test-results - path: tests/language-test-results.xml \ No newline at end of file + path: tests/language-test-results.xml + + - name: Test Skills Module + run: | + pytest test/test_skills_module.py --doctest-modules --junitxml=tests/skills-module-test-results.xml + - name: Upload Language test results + uses: actions/upload-artifact@v2 + with: + name: skills-module-test-results + path: tests/skills-module-test-results.xml \ No newline at end of file diff --git a/neon_core/skills/display_service.py b/neon_core/skills/display_service.py index 7eef24561..1c239dce1 100644 --- a/neon_core/skills/display_service.py +++ b/neon_core/skills/display_service.py @@ -28,6 +28,8 @@ from neon_core.messagebus import get_messagebus from os.path import abspath +# TODO: Deprecate this module + def ensure_uri(s): """ diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index eb0b9df13..59cbf1741 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -30,7 +30,6 @@ from neon_core.language import get_lang_config from neon_core.processing_modules.text import TextParsersService -from copy import copy from mycroft_bus_client import Message from neon_utils.message_utils import get_message_user from neon_utils.metrics_utils import Stopwatch @@ -65,7 +64,7 @@ def __init__(self, bus): set_default_lang(self.language_config["internal"]) - self._setup_converse_handlers() + # self._setup_converse_handlers() self.parser_service = TextParsersService(self.bus) self.parser_service.start() @@ -75,32 +74,13 @@ def __init__(self, bus): else: self.transcript_service = None - def _setup_converse_handlers(self): - self.bus.on('skill.converse.error', self.handle_converse_error) - self.bus.on('skill.converse.activate_skill', - self.handle_activate_skill) - self.bus.on('skill.converse.deactivate_skill', - self.handle_deactivate_skill) - # backwards compat - self.bus.on('active_skill_request', - self.handle_activate_skill) - - def handle_activate_skill(self, message): - self.add_active_skill(message.data['skill_id']) - - def handle_deactivate_skill(self, message): - self.remove_active_skill(message.data['skill_id']) - - def reset_converse(self, message): - """Let skills know there was a problem with speech recognition""" - lang = message.data.get('lang', "en-us") - set_default_lang(lang) - for skill in copy(self.active_skills): - self.do_converse([], skill[0], lang, message) + def shutdown(self): + self.parser_service.shutdown() def _save_utterance_transcription(self, message): """ - Record a user utterance with the transcript_service. Adds the `audio_file` context to message context. + Record a user utterance with the transcript_service. + Adds the `audio_file` context to message context. Args: message (Message): message associated with user input @@ -111,21 +91,25 @@ def _save_utterance_transcription(self, message): if audio: audio = wave.open(audio, 'r') audio = audio.readframes(audio.getnframes()) - timestamp = message.context["timing"].get("transcribed", time.time()) - audio_file = self.transcript_service.write_transcript(get_message_user(message), - message.data.get('utterances', [''])[0], - timestamp, audio) + timestamp = message.context["timing"].get("transcribed", + time.time()) + audio_file = self.transcript_service.write_transcript( + get_message_user(message), + message.data.get('utterances', [''])[0], timestamp, audio) message.context["audio_file"] = audio_file - def _get_parsers_service_context(self, message, lang): + def _get_parsers_service_context(self, message: Message): """ Pipe utterance thorough text parsers to get more metadata. Utterances may be modified by any parser and context overwritten + :param message: Message to parse """ utterances = message.data.get('utterances', []) + lang = message.data.get('lang') for parser in self.parser_service.modules: # mutate utterances and retrieve extra data - utterances, data = self.parser_service.parse(parser, utterances, lang) + utterances, data = self.parser_service.parse(parser, utterances, + lang) # update message context with extra data message.context = merge_dict(message.context, data) message.data["utterances"] = utterances @@ -145,7 +129,8 @@ def handle_utterance(self, message): try: # Get language of the utterance lang = get_full_lang_code( - message.data.get('lang', self.language_config["user"])) + message.data.get('lang') or self.language_config["user"]) + message.data["lang"] = lang # Add or init timing data message.context = message.context or {} @@ -157,16 +142,19 @@ def handle_utterance(self, message): # Ensure user profile data is present if "user_profiles" not in message.context: message.context["user_profiles"] = [self.default_user.content] - message.context["username"] = self.default_user.content["username"] + message.context["username"] = \ + self.default_user.content["user"]["username"] - # Make sure there is a `transcribed` timestamp (should have been added in speech module) + # Make sure there is a `transcribed` timestamp if not message.context["timing"].get("transcribed"): - message.context["timing"]["transcribed"] = message.context["timing"]["handle_utterance"] + message.context["timing"]["transcribed"] = \ + message.context["timing"]["handle_utterance"] stopwatch = Stopwatch() - # TODO: Consider saving transcriptions after text parsers cleanup utterance. This should retain the raw - # transcription, in addition to the one modified by the parsers DM + # TODO: Consider saving transcriptions after text parsers cleanup + # utterance. This should retain the raw transcription, in addition + # to the one modified by the parsers DM # Write out text and audio transcripts if service is available with stopwatch: self._save_utterance_transcription(message) @@ -174,42 +162,23 @@ def handle_utterance(self, message): # Get text parser context with stopwatch: - self._get_parsers_service_context(message, lang) + self._get_parsers_service_context(message) message.context["timing"]["text_parsers"] = stopwatch.time # Catch empty utterances after parser service - if len([u for u in message.data["utterances"] if u.strip()]) == 0: + if len([u for u in message.data.get("utterances", []) + if u.strip()]) == 0: LOG.debug("Received empty utterance!!") - reply = message.reply('intent_aborted', - {'utterances': message.data.get('utterances', []), - 'lang': lang}) + reply = \ + message.reply('intent_aborted', + {'utterances': message.data.get('utterances', + []), + 'lang': lang}) self.bus.emit(reply) return - # now pass our modified message to mycroft-lib - message.data["lang"] = lang - # TODO: Consider how to implement 'and' parsing and converse here DM + # now pass our modified message to Mycroft + # TODO: Consider how to implement 'and' parsing and converse DM super().handle_utterance(message) except Exception as err: LOG.exception(err) - - def _converse(self, utterances, lang, message): - """ - Wraps the actual converse method to add timing data - - Args: - utterances (list): list of utterances - lang (string): 4 letter ISO language code - message (Message): message to use to generate reply - - Returns: - IntentMatch if handled otherwise None. - """ - stopwatch = Stopwatch() - with stopwatch: - match = super()._converse(utterances, lang, message) - message.context["timing"]["check_converse"] = stopwatch.time - if match: - LOG.info(f"converse handling response: {match.skill_id}") - return match - diff --git a/neon_core/skills/neon_skill.py b/neon_core/skills/neon_skill.py index 82ea56f6b..c670e1265 100644 --- a/neon_core/skills/neon_skill.py +++ b/neon_core/skills/neon_skill.py @@ -49,6 +49,8 @@ AbortQuestion, killable_event from mycroft.skills import MycroftSkill +# TODO: Deprecate this module + class UserReply(str, Enum): YES = "yes" diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index bac0a5caa..c554ea087 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -24,10 +24,10 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time +from typing import Optional from neon_utils.configuration_utils import get_neon_skills_config, \ get_neon_lang_config, get_neon_local_config -from neon_utils.net_utils import check_online from neon_utils import LOG from neon_utils.metrics_utils import announce_connection from neon_utils.signal_utils import init_signal_handlers, init_signal_bus @@ -66,9 +66,14 @@ def on_stopping(): class NeonSkillService: - def __init__(self, alive_hook=on_alive, started_hook=on_started, - ready_hook=on_ready, error_hook=on_error, - stopping_hook=on_stopping, watchdog=None): + def __init__(self, + alive_hook: callable = on_alive, + started_hook: callable = on_started, + ready_hook: callable = on_ready, + error_hook: callable = on_error, + stopping_hook: callable = on_stopping, + watchdog: Optional[callable] = None, + config: Optional[dict] = None): self.bus = None self.skill_manager = None self.event_scheduler = None @@ -79,10 +84,10 @@ def __init__(self, alive_hook=on_alive, started_hook=on_started, on_ready=ready_hook, on_error=error_hook, on_stopping=stopping_hook) - skill_config = get_neon_skills_config() - if skill_config.get("run_gui_file_server"): + self.config = config or get_neon_skills_config() + if self.config.get("run_gui_file_server"): self.http_server = start_qml_http_server( - skill_config["directory"]) + self.config["directory"]) else: self.http_server = None @@ -100,13 +105,12 @@ def start(self): self.event_scheduler = EventScheduler(self.bus) self.status = ProcessStatus('skills', self.bus, self.callbacks) SkillApi.connect_bus(self.bus) - self.skill_manager = NeonSkillManager(self.bus, self.watchdog) - self.status.set_started() - # self._wait_for_internet_connection() - # # TODO can this be removed? its a hack around msm requiring internet... - # if self.skill_manager is None: - # self._initialize_skill_manager() + self.skill_manager = NeonSkillManager(self.bus, self.watchdog, + config=self.config) self.skill_manager.start() + self.status.set_started() + + # TODO: These should be event-based in Mycroft/OVOS while not self.skill_manager.is_alive(): time.sleep(0.1) self.status.set_alive() @@ -156,12 +160,12 @@ def _register_intent_services(self): # LOG.info(f"Blacklisted={self.skill_manager.config['skills']['blacklisted_skills']}") # # self.skill_manager.load_priority() - def _wait_for_internet_connection(self): - if get_neon_skills_config().get("wait_for_internet", True): - while not check_online(): - time.sleep(1) - else: - LOG.info("Online check disabled, device may be offline") + # def _wait_for_internet_connection(self): + # if get_neon_skills_config().get("wait_for_internet", True): + # while not check_online(): + # time.sleep(1) + # else: + # LOG.info("Online check disabled, device may be offline") def shutdown(self): LOG.info('Shutting down Skills service') diff --git a/neon_core/skills/skill_manager.py b/neon_core/skills/skill_manager.py index 112a62e02..38bb77777 100644 --- a/neon_core/skills/skill_manager.py +++ b/neon_core/skills/skill_manager.py @@ -23,6 +23,9 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from os import makedirs +from os.path import isdir, expanduser + from neon_utils.configuration_utils import get_neon_skills_config from neon_utils.log_utils import LOG @@ -32,15 +35,24 @@ from mycroft.skills.skill_manager import SkillManager SKILL_MAIN_MODULE = '__init__.py' +# TODO: deprecate `SKILL_MAIN_MODULE`? class NeonSkillManager(SkillManager): def __init__(self, *args, **kwargs): + config = kwargs.pop("config") if "config" in kwargs else \ + get_neon_skills_config() + config["directory"] = expanduser(config["directory"]) super().__init__(*args, **kwargs) - self.skill_config = kwargs.get("config") or get_neon_skills_config() - self.skill_downloader = SkillsStore(skills_dir=self.skill_config["directory"], config=self.skill_config, - bus=self.bus) + self.skill_config = config + if not isdir(self.skill_config["directory"]): + LOG.warning("Creating requested skill directory") + makedirs(self.skill_config["directory"]) + + self.skill_downloader = SkillsStore( + skills_dir=self.skill_config["directory"], + config=self.skill_config, bus=self.bus) self.skill_downloader.skills_dir = self.skill_config["directory"] def download_or_update_defaults(self): @@ -62,12 +74,3 @@ def run(self): """Load skills and update periodically from disk and internet.""" self.download_or_update_defaults() super().run() - - def _emit_converse_error(self, message, skill_id, error_msg): - if hasattr(super(), "_emit_converse_error"): - super()._emit_converse_error(message, skill_id, error_msg) - # Also emit the old error message to keep compatibility and for any - # listener on the bus - reply = message.reply('skill.converse.error', - data=dict(skill_id=skill_id, error=error_msg)) - self.bus.emit(reply) diff --git a/neon_core/skills/skill_store.py b/neon_core/skills/skill_store.py index fd1bae00d..104a5b24c 100644 --- a/neon_core/skills/skill_store.py +++ b/neon_core/skills/skill_store.py @@ -23,15 +23,18 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from os import makedirs +from os.path import isdir +from typing import List, Optional, Generator, Union from ovos_skills_manager.osm import OVOSSkillsManager from ovos_skills_manager.skill_entry import SkillEntry -from neon_utils import LOG +from neon_utils.logger import LOG from neon_utils.net_utils import check_online from neon_utils.authentication_utils import repo_is_neon from neon_utils.configuration_utils import get_neon_skills_config from datetime import datetime, timedelta - from neon_utils.messagebus_utils import get_messagebus + from neon_core.util.skill_utils import get_remote_entries from mycroft.skills.event_scheduler import EventSchedulerInterface @@ -55,6 +58,9 @@ def __init__(self, skills_dir, config=None, bus=None): self.schedule_sync() def schedule_sync(self): + """ + Use the EventScheduler to update osm with updated appstore data + """ # every X hours interval = 60 * 60 * self.config["appstore_sync_interval"] when = datetime.now() + timedelta(seconds=interval) @@ -63,6 +69,9 @@ def schedule_sync(self): name="appstores.sync") def schedule_update(self): + """ + Use the EventScheduler to update default skills + """ # every X hours interval = 60 * 60 * self.config["auto_update_interval"] when = datetime.now() + timedelta(seconds=interval) @@ -71,7 +80,11 @@ def schedule_update(self): name="default_skills.update") def handle_update(self, _): + """ + Scheduled action to update installed skills + """ try: + # TODO: Include non-default installed skills? self.install_default_skills(update=True) except Exception as e: if check_online(): @@ -83,9 +96,13 @@ def handle_update(self, _): LOG.error("no internet, skipped skills update") def handle_sync_appstores(self, _): + """ + Scheduled action to update OSM appstore listings + """ try: self.osm.sync_appstores() except Exception as e: + # TODO: OSM should raise more specific exceptions if check_online(): # if there is internet log the error LOG.exception(e) @@ -98,6 +115,15 @@ def shutdown(self): self.scheduler.shutdown() def load_osm(self): + """ + Get an authenticated instance of OSM if not disabled + """ + from ovos_utils.skills import get_skills_folder + osm_skill_dir = get_skills_folder() + if osm_skill_dir != self.skills_dir: + LOG.warning(f"OSM configured local skills: {osm_skill_dir}") + if not isdir(osm_skill_dir): + makedirs(osm_skill_dir) if self.disabled: return None osm = OVOSSkillsManager() @@ -150,6 +176,9 @@ def default_skills(self): return self._default_skills def authenticate_neon(self): + """ + Enable and authenticate the Neon skills store + """ self.osm.enable_appstore("neon") neon = self.osm.get_appstore("neon") neon_token = self.config.get("neon_token") @@ -159,36 +188,53 @@ def authenticate_neon(self): neon.authenticate(bootstrap=False) def deauthenticate_neon(self): + """ + Clear authentication for the Neon skills store + """ neon = self.osm.get_appstore("neon") neon.clear_authentication() - def get_skill_entry(self, skill): + def get_skill_entry(self, skill: str) -> Optional[SkillEntry]: + """ + Build a SkillEntry object from the passed skill URL or ID + :param skill: str skill to search + :returns best match of input skill or None + """ if "http" in skill: if "/neongeckocom/" in skill.lower(): # TODO: This is just patching OSM updates DM store_skill = None else: store_skill = self.osm.search_skills_by_url(skill) - if not store_skill: - # skill is not in any appstore - if "/neon" in skill.lower() and "github" in skill: - self.authenticate_neon() - entry = SkillEntry.from_github_url(skill) - self.deauthenticate_neon() - else: - entry = SkillEntry.from_github_url(skill) - return entry - elif isinstance(store_skill, list): - return store_skill[0] + if isinstance(store_skill, SkillEntry): + return store_skill + elif isinstance(store_skill, list): + return store_skill[0] + elif isinstance(store_skill, Generator): + # Return the first item + for s in store_skill: + return s + # skill is not in any appstore + if "/neon" in skill.lower() and "github" in skill: + self.authenticate_neon() + entry = SkillEntry.from_github_url(skill) + self.deauthenticate_neon() else: - return store_skill + entry = SkillEntry.from_github_url(skill) + return entry elif "." in skill: - return self.osm.search_skills_by_id(skill) + # Return the first item + for skill in self.osm.search_skills_by_id(skill): + return skill return None - def get_remote_entries(self, url): - """ parse url and return a list of SkillEntry, - expects 1 skill per line, can be a skill_id or url""" + def get_remote_entries(self, url: str) -> List[str]: + """ + Wraps a call to `neon_core.util.skill_utils.get_remote_entries` to + include authentication. + :param url: URL of skill list to parse (one skill per line) + :returns: list of skills by name, url, and/or ID + """ authenticated = False if repo_is_neon(url): self.authenticate_neon() @@ -198,30 +244,45 @@ def get_remote_entries(self, url): self.deauthenticate_neon() return skills_list - def _parse_config_entry(self, entry): + def _parse_config_entry(self, entry: Union[list, str]) -> List[SkillEntry]: """ - entry can be - - an url (str) to download essential skill list - - can be a list of skill repo urls, or skill_ids - - list (list) of names (str) - - list (list) of skill_urls (str) + Parse a config value into a list of SkillEntry objects + :param entry: Configuration value, one of: + - str url of a skill list of skill repo urls, or skill_ids + - list of skill IDs (str) + - list of skill_urls (str) + :returns: list of parsed SkillEntry objects """ if self.disabled: + LOG.warning("Ignoring parse request as SkillStore is disabled") return [] if isinstance(entry, str): if not entry.startswith("http"): - raise ValueError # TODO new exception + raise ValueError(f"passed entry not a valid URL or list: " + f"{entry}") skills = self.get_remote_entries(entry) elif isinstance(entry, list): skills = entry else: - raise ValueError("invalid configuration entry") - for idx, skill in enumerate(skills): - skills[idx] = self.get_skill_entry(skill) - skills = [s for s in skills if s] - return skills + raise ValueError(f"invalid configuration entry: {entry}") + + skill_entries = list() + for skill in skills: + entry = self.get_skill_entry(skill) + if entry: + skill_entries.append(entry) - def install_skill(self, skill_entry, folder=None, *args, **kwargs): + return skill_entries + + def install_skill(self, skill_entry: SkillEntry, + folder: Optional[str] = None, *args, **kwargs) -> bool: + """ + Install a SkillEntry to a local directory. + args/kwargs are passed to `skill_entry.install` + :param skill_entry: SkillEntry to install + :param folder: Skill installation directory (default self.skills_dir) + :returns: True if skill is installed or updated + """ if self.disabled: return False self.authenticate_neon() diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index 1fe72e1fe..8a15b3941 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -121,9 +121,12 @@ def install_skills_default(config: dict = None): clear_github_token() -def get_remote_entries(url): - """ parse url and return a list of SkillEntry, - expects 1 skill per line, can be a skill_id or url""" +def get_remote_entries(url: str): + """ + Parse a skill list at a given URL + :param url: URL of skill list to parse (one skill per line) + :returns: list of skills by name, url, and/or ID + """ r = SESSION.get(url) if not r.ok: LOG.warning(f"Cached response returned: {r.status_code}") diff --git a/test/test_skills_module.py b/test/test_skills_module.py new file mode 100644 index 000000000..c42af8881 --- /dev/null +++ b/test/test_skills_module.py @@ -0,0 +1,398 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import importlib +import os +import sys +import unittest +import wave +from copy import deepcopy +from os.path import join, dirname +from threading import Thread, Event +from time import time, sleep + +from mock import Mock +from mock.mock import patch +from mycroft_bus_client import Message +from ovos_utils.messagebus import FakeBus + + +sys.path.append(os.path.dirname(os.path.dirname(__file__))) +from neon_core import NeonIntentService + + +class MockEventSchedulerInterface(Mock): + def __init__(self, *_, **__): + super().__init__() + + +class TestIntentService(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.bus = FakeBus() + cls.intent_service = NeonIntentService(cls.bus) + + @classmethod + def tearDownClass(cls) -> None: + cls.intent_service.shutdown() + + def test_save_utterance_transcription(self): + self.intent_service.transcript_service = Mock() + transcribe_time = time() + test_message = Message("recognizer_loop:utterance", + {"utterances": ["test 1", "test one"], + "lang": "en-us"}, + {"timing": {"transcribed": transcribe_time}}) + self.intent_service._save_utterance_transcription(test_message) + self.intent_service.transcript_service.write_transcript.\ + assert_called_once_with(None, test_message.data["utterances"][0], + transcribe_time, None) + + test_audio = os.path.join(os.path.dirname(__file__), + "audio_files", "stop.wav") + test_message.context["raw_audio"] = test_audio + audio = wave.open(test_audio, 'r') + audio = audio.readframes(audio.getnframes()) + self.intent_service._save_utterance_transcription(test_message) + self.intent_service.transcript_service.write_transcript. \ + assert_called_with(None, test_message.data["utterances"][0], + transcribe_time, audio) + + def test_get_parsers_service_context(self): + utterances = ["test 1", "test one"] + lang = "en-us" + test_message = Message("recognizer_loop:utterance", + {"utterances": deepcopy(utterances), + "lang": lang}, {}) + + def mod_1_parse(utterances, lang): + utterances.append("mod 1 parsed") + return utterances, {"parser_context": "mod_1"} + + def mod_2_parse(utterances, lang): + utterances.append("mod 2 parsed") + return utterances, {"parser_context": "mod_2"} + + real_modules = self.intent_service.parser_service.loaded_modules + mod_1 = Mock() + mod_1.priority = 2 + mod_1.parse = mod_1_parse + mod_2 = Mock() + mod_2.priority = 100 + mod_2.parse = mod_2_parse + self.intent_service.parser_service.loaded_modules = \ + {"test_mod_1": {"instance": mod_1}, + "test_mod_2": {"instance": mod_2}} + self.intent_service._get_parsers_service_context(test_message) + self.assertEqual(test_message.context["parser_context"], "mod_2") + self.assertNotEqual(utterances, test_message.data['utterances']) + self.assertEqual(len(test_message.data['utterances']), + len(utterances) + 2) + + mod_2.priority = 1 + self.intent_service._get_parsers_service_context(test_message) + self.assertEqual(test_message.context["parser_context"], "mod_1") + self.intent_service.parser_service.loaded_modules = real_modules + + valid_parsers = {"cancel", "entity_parser", "translator"} + self.assertTrue(all([p for p in valid_parsers if p in + self.intent_service.parser_service.loaded_modules])) + + @patch("mycroft.skills.intent_service.IntentService.handle_utterance") + def test_handle_utterance(self, patched): + intent_service = NeonIntentService(self.bus) + + test_message_invalid = Message("test", {"utterances": [' ', ' ']}) + intent_service.handle_utterance(test_message_invalid) + patched.assert_not_called() + + test_message_valid = Message("test", {"utterances": ["test", "tests"]}) + intent_service.handle_utterance(test_message_valid) + + patched.assert_called_once_with(test_message_valid) + self.assertIn("lang", test_message_valid.data) + self.assertIn('-', test_message_valid.data['lang']) # full code + self.assertIsInstance(test_message_valid.context["timing"], dict) + self.assertIsInstance(test_message_valid.context["user_profiles"], + list) + self.assertIsInstance(test_message_valid.context["username"], str) + + intent_service.shutdown() + + +class TestSkillManager(unittest.TestCase): + @patch("neon_core.skills.skill_store.SkillsStore.install_default_skills") + @patch("mycroft.skills.skill_manager.SkillManager.run") + def test_download_or_update_defaults(self, patched_run, patched_installer): + from neon_core.skills.skill_manager import NeonSkillManager + config = { + "disable_osm": False, + "auto_update": True, + "directory": join(dirname(__file__), "skill_module_skills") + } + manager = NeonSkillManager(FakeBus(), config=config) + manager.run() + patched_run.assert_called_once() + self.assertEqual(manager.skill_config, config) + patched_installer.assert_called_once() + + patched_installer.reset_mock() + manager.skill_config["auto_update"] = False + manager.download_or_update_defaults() + patched_installer.assert_not_called() + manager.stop() + + +class TestSkillStore(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + import mycroft.skills.event_scheduler + mocked_scheduler = MockEventSchedulerInterface + mycroft.skills.event_scheduler.EventSchedulerInterface = \ + mocked_scheduler + import neon_core.skills.skill_store + importlib.reload(neon_core.skills.skill_store) + + from neon_core.skills.skill_store import SkillsStore + + cls.essential = ["https://github.com/OpenVoiceOS/skill-ovos-homescreen/tree/main"] + cls.config = { + "disable_osm": False, + "auto_update": True, + "auto_update_interval": 1, + "appstore_sync_interval": 1, + "neon_token": None, + "essential_skills": cls.essential, + "default_skills": "https://raw.githubusercontent.com/NeonGeckoCom/" + "neon_skills/TEST_ShortSkillsList/skill_lists/" + "TEST-SHORTLIST" + } + cls.skill_dir = join(dirname(__file__), "skill_module_skills") + cls.bus = FakeBus() + cls.skill_store = SkillsStore(cls.skill_dir, cls.config, cls.bus) + + @classmethod + def tearDownClass(cls) -> None: + cls.skill_store.shutdown() + + def test_00_store_init(self): + self.assertEqual(self.skill_store.config, self.config) + self.assertFalse(self.skill_store.disabled) + self.assertEqual(self.skill_store.skills_dir, self.skill_dir) + self.assertEqual(self.skill_store.bus, self.bus) + self.assertIsNotNone(self.skill_store.osm) + self.assertIsInstance(self.skill_store.scheduler, + MockEventSchedulerInterface) + self.assertEqual( + self.skill_store.scheduler.schedule_repeating_event.call_count, 2) + + def test_schedule_sync(self): + pass + + def test_schedule_update(self): + pass + + def test_handle_update(self): + pass + + def test_handle_sync_appstores(self): + pass + + def test_handle_load_osm(self): + from ovos_skills_manager import OVOSSkillsManager + self.skill_store.disabled = True + self.assertIsNone(self.skill_store.load_osm()) + + self.skill_store.disabled = False + self.assertIsInstance(self.skill_store.load_osm(), OVOSSkillsManager) + + def test_essential_skills(self): + self.assertFalse(self.skill_store.disabled) + self.assertEqual(len(self.skill_store.essential_skills), + len(self.essential)) + + def test_default_skills(self): + self.assertFalse(self.skill_store.disabled) + self.assertIsInstance(self.skill_store.default_skills, list) + self.assertGreater(len(self.skill_store.default_skills), 0) + + def test_authenticate_neon(self): + pass + + def test_deauthenticate_neon(self): + pass + + def test_get_skill_entry(self): + # TODO: Implement skills by ID after fixing in OSM + # TODO: Support missing branch specs + from ovos_skills_manager import SkillEntry + url = "https://github.com/OpenVoiceOS/skill-ovos-homescreen/tree/main" + # skill_id = "skill-ovos-homescreen.openvoiceos" + url_entry = self.skill_store.get_skill_entry(url) + self.assertIsInstance(url_entry, SkillEntry) + # id_entry = self.skill_store.get_skill_entry(skill_id) + # self.assertIsInstance(id_entry, SkillEntry) + # self.assertEqual(url_entry.skill_name, id_entry.skill_name) + + def test_get_remote_entries(self): + from neon_core.util.skill_utils import get_remote_entries + test_urls = { + "https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS", + "https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-PREMIUM-SKILLS" + } + for url in test_urls: + self.assertEqual(self.skill_store.get_remote_entries(url), + get_remote_entries(url)) + + def test_parse_config_entry(self): + # TODO: Implement skills by ID after fixing in OSM + from ovos_skills_manager import SkillEntry + self.skill_store.osm.disable_appstore("local") + + valid_entry_url = self.config["default_skills"] + valid_entry_list_url = self.config["essential_skills"] + # valid_entry_list_id = ["skill-ovos-homescreen.openvoiceos", + # "caffeinewiz.neon.neongeckocom"] + + self.skill_store.disabled = True + self.assertEqual(self.skill_store._parse_config_entry(valid_entry_url), + list()) + self.skill_store.disabled = False + + # with self.assertRaises(ValueError): + # self.skill_store._parse_config_entry(valid_entry_list_id[0]) + + with self.assertRaises(ValueError): + self.skill_store._parse_config_entry(None) + + default_entries = self.skill_store._parse_config_entry(valid_entry_url) + self.assertIsInstance(default_entries, list) + self.assertTrue(all([isinstance(x, SkillEntry) + for x in default_entries]), default_entries) + + essential_entries = \ + self.skill_store._parse_config_entry(valid_entry_list_url) + self.assertIsInstance(essential_entries, list) + self.assertEqual(len(essential_entries), 1, essential_entries) + self.assertIsInstance(essential_entries[0], SkillEntry) + + # list_entries = \ + # self.skill_store._parse_config_entry(valid_entry_list_id) + # self.assertIsInstance(list_entries, list) + # self.assertEqual(len(list_entries), 2, list_entries) + # self.assertTrue(all([isinstance(x, SkillEntry) + # for x in list_entries]), list_entries) + + def test_install_skill(self): + skill_entry = Mock() + install_dir = self.skill_dir + + def skill_entry_installer(*_, **kwargs): + self.assertEqual(kwargs["folder"], install_dir) + if kwargs.get("update"): + return True + return False + + self.skill_store.disabled = True + self.assertFalse(self.skill_store.install_skill(skill_entry)) + self.skill_store.disabled = False + + skill_entry.install = skill_entry_installer + self.assertFalse(self.skill_store.install_skill(skill_entry)) + + install_dir = "/tmp" + self.assertTrue(self.skill_store.install_skill(skill_entry, "/tmp", + update=True)) + + def test_install_default_skills(self): + install_skill = Mock() + real_install_skill = self.skill_store.install_skill + self.skill_store.install_skill = install_skill + + self.skill_store.disabled = True + self.assertEqual(self.skill_store.install_default_skills(), list()) + self.assertEqual(self.skill_store.install_default_skills(True), list()) + self.skill_store.disabled = False + + install_skill.reset_mock() + skills = self.skill_store.install_default_skills(False) + self.assertEqual(install_skill.call_count, len(skills)) + + install_skill.reset_mock() + skills = self.skill_store.install_default_skills(True) + self.assertEqual(install_skill.call_count, len(skills)) + + self.skill_store.install_skill = real_install_skill + + +class TestSkillService(unittest.TestCase): + @patch("mycroft.skills.skill_manager.SkillManager.run") + def test_neon_skills_service(self, run): + from neon_core.skills.service import NeonSkillService + from neon_core.skills.skill_manager import NeonSkillManager + from mycroft.util.process_utils import ProcessState + + config = { + "disable_osm": False, + "auto_update": True, + "directory": join(dirname(__file__), "skill_module_skills"), + "run_gui_file_server": True + } + + started = Event() + + def started_hook(): + started.set() + + alive_hook = Mock() + ready_hook = Mock() + error_hook = Mock() + stopping_hook = Mock() + service = NeonSkillService(alive_hook, started_hook, ready_hook, + error_hook, stopping_hook, config=config) + self.assertIsNotNone(service.http_server) + self.assertEqual(service.config, config) + service.bus = FakeBus() + service.bus.connected_event = Event() + skills_thread = Thread(target=service.start, daemon=True) + skills_thread.start() + + started.wait(30) + run.assert_called_once() + + self.assertIsInstance(service.skill_manager, NeonSkillManager) + service.skill_manager.status.state = ProcessState.ALIVE + sleep(1) + alive_hook.assert_called_once() + service.skill_manager.status.state = ProcessState.READY + sleep(1) + ready_hook.assert_called_once() + + service.shutdown() + stopping_hook.assert_called_once() + skills_thread.join(10) + + +if __name__ == "__main__": + unittest.main() From b869ee4f2c06ed2ed06d1445135f164f23667a96 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 16 Mar 2022 17:09:31 +0000 Subject: [PATCH 51/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 44c28d86a..586756d18 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a7" +__version__ = "21.10.3a8" From 2bb6d2131f364a5a92d14b7e769fcddaf3a11c7f Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 16 Mar 2022 15:57:57 -0700 Subject: [PATCH 52/70] Patch translation bugs (#221) * Pass input language to translation plugin in translator Update intent service to update message.data['lang'] when utterances are translated * Fix language handling for profile context Co-authored-by: Daniel McKnight --- .../text/modules/translator/__init__.py | 12 ++++++++---- neon_core/skills/intent_service.py | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/neon_core/processing_modules/text/modules/translator/__init__.py b/neon_core/processing_modules/text/modules/translator/__init__.py index 7c4974d52..5b69c2ddd 100644 --- a/neon_core/processing_modules/text/modules/translator/__init__.py +++ b/neon_core/processing_modules/text/modules/translator/__init__.py @@ -37,19 +37,23 @@ def __init__(self, name="utterance_translator", priority=5): self.lang_detector = DetectorFactory.create() self.translator = TranslatorFactory.create() - def parse(self, utterances, lang="en-us"): + def parse(self, utterances, lang=None): metadata = [] for idx, ut in enumerate(utterances): try: original = ut detected_lang = self.lang_detector.detect(original) - LOG.debug("Detected language: {lang}".format(lang=detected_lang)) + if lang and detected_lang != lang.split('-', 1)[0]: + LOG.warning(f"Specified lang: {lang} but detected {detected_lang}") + else: + LOG.debug(f"Detected language: {detected_lang}") if detected_lang != self.language_config["internal"].split("-")[0]: utterances[idx] = self.translator.translate(original, - self.language_config["internal"]) + self.language_config["internal"], lang.split('-', 1)[0] + or detected_lang) # add language metadata to context metadata += [{ - "source_lang": lang, + "source_lang": lang or self.language_config['internal'], "detected_lang": detected_lang, "internal": self.language_config["internal"], "was_translated": detected_lang != self.language_config["internal"].split("-")[0], diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index 59cbf1741..e600acd79 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -177,6 +177,11 @@ def handle_utterance(self, message): self.bus.emit(reply) return + # TODO: Try the original lang and fallback to translation + # If translated, make sure message.data['lang'] is updated + if message.context.get("translation_data", + [{}])[0].get("was_translated"): + message.data["lang"] = self.language_config["internal"] # now pass our modified message to Mycroft # TODO: Consider how to implement 'and' parsing and converse DM super().handle_utterance(message) From fbaa9f97110910d57acfef8b50a461146aaf9d90 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 16 Mar 2022 22:58:22 +0000 Subject: [PATCH 53/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 586756d18..14a72a4a7 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a8" +__version__ = "21.10.3a9" From cba4a22f80ae110d63f20906caf7828108ee3052 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 7 Apr 2022 18:44:01 -0700 Subject: [PATCH 54/70] Add init file to text.modules module to be included in installations (#222) Co-authored-by: Daniel McKnight --- .../text/modules/__init__.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 neon_core/processing_modules/text/modules/__init__.py diff --git a/neon_core/processing_modules/text/modules/__init__.py b/neon_core/processing_modules/text/modules/__init__.py new file mode 100644 index 000000000..248ce6262 --- /dev/null +++ b/neon_core/processing_modules/text/modules/__init__.py @@ -0,0 +1,24 @@ +# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# # All trademark and other rights reserved by their respective owners +# # Copyright 2008-2021 Neongecko.com Inc. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 116a6a5e960570a3cbd98ce9c064515408235e82 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 8 Apr 2022 01:44:21 +0000 Subject: [PATCH 55/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 14a72a4a7..c16f08330 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a9" +__version__ = "21.10.3a10" From 1ceeb502f16f07d7053af11aa4a161dfdd0a694b Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 8 Apr 2022 10:40:53 -0700 Subject: [PATCH 56/70] Docker bugfixes (#223) * Refactor docker overlay files Patch skill dependencies in docker.txt Update skill dependency installation to install before module load * Troubleshooting local speech processing test failures * Troubleshooting local speech processing test failures * Remove outdated deepspeech config from neon.conf * Rollback testing changes Co-authored-by: Daniel McKnight --- Dockerfile | 19 +++-------- .../{ => config}/ngi_local_conf.yml | 0 docker_overlay/{asoundrc => root/.asoundrc} | 0 .../{ => root/.config/neon}/neon.conf | 0 .../settings.json | 0 docker_overlay/root/run.sh | 32 +++++++++++++++++++ neon_core/cli.py | 5 ++- neon_core/configuration/neon.conf | 6 +--- requirements/docker.txt | 3 ++ setup.py | 3 +- 10 files changed, 47 insertions(+), 21 deletions(-) rename docker_overlay/{ => config}/ngi_local_conf.yml (100%) rename docker_overlay/{asoundrc => root/.asoundrc} (100%) rename docker_overlay/{ => root/.config/neon}/neon.conf (100%) rename docker_overlay/{skill_settings => root/.local/share/neon/skills}/skill-ovos-homescreen.openvoiceos/settings.json (100%) create mode 100644 docker_overlay/root/run.sh create mode 100644 requirements/docker.txt diff --git a/Dockerfile b/Dockerfile index a4a904fe0..f63ae2b25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,21 +22,12 @@ RUN apt-get update && \ ADD . /neon_core WORKDIR /neon_core -COPY docker_overlay/asoundrc /root/.asoundrc - -RUN mkdir -p /root/.config/neon -COPY docker_overlay/neon.conf /root/.config/neon/neon.conf - RUN pip install wheel && \ - pip install . + pip install .[docker] -CMD ["neon", "run-skills", "-i", "/skills"] +COPY docker_overlay/ / +RUN chmod ugo+x /root/run.sh +CMD ["/root/run.sh"] FROM base as default_skills - -RUN mkdir -p /root/.local/share/neon -# TODO: This path should move to config in the future -COPY docker_overlay/skill_settings /root/.local/share/neon/skills -COPY docker_overlay/ngi_local_conf.yml /config/ -RUN neon-install-default-skills -RUN rm /config/ngi_local_conf.yml \ No newline at end of file +RUN neon-install-default-skills \ No newline at end of file diff --git a/docker_overlay/ngi_local_conf.yml b/docker_overlay/config/ngi_local_conf.yml similarity index 100% rename from docker_overlay/ngi_local_conf.yml rename to docker_overlay/config/ngi_local_conf.yml diff --git a/docker_overlay/asoundrc b/docker_overlay/root/.asoundrc similarity index 100% rename from docker_overlay/asoundrc rename to docker_overlay/root/.asoundrc diff --git a/docker_overlay/neon.conf b/docker_overlay/root/.config/neon/neon.conf similarity index 100% rename from docker_overlay/neon.conf rename to docker_overlay/root/.config/neon/neon.conf diff --git a/docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json b/docker_overlay/root/.local/share/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json similarity index 100% rename from docker_overlay/skill_settings/skill-ovos-homescreen.openvoiceos/settings.json rename to docker_overlay/root/.local/share/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json diff --git a/docker_overlay/root/run.sh b/docker_overlay/root/run.sh new file mode 100644 index 000000000..ae5b8ffc9 --- /dev/null +++ b/docker_overlay/root/run.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Plugin installation must occur in a separate thread, before module load, for the entry point to be loaded. +neon install-skill-requirements /skills +neon run-skills \ No newline at end of file diff --git a/neon_core/cli.py b/neon_core/cli.py index 669bf9168..8e5105f90 100644 --- a/neon_core/cli.py +++ b/neon_core/cli.py @@ -102,7 +102,10 @@ def run_skills(install_skills): from neon_core.skills.__main__ import main if install_skills: click.echo(f"Handling installation of skills in: {install_skills}") - install_local_skills(install_skills) + try: + install_local_skills(install_skills) + except ValueError as e: + click.echo(f"Skill Installation Failed: {e}") click.echo("Starting Skills Service") main() click.echo("Skills Service Shutdown") diff --git a/neon_core/configuration/neon.conf b/neon_core/configuration/neon.conf index 3442f2324..66296cfc8 100644 --- a/neon_core/configuration/neon.conf +++ b/neon_core/configuration/neon.conf @@ -435,11 +435,7 @@ "stt": { // Engine. Options: "mycroft", "google", "wit", "ibm", "kaldi", "bing", // "houndify", "deepspeech_server", "govivace", "yandex" - "module": "google", - "deepspeech_stream_local": { - "model_path": "~/.local/share/neon/deepspeech-0.8.1-models.pbmm", - "scorer_path": "~/.local/share/neon/deepspeech-0.8.1-models.scorer" - } + "module": "google" // "deepspeech_server": { // "uri": "http://localhost:8080/stt" // }, diff --git a/requirements/docker.txt b/requirements/docker.txt new file mode 100644 index 000000000..b67161711 --- /dev/null +++ b/requirements/docker.txt @@ -0,0 +1,3 @@ +# These are just patching default skill dependencies +ifaddr~=0.1 +pyjokes \ No newline at end of file diff --git a/setup.py b/setup.py index 06af6b32d..256dda35c 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,8 @@ def get_requirements(requirements_filename: str): "remote": get_requirements("remote_speech_processing.txt"), "vision": get_requirements("vision.txt"), "test": get_requirements("test.txt"), - "pi": get_requirements("pi.txt") + "pi": get_requirements("pi.txt"), + "docker": get_requirements("docker.txt") }, packages=find_packages(include=['neon_core*']), package_data={'neon_core': ['res/precise_models/*', 'res/snd/*', 'res/text/*/*.voc', 'res/text/*/*.dialog', From 4f5a8bd54accc96686339ff774e46f6f5af488c9 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 8 Apr 2022 17:41:12 +0000 Subject: [PATCH 57/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index c16f08330..b10fff495 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a10" +__version__ = "21.10.3a11" From 0f1ec57609fbecbf42b7c34086738ced9b551388 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:23:13 -0700 Subject: [PATCH 58/70] add license tests and workflow (#224) * add license tests and workflow * Update license tests to include additional apt dependencies and license overrides * Update license tests overrides and whitelist * Fix typo in license whitelist * Fix typo in license overrides * Add pyparsing license override Co-authored-by: jarbasai Co-authored-by: Daniel McKnight --- .github/workflows/license_tests.yml | 35 +++++++++++++++ test/license_tests.py | 69 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 .github/workflows/license_tests.yml create mode 100644 test/license_tests.py diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 000000000..8d154e43c --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,35 @@ +name: Run License Tests +on: + push: + workflow_dispatch: + +jobs: + license_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install gcc libfann-dev swig libssl-dev portaudio19-dev git libpulse-dev + - name: Install core repo + run: | + pip install . + - name: Install licheck + run: | + pip install git+https://github.com/NeonJarbas/lichecker + - name: Install test dependencies + run: | + pip install pytest pytest-timeout pytest-cov + - name: Test Licenses + run: | + pytest test/license_tests.py \ No newline at end of file diff --git a/test/license_tests.py b/test/license_tests.py new file mode 100644 index 000000000..95f835ca4 --- /dev/null +++ b/test/license_tests.py @@ -0,0 +1,69 @@ +import unittest +from pprint import pprint + +from lichecker import LicenseChecker + +# these packages dont define license in setup.py +# manually verified and injected +license_overrides = { + "kthread": "MIT", + 'yt-dlp': "Unlicense", + 'pyxdg': 'GPL-2.0', + 'ptyprocess': 'ISC license', + 'psutil': 'BSD3', + 'pyaudio': 'MIT', + 'petact': 'MIT', + "precise-runner": "Apache-2.0", + 'soupsieve': 'MIT', + 'setuptools': 'MIT', + 'sonopy': 'Apache-2.0', + "ovos-skill-installer": "MIT", + "python-dateutil": "Apache-2.0", + "pyparsing": "MIT" +} +# explicitly allow these packages that would fail otherwise +whitelist = ["neon-core", + "neon-audio", + "neon-speech", + "neon-gui", + "neon-messagebus", + "neon-api-proxy" + # "python-vlc" # This may be installed optionally + ] + +# validation flags +allow_nonfree = False +allow_viral = False +allow_unknown = False +allow_unlicense = True +allow_ambiguous = False + +pkg_name = "neon-core" + + +class TestLicensing(unittest.TestCase): + @classmethod + def setUpClass(self): + licheck = LicenseChecker(pkg_name, + license_overrides=license_overrides, + whitelisted_packages=whitelist, + allow_ambiguous=allow_ambiguous, + allow_unlicense=allow_unlicense, + allow_unknown=allow_unknown, + allow_viral=allow_viral, + allow_nonfree=allow_nonfree) + print("Package", pkg_name) + print("Version", licheck.version) + print("License", licheck.license) + print("Transient Requirements (dependencies of dependencies)") + pprint(licheck.transient_dependencies) + self.licheck = licheck + + def test_license_compliance(self): + print("Package Versions") + pprint(self.licheck.versions) + + print("Dependency Licenses") + pprint(self.licheck.licenses) + + self.licheck.validate() From c1ab1bb9168fa3f557a6fcab4bf65a4e3a22d82f Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 14 Apr 2022 22:23:32 +0000 Subject: [PATCH 59/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index b10fff495..bcc531ce7 100644 --- a/version.py +++ b/version.py @@ -23,4 +23,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a11" +__version__ = "21.10.3a12" From 179fef42afb29c3991766b5db11a8606b39bfbac Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:40:15 -0700 Subject: [PATCH 60/70] Update license headers to updated 2022 version (#225) Co-authored-by: Daniel McKnight --- LICENSE.md | 23 ++++++++-- neon_core/__init__.py | 9 ++-- neon_core/cli.py | 1 + neon_core/config.py | 9 ++-- neon_core/configuration/__init__.py | 9 ++-- neon_core/dialog/__init__.py | 9 ++-- neon_core/language/__init__.py | 9 ++-- neon_core/launcher.py | 9 ++-- neon_core/messagebus/__init__.py | 9 ++-- neon_core/processing_modules/__init__.py | 9 ++-- neon_core/processing_modules/text/__init__.py | 9 ++-- .../text/modules/__init__.py | 9 ++-- .../text/modules/cancel/__init__.py | 9 ++-- .../text/modules/entity_parser/__init__.py | 9 ++-- .../text/modules/translator/__init__.py | 9 ++-- neon_core/run_neon.py | 9 ++-- neon_core/skills/__init__.py | 9 ++-- neon_core/skills/__main__.py | 9 ++-- neon_core/skills/decorators.py | 9 ++-- neon_core/skills/display_service.py | 9 ++-- neon_core/skills/fallback_skill.py | 9 ++-- neon_core/skills/intent_service.py | 9 ++-- neon_core/skills/neon_skill.py | 9 ++-- neon_core/skills/service.py | 9 ++-- neon_core/skills/skill_manager.py | 9 ++-- neon_core/skills/skill_store.py | 9 ++-- neon_core/stt/__init__.py | 9 ++-- neon_core/tts/__init__.py | 9 ++-- neon_core/util/__init__.py | 9 ++-- neon_core/util/diagnostic_utils.py | 46 +++++++++++-------- neon_core/util/runtime_utils.py | 9 ++-- neon_core/util/skill_utils.py | 9 ++-- setup.py | 9 ++-- setup.sh | 9 ++-- test/setup_dev_local.sh | 9 ++-- test/setup_remote.sh | 9 ++-- test/test_configuration.py | 9 ++-- test/test_diagnostic_utils.py | 9 ++-- test/test_language.py | 9 ++-- test/test_qml_file_server.py | 9 ++-- test/test_run_modules.py | 9 ++-- test/test_run_neon.py | 9 ++-- test/test_setup_dev_local.py | 9 ++-- test/test_setup_remote.py | 9 ++-- test/test_skill_utils.py | 9 ++-- test/test_skills_module.py | 9 ++-- version.py | 9 ++-- version_bump.py | 45 ++++++++++-------- 48 files changed, 338 insertions(+), 173 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index de0ff30ec..0f8494496 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,8 +1,21 @@ # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System # All trademark and other rights reserved by their respective owners # Copyright 2008-2021 Neongecko.com Inc. -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +# BSD-3 License + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/neon_core/__init__.py b/neon_core/__init__.py index 62d942b91..c0994d1d1 100644 --- a/neon_core/__init__.py +++ b/neon_core/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/cli.py b/neon_core/cli.py index 8e5105f90..f87d09156 100644 --- a/neon_core/cli.py +++ b/neon_core/cli.py @@ -25,6 +25,7 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from threading import Thread import click diff --git a/neon_core/config.py b/neon_core/config.py index 26751ed47..8f3d55180 100644 --- a/neon_core/config.py +++ b/neon_core/config.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/configuration/__init__.py b/neon_core/configuration/__init__.py index b434a72b9..c2bdacabe 100644 --- a/neon_core/configuration/__init__.py +++ b/neon_core/configuration/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/dialog/__init__.py b/neon_core/dialog/__init__.py index 2b080e72a..a7c7f21ed 100644 --- a/neon_core/dialog/__init__.py +++ b/neon_core/dialog/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/language/__init__.py b/neon_core/language/__init__.py index 204d6a2d3..1d2c8d0cc 100644 --- a/neon_core/language/__init__.py +++ b/neon_core/language/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/launcher.py b/neon_core/launcher.py index eb613efc2..754bd7956 100644 --- a/neon_core/launcher.py +++ b/neon_core/launcher.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/messagebus/__init__.py b/neon_core/messagebus/__init__.py index b158696d2..7d2c3cd21 100644 --- a/neon_core/messagebus/__init__.py +++ b/neon_core/messagebus/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/__init__.py b/neon_core/processing_modules/__init__.py index da0b0f9e8..be9944523 100644 --- a/neon_core/processing_modules/__init__.py +++ b/neon_core/processing_modules/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/text/__init__.py b/neon_core/processing_modules/text/__init__.py index 11d9c76e0..7b392de25 100644 --- a/neon_core/processing_modules/text/__init__.py +++ b/neon_core/processing_modules/text/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/text/modules/__init__.py b/neon_core/processing_modules/text/modules/__init__.py index 248ce6262..718d1b001 100644 --- a/neon_core/processing_modules/text/modules/__init__.py +++ b/neon_core/processing_modules/text/modules/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/text/modules/cancel/__init__.py b/neon_core/processing_modules/text/modules/cancel/__init__.py index eee02ff01..7f7d1c23f 100644 --- a/neon_core/processing_modules/text/modules/cancel/__init__.py +++ b/neon_core/processing_modules/text/modules/cancel/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/text/modules/entity_parser/__init__.py b/neon_core/processing_modules/text/modules/entity_parser/__init__.py index 61534b52f..29899d271 100644 --- a/neon_core/processing_modules/text/modules/entity_parser/__init__.py +++ b/neon_core/processing_modules/text/modules/entity_parser/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/processing_modules/text/modules/translator/__init__.py b/neon_core/processing_modules/text/modules/translator/__init__.py index 5b69c2ddd..2f30bf755 100644 --- a/neon_core/processing_modules/text/modules/translator/__init__.py +++ b/neon_core/processing_modules/text/modules/translator/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/run_neon.py b/neon_core/run_neon.py index 7738a5cf4..f28722db9 100644 --- a/neon_core/run_neon.py +++ b/neon_core/run_neon.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/__init__.py b/neon_core/skills/__init__.py index 525d5c077..d5669b2cc 100644 --- a/neon_core/skills/__init__.py +++ b/neon_core/skills/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/__main__.py b/neon_core/skills/__main__.py index 6f1c39795..43d146dfb 100644 --- a/neon_core/skills/__main__.py +++ b/neon_core/skills/__main__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/decorators.py b/neon_core/skills/decorators.py index 7ad338e28..1d0c893fa 100644 --- a/neon_core/skills/decorators.py +++ b/neon_core/skills/decorators.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/display_service.py b/neon_core/skills/display_service.py index 1c239dce1..4555d3963 100644 --- a/neon_core/skills/display_service.py +++ b/neon_core/skills/display_service.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/fallback_skill.py b/neon_core/skills/fallback_skill.py index eadc295cf..67dd33295 100644 --- a/neon_core/skills/fallback_skill.py +++ b/neon_core/skills/fallback_skill.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index e600acd79..026edde72 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/neon_skill.py b/neon_core/skills/neon_skill.py index c670e1265..434deedfa 100644 --- a/neon_core/skills/neon_skill.py +++ b/neon_core/skills/neon_skill.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/service.py b/neon_core/skills/service.py index c554ea087..26e66abe5 100644 --- a/neon_core/skills/service.py +++ b/neon_core/skills/service.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/skill_manager.py b/neon_core/skills/skill_manager.py index 38bb77777..239b59be5 100644 --- a/neon_core/skills/skill_manager.py +++ b/neon_core/skills/skill_manager.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/skills/skill_store.py b/neon_core/skills/skill_store.py index 104a5b24c..69d88c545 100644 --- a/neon_core/skills/skill_store.py +++ b/neon_core/skills/skill_store.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/stt/__init__.py b/neon_core/stt/__init__.py index 087474ef7..004ff5045 100644 --- a/neon_core/stt/__init__.py +++ b/neon_core/stt/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/tts/__init__.py b/neon_core/tts/__init__.py index abc77bfce..9e888067d 100644 --- a/neon_core/tts/__init__.py +++ b/neon_core/tts/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/util/__init__.py b/neon_core/util/__init__.py index 248ce6262..718d1b001 100644 --- a/neon_core/util/__init__.py +++ b/neon_core/util/__init__.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/util/diagnostic_utils.py b/neon_core/util/diagnostic_utils.py index f5fb50b14..5f9ada0e8 100644 --- a/neon_core/util/diagnostic_utils.py +++ b/neon_core/util/diagnostic_utils.py @@ -1,21 +1,31 @@ -# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# -# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved -# -# Notice of License - Duplicating this Notice of License near the start of any file containing -# a derivative of this software is a condition of license for this software. -# Friendly Licensing: -# No charge, open source royalty free use of the Neon AI software source and object is offered for -# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and -# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai -# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai -# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. -# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) -# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds -# -# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. -# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 -# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import json import socket import glob diff --git a/neon_core/util/runtime_utils.py b/neon_core/util/runtime_utils.py index 291c91afe..73ccac783 100644 --- a/neon_core/util/runtime_utils.py +++ b/neon_core/util/runtime_utils.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index 8a15b3941..ec7b63918 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/setup.py b/setup.py index 256dda35c..613ce3807 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/setup.sh b/setup.sh index 6ec63402f..692e580b8 100644 --- a/setup.sh +++ b/setup.sh @@ -1,8 +1,11 @@ #!/bin/bash -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/setup_dev_local.sh b/test/setup_dev_local.sh index dd17eceb6..e204f0d82 100644 --- a/test/setup_dev_local.sh +++ b/test/setup_dev_local.sh @@ -1,8 +1,11 @@ #!/bin/bash -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/setup_remote.sh b/test/setup_remote.sh index 22072b458..2497646c0 100644 --- a/test/setup_remote.sh +++ b/test/setup_remote.sh @@ -1,8 +1,11 @@ #!/bin/bash -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_configuration.py b/test/test_configuration.py index 514859ba7..e41c71410 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_diagnostic_utils.py b/test/test_diagnostic_utils.py index 64aa77898..3fd972458 100644 --- a/test/test_diagnostic_utils.py +++ b/test/test_diagnostic_utils.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_language.py b/test/test_language.py index 5ffd9a11d..6ecf4ed47 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_qml_file_server.py b/test/test_qml_file_server.py index 5937c6f0a..dbaae69d1 100644 --- a/test/test_qml_file_server.py +++ b/test/test_qml_file_server.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_run_modules.py b/test/test_run_modules.py index 19fc18fae..2c496996d 100644 --- a/test/test_run_modules.py +++ b/test/test_run_modules.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_run_neon.py b/test/test_run_neon.py index 622c6e4cd..303127482 100644 --- a/test/test_run_neon.py +++ b/test/test_run_neon.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_setup_dev_local.py b/test/test_setup_dev_local.py index 3fcb55a81..c5a6b6e70 100644 --- a/test/test_setup_dev_local.py +++ b/test/test_setup_dev_local.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_setup_remote.py b/test/test_setup_remote.py index cfac14ba9..42a1c04ba 100644 --- a/test/test_setup_remote.py +++ b/test/test_setup_remote.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_skill_utils.py b/test/test_skill_utils.py index c14fa16cf..aac9843f2 100644 --- a/test/test_skill_utils.py +++ b/test/test_skill_utils.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/test/test_skills_module.py b/test/test_skills_module.py index c42af8881..6a7a1224d 100644 --- a/test/test_skills_module.py +++ b/test/test_skills_module.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/version.py b/version.py index bcc531ce7..bbc2a218e 100644 --- a/version.py +++ b/version.py @@ -1,6 +1,9 @@ -# # NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# # All trademark and other rights reserved by their respective owners -# # Copyright 2008-2021 Neongecko.com Inc. +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # 1. Redistributions of source code must retain the above copyright notice, diff --git a/version_bump.py b/version_bump.py index a55f15865..5a01c558c 100644 --- a/version_bump.py +++ b/version_bump.py @@ -1,21 +1,30 @@ -# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System -# -# Copyright 2008-2021 Neongecko.com Inc. | All Rights Reserved -# -# Notice of License - Duplicating this Notice of License near the start of any file containing -# a derivative of this software is a condition of license for this software. -# Friendly Licensing: -# No charge, open source royalty free use of the Neon AI software source and object is offered for -# educational users, noncommercial enthusiasts, Public Benefit Corporations (and LLCs) and -# Social Purpose Corporations (and LLCs). Developers can contact developers@neon.ai -# For commercial licensing, distribution of derivative works or redistribution please contact licenses@neon.ai -# Distributed on an "AS IS” basis without warranties or conditions of any kind, either express or implied. -# Trademarks of Neongecko: Neon AI(TM), Neon Assist (TM), Neon Communicator(TM), Klat(TM) -# Authors: Guy Daniels, Daniel McKnight, Regina Bloomstine, Elon Gasper, Richard Leeds -# -# Specialized conversational reconveyance options from Conversation Processing Intelligence Corp. -# US Patents 2008-2021: US7424516, US20140161250, US20140177813, US8638908, US8068604, US8553852, US10530923, US10530924 -# China Patent: CN102017585 - Europe Patent: EU2156652 - Patents Pending +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import fileinput from os.path import join, dirname From 3b802849dc1ca794e408deb73935f296dbeb9e3b Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 14 Apr 2022 22:40:33 +0000 Subject: [PATCH 61/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index bbc2a218e..29ed66b40 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a12" +__version__ = "21.10.3a13" From 2796cdd3b2ef43380735301d4c80fdc9033ced9d Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Tue, 26 Apr 2022 17:21:10 -0700 Subject: [PATCH 62/70] Fix unit test errors (#228) * Set automated test devType to server to resolve speech/audio module test failures * Update setup tests * Remove neon-core-server dependency (is now a standalone package) * Handle errors initializing transcription service Co-authored-by: Daniel McKnight --- neon_core/skills/intent_service.py | 8 +++++--- requirements/server.txt | 2 +- test/setup_dev_local.sh | 2 +- test/setup_remote.sh | 2 +- test/test_setup_dev_local.py | 2 +- test/test_setup_remote.py | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/neon_core/skills/intent_service.py b/neon_core/skills/intent_service.py index 026edde72..ed76ded29 100644 --- a/neon_core/skills/intent_service.py +++ b/neon_core/skills/intent_service.py @@ -72,10 +72,12 @@ def __init__(self, bus): self.parser_service = TextParsersService(self.bus) self.parser_service.start() + self.transcript_service = None if Transcribe: - self.transcript_service = Transcribe() - else: - self.transcript_service = None + try: + self.transcript_service = Transcribe() + except Exception as e: + LOG.exception(e) def shutdown(self): self.parser_service.shutdown() diff --git a/requirements/server.txt b/requirements/server.txt index db108c2d7..667d2b9d9 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -1,2 +1,2 @@ -neon-core-server @ git+https://github.com/NeonGeckoCom/neon-core-server +#neon-core-server @ git+https://github.com/NeonGeckoCom/neon-core-server neon-transcripts-controller @ git+https://github.com/NeonGeckoCom/transcripts_controller diff --git a/test/setup_dev_local.sh b/test/setup_dev_local.sh index e204f0d82..e635c70d2 100644 --- a/test/setup_dev_local.sh +++ b/test/setup_dev_local.sh @@ -38,7 +38,7 @@ export devMode=true # false will enable fullscreen gui, isolated directo export autoStart=false # enables neonAI to run at login of installUser export autoUpdate=false # enables neonAI to check for updates at runtime export devName=${HOSTNAME} # device name used to identify uploads -export installServer=false # enables neonAI server module +export installServer=true # enables neonAI server module export sttModule="deepspeech_stream_local" export ttsModule="neon_tts_mimic" diff --git a/test/setup_remote.sh b/test/setup_remote.sh index 2497646c0..24854f343 100644 --- a/test/setup_remote.sh +++ b/test/setup_remote.sh @@ -39,7 +39,7 @@ export devMode=false # false will enable fullscreen gui, isolated direct export autoStart=false # enables neonAI to run at login of installUser export autoUpdate=false # enables neonAI to check for updates at runtime export devName=${HOSTNAME} # device name used to identify uploads -export installServer=false # enables neonAI server module +export installServer=true # enables neonAI server module export sttModule="google_cloud_streaming" export ttsModule="amazon" diff --git a/test/test_setup_dev_local.py b/test/test_setup_dev_local.py index c5a6b6e70..94d0e3e9e 100644 --- a/test/test_setup_dev_local.py +++ b/test/test_setup_dev_local.py @@ -34,7 +34,7 @@ class TestSetupDevLocal(unittest.TestCase): def test_config_from_setup(self): local_config = get_neon_local_config() - self.assertEqual(local_config["devVars"]["devType"], "linux") + self.assertEqual(local_config["devVars"]["devType"], "server") self.assertTrue(local_config["prefFlags"]["devMode"]) self.assertEqual(local_config["stt"]["module"], "deepspeech_stream_local") self.assertEqual(local_config["tts"]["module"], "neon_tts_mimic") diff --git a/test/test_setup_remote.py b/test/test_setup_remote.py index 42a1c04ba..8fec8b6ad 100644 --- a/test/test_setup_remote.py +++ b/test/test_setup_remote.py @@ -34,7 +34,7 @@ class TestSetupRemote(unittest.TestCase): def test_config_from_setup(self): local_config = get_neon_local_config() - self.assertEqual(local_config["devVars"]["devType"], "linux") + self.assertEqual(local_config["devVars"]["devType"], "server") self.assertFalse(local_config["prefFlags"]["devMode"]) self.assertEqual(local_config["stt"]["module"], "google_cloud_streaming") self.assertEqual(local_config["tts"]["module"], "amazon") From 3a693867725a2373753f70acc457277775c016cf Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 27 Apr 2022 00:21:28 +0000 Subject: [PATCH 63/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 29ed66b40..607f3af76 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a13" +__version__ = "21.10.3a14" From afbd1121bab06aa557bf818dca83757d07f7dfbc Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 27 Apr 2022 10:17:12 -0700 Subject: [PATCH 64/70] Minor updates and bugfixes, Update docker overlay and documentation (#227) * Update launcher.py with speech service Fix bug in appstore authentication in skill_utils Update gitignore to include local config files Update docker overlay files Add docker-compose with reference config file Update ovos dependency specs Add Quick Start section to README.md * Update docker overlay file to reference main branch of skills * Update neon-utils dependency version * Fix skills image reference * Cleanup logging Update requirements Add docker .env file for default skills * Add first_run param to demo skill settings overlay file Co-authored-by: Daniel McKnight --- .gitignore | 4 + Dockerfile | 5 + README.md | 84 ++++++- docker/.env | 1 + docker/docker-compose.yml | 77 ++++++ docker/ngi_local_conf.yml | 234 ++++++++++++++++++ docker_overlay/config/ngi_local_conf.yml | 11 +- .../skill-demo.neongeckocom/settings.json | 4 + .../settings.json | 1 + neon_core/launcher.py | 12 +- neon_core/util/skill_utils.py | 3 +- requirements/requirements.txt | 8 +- 12 files changed, 433 insertions(+), 11 deletions(-) create mode 100644 docker/.env create mode 100644 docker/docker-compose.yml create mode 100755 docker/ngi_local_conf.yml create mode 100644 docker_overlay/root/.config/neon/skills/skill-demo.neongeckocom/settings.json rename docker_overlay/root/{.local/share => .config}/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json (88%) diff --git a/.gitignore b/.gitignore index fcee7028b..82686ea67 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ doc/_build/ test/unittests/skills/test_skill/settings.json test_conf.json .pytest_cache/ + +# Neon dev files +.*.tmp +./ngi*.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f63ae2b25..0308409fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,11 @@ RUN pip install wheel && \ COPY docker_overlay/ / RUN chmod ugo+x /root/run.sh + +# TODO: Below link is patching a bug in the homescreen skill/ovos-utils +RUN mkdir /opt/mycroft && \ + ln -s /root/.local/share/neon/skills /opt/mycroft/skills + CMD ["/root/run.sh"] FROM base as default_skills diff --git a/README.md b/README.md index affe2ea4f..e542830e7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Table of Contents +0. [Quick Start](#quick-start) 1. [Optional Service Account Setup](#optional-service-account-setup) * [a. Google Cloud Speech](#a-google-cloud-speech-setup) * [b. Amazon Polly and Translate](#b-amazon-polly-and-translate-setup) @@ -11,15 +12,94 @@ * [a. Activating the venv](#a-activating-the-venv) * [c. Running Tests](#c-running-tests) * [d. Troubleshooting](#d-troubleshooting) -6. [Making Changes](#making-code-changes) +5. [Making Changes](#making-code-changes) * [a. System Overview](#a-system-overview) * [b. Creating a Skill](#b-creating-a-skill) -8. [Removing and re-installing Neon](#removing-and-re-installing-neon-ai) +6. [Removing and re-installing Neon](#removing-and-re-installing-neon-ai) # Welcome to Neon AI Neon AI is an open source voice assistant. Follow these instructions to start using Neon on your computer. If you are using a Raspberry Pi, you may use the prebuilt image available [on our website](https://neon.ai/DownloadNeonAI). +# Quick Start +The fastest method for getting started with Neon is to run the modules in Docker containers. +The `docker` directory contains everything you need to run Neon Core with default skills. + +## a. Prerequisite Setup +You will need `docker` and `docker-compose` available. Docker provides updated guides for installing +[docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/). +Neon Core is only tested on Ubuntu, but should be compatible with any linux distribution that uses +[PulseAudio](https://www.freedesktop.org/wiki/Software/PulseAudio/). + +## b. Running Neon +You can clone the repository, or just copy the `docker` directory contents onto your local system; this document will +assume that the repository is cloned to: `~/NeonCore`. + +You can start all core modules with: +```shell +# cd into the directory containing docker-compose.yml +cd ~/NeonCore/docker +docker-compose up -d +``` + +Stop all modules with: +```shell +# cd into the directory containing docker-compose.yml +cd ~/NeonCore/docker +docker-compose down +``` + +### Optional GUI +The Mycroft GUI is an optional component that can be run on Linux host systems. +The GUI is available with instructions [on GitHub](https://github.com/MycroftAI/mycroft-gui) + +## c. Interacting with Neon +With the containers running, you can interact with Neon by voice (i.e. "hey Neon, what time is it?"), or using one of +our CLI utilities, like [mana](https://pypi.org/project/neon-mana-utils/) or the +[neon_cli_client](https://pypi.org/project/neon-cli-client/). +You can view module logs via docker with: + +```shell +docker logs -f neon-skills # skills module +docker logs -f neon-speech # voice module (STT and WW) +docker logs -f neon-audio # audio module (TTS) +docker logs -f neon-gui # gui module (Optional) +docker logs -f neon-messagebus # messagebus module (includes signal manager) +``` + +## d. Skill Development +By default, the skills container includes a set of default skills to provide base functionality. +You can pass a local skill directory into the skills container to develop skills and have them +reloaded in real-time for testing. Just set the environment variable `NEON_SKILLS_DIR` before starting +the skills module. Dependency installation is handled on container start automatically. + +```shell +export NEON_SKILLS_DIR=~/PycharmProjects/SKILLS +cd ~/NeonCore/docker +docker-compose up +``` + +To run the skills module without any bundled skills, the image referenced in `docker-compose.yml` can be changed from: + +```yaml + neon-skills: + container_name: neon-skills + image: ghcr.io/neongeckocom/neon_skills-default_skills:dev +``` +to: +```yaml + neon-skills: + container_name: neon-skills + image: ghcr.io/neongeckocom/neon_skills:dev +``` + +## e. Configuration +The `ngi_local_conf.yml` file included in the `docker` directory contains a default configuration +that may be modified to specify different plugins and other runtime settings. +The `docker` directory is mounted read-only to `/config` in each of the containers, +so model files may be placed there and the configuration updated to use different STT/TTS plugins with +local models. + # Optional Service Account Setup There are several online services that may be used with Neon. Speech-to-Text (STT) and Text-to-Speech (TTS) may be run locally, but remote implementations are often faster and more accurate. Following are some instructions for getting diff --git a/docker/.env b/docker/.env new file mode 100644 index 000000000..5238106aa --- /dev/null +++ b/docker/.env @@ -0,0 +1 @@ +NEON_SKILLS_DIR=./ \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..73278e959 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,77 @@ +version: '3.1' +networks: + neon-core: +volumes: + config: + driver_opts: + type: none-config + o: bind + device: ./ +services: + neon-messagebus: + container_name: neon-messagebus + image: ghcr.io/neongeckocom/neon_messagebus:dev + ports: + - 8181:8181 + networks: + neon-core: + aliases: + - messagebus + volumes: + - config:/config:ro + neon-speech: + container_name: neon-speech + image: ghcr.io/neongeckocom/neon_speech:dev + networks: + - neon-core + volumes: + - config:/config:ro + - ~/.config/pulse/cookie:/root/.config/pulse/cookie:ro + - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro + environment: + - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native + - PULSE_COOKIE=/root/.config/pulse/cookie + devices: + - /dev/snd:/dev/snd + neon-skills: + container_name: neon-skills + image: ghcr.io/neongeckocom/neon_skills-default_skills:dev + networks: + - neon-core + ports: + - 8000:8000 + volumes: + - config:/config:ro + - ~/.config/pulse/cookie:/root/.config/pulse/cookie:ro + - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro + - ${NEON_SKILLS_DIR}:/skills:ro + environment: + - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native + - PULSE_COOKIE=/root/.config/pulse/cookie + devices: + - /dev/snd:/dev/snd + neon-audio: + container_name: neon-audio + image: ghcr.io/neongeckocom/neon_audio:dev + networks: + - neon-core + volumes: + - config:/config:ro + - ~/.config/pulse/cookie:/root/.config/pulse/cookie:ro + - ${XDG_RUNTIME_DIR}/pulse:${XDG_RUNTIME_DIR}/pulse:ro + environment: + - PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native + - PULSE_COOKIE=/root/.config/pulse/cookie + devices: + - /dev/snd:/dev/snd + neon-gui: + container_name: neon-gui + image: ghcr.io/neongeckocom/neon_gui:dev + networks: + neon-core: + aliases: + - gui + ports: + - 18181:18181 + volumes: + - config:/config:ro \ No newline at end of file diff --git a/docker/ngi_local_conf.yml b/docker/ngi_local_conf.yml new file mode 100755 index 000000000..1231b9923 --- /dev/null +++ b/docker/ngi_local_conf.yml @@ -0,0 +1,234 @@ +prefFlags: + codeSource: git + devMode: true + autoStart: true + autoUpdate: false + # Flag if gui is installed locally + localGui: true + # Flag if enclosure service should run to generate gui events + guiEvents: true + notifyRelease: true + showDemo: false + optDiags: false + metrics: false + + saveAudio: false + saveText: true +devVars: + devName: docker + devType: generic + # generic, server, pi, neonK, neonX, neonPi, neonAlpha, neonU + version: 2021-02-26-185606 + installUser: neon + touchDev: + micDev: + camDev: 0 + soundDev: + defaultVolume: 60 + defaultMicVolume: 100 + +hotwords: + Hey Neon: {module: ovos-ww-plugin-pocketsphinx, phonemes: HH EY . N IY AA N ., threshold: 1e-20, + lang: en-us, sample_rate: 16000, listen: true, sound: snd/start_listening.wav, + local_model_file: None} +interface: +# True, False + display_neon_brain: false + wake_word_enabled: true + clap_commands_enabled: false + blink_commands_enabled: false + random_dialog_enabled: false + confirm_listening: true + mute_on_listen: false + # True=Muted, False=lowered vol + use_hesitation: false + +gestures: + clapThreshold: 10.0e10 + +audioService: + backends: + local: {type: simple, active: true} + vlc: {type: vlc, active: true, duck: true} + defaultBackend: vlc + debug: true + +padatious: + intent_cache: ~/.neon/intent_cache + train_delay: 4 + single_thread: false + +websocket: + host: neon-messagebus + port: 8181 + route: /core + ssl: false + allow_self_signed: true + ssl_cert: + ssl_key: +gui: + # Host information for the gui server + lang: en-us + enclosure: generic + host: neon-gui + port: 18181 + route: /gui + ssl: false + file_server: http://192.168.1.110:8000 + resource_root: https://0000.us/klatchat/app/files/neon_qml +listener: + sample_rate: 16000 + channels: 1 + wake_word_upload: {disable: true, url: https://training.mycroft.ai/precise/upload} + mute_during_output: true + duck_while_listening: 0.3 + dev_index: + phoneme_duration: 120 + multiplier: 1.0 + energy_ratio: 1.5 + wake_word: hey neon + stand_up_word: wake up + recording_timeout: 10.0 + recording_timeout_with_silence: 3.0 + +skills: + auto_update: false + install_essential: true + install_default: true + debug: true + blacklist: [] + priority: [skill-weather.neongeckocom, skill-date_time.neongeckocom, skill-about.neongeckocom] + essential_skills: [] + default_skills: https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS-DOCKER + skill_manager: osm + # recommended osm, optional msm + appstore_sync_interval: 6.0 + # time between server syncs in hours + auto_update_interval: 24.0 + # time between automatic skill updates in hours + msm_ver: false + repo_url: https://github.com/MycroftAI/mycroft-skills + repo_branch: '18.08' + data_dir: ~/.neon/msm + neon_token: + wait_for_internet: false + run_gui_file_server: true +audio_parsers: + blacklist: [gender] +text_parsers: + blacklist: [] +session: + ttl: 180 +tts: + module: coqui + package_spec: neon-tts-plugin-coqui + mozilla: {request_url: http://0.0.0.0:5002/api/tts?} + mozilla_remote: {api_url: https://mtts.2022.us/api/tts} + mimic: {voice: ap} + mimic2: {lang: en-us, url: https://mimic-api.mycroft.ai/synthesize?text=} +stt: + module: deepspeech_stream_local +logs: + blacklist: [enclosure.mouth.viseme, enclosure.mouth.display] +device: + mac: 00:00:00:00:00:00 + ip4: 127.0.0.1 + ip6: ::1 + ver: '' + +api: + url: https://api.mycroft.ai + version: v1 + update: true + disabled: true + sync_skill_settings: false +remoteVars: + coreGit: + coreBranch: + skillsGit: + skillsBranch: + guiGit: + guiBranch: + url: https://api.mycroft.ai + ver: v1 + remoteHost: 167.172.112.7 + + enableConnection: true +gnome: + favApps: + appFolders: "['accessories', 'office', 'Neongecko']" + +dirVars: + coreDir: /opt/neon + # coreDir is depreciated for packaged Neon Installations + rootDir: ~/.local/share/neon + confDir: ~/.config/neon + cacheDir: ~/.cache/neon + skillsDir: ~/.local/share/neon/skills + ngiDir: /opt/neon/NGI + # ngiDir is depreciated for packaged Neon Installations + guiDir: + tempDir: /tmp/neon + docsDir: ~/Documents/NeonGecko + diagsDir: ~/Documents/NeonGecko/Diagnostics + ipcDir: /tmp/neon/ipc + # Changes to ipcDir must also be made in NGI/utilities/configHelper + musicDir: ~/Music + videoDir: ~/Videos/NeonGecko + picsDir: ~/Pictures/NeonGecko + logsDir: /var/log/mycroft + repoDir: ~/.neon/skills-repo + padatiousDir: ~/.neon/intent_cache + +# File paths should be absolute (or ~/ relative) or relative to /res/ (see resolve_resource_file) +fileVars: + sshKey: ~/.ssh/id_rsa + notify: snd/loaded.wav + +sounds: + startListening: snd/start_listening.wav + endListening: snd/end_listening.wav + acknowledge: snd/acknowledge.mp3 + +ttsOpts: {None: '', 'Chinese, Mandarin': zh-ZH, Danish: da-DK, Dutch: nl-NL, 'English, Australian': en-AU, + 'English, British': en-GB, 'English, Indian': en-IN, 'English, US': en-US, 'English, Welsh': en-GB-WLS, + French: fr-FR, 'French, Canadian': fr-CA, Hindi: hi-IN, German: de-DE, Icelandic: is-IS, + Italian: it-IT, Japanese: ja-JP, Korean: ko-KR, Norwegian: nb-NO, Polish: pl-PL, + 'Portuguese, Brazilian': pt-BR, 'Portuguese, European': pt-PT, Romanian: ro-RO, + Russian: ru-RU, 'Spanish, European': es-ES, 'Spanish, Mexican': es-MX, 'Spanish, US': es-US, + Swedish: sv-SE, Turkish: tr-TR, Welsh: cy-GB} +sttOpts: {Deutsch (Deutschland): de-DE, English (United States): en-US, Español (España): es-ES, + Español (México): es-MX, Français (Canada): fr-CA, Français (France): fr-FR, Italiano (Italia): it-IT, + Português (Portugal): pt-PT} +sttSpokenOpts: {Afrikaans: af-ZA, Amharic: am-ET, Armenian: hy-AM, Azerbaijani: az-AZ, + Indonesian: id-ID, Malay: ms-MY, Bengali: bn-BD, Catalan: ca-ES, Czech: cs-CZ, Danish: da-DK, + German: de-DE, 'English, British': en-GB, 'English, American': en-US, Spanish: es-ES, + 'Spanish, American': es-US, 'Spanish, Mexican': es-MX, Basque: eu-ES, Filipino: fil-PH, + 'French, Canadian': fr-CA, French: fr-FR, Galician: gl-ES, Georgian: ka-GE, Gujarati: gu-IN, + Croatian: hr-HR, Zulu: zu-ZA, Icelandic: is-IS, Italian: it-IT, Javanese: jv-ID, + Kannada: kn-IN, Khmer: km-KH, Lao: lo-LA, Latvian: lv-LV, Lithuanian: lt-LT, Hungarian: hu-HU, + Malayalam: ml-IN, Marathi: mr-IN, Dutch: nl-NL, Nepali: ne-NP, Norwegian Bokmål: nb-NO, + Polish: pl-PL, 'Portuguese, Brazilian': pt-BR, Portuguese: pt-PT, Romanian: ro-RO, + Sinhala: si-LK, Slovak: sk-SK, Slovenian: sl-SI, Sundanese: su-ID, Swahili: sw-TZ, + Finnish: fi-FI, Swedish: sv-SE, Tamil: ta-IN, Telugu: te-IN, Vietnamese: vi-VN, + Turkish: tr-TR, Urdu: ur-PK, Greek: el-GR, Bulgarian: bg-BG, Russian: ru-RU, Serbian: sr-RS, + Ukrainian: uk-UA, Hebrew: he-IL, Arabic: ar-IL, Persian: fa-IR, Hindi: hi-IN, Thai: th-TH, + Korean: ko-KR, Taiwanese: zh-TW, 'Chinese, Cantonese': yue-Hant-HK, Japanese: ja-JP, + 'Chinese, Mandarin': zh} +MQ: + server: api.neon.ai + users: + mq_handler: + user: neon_api_utils + password: Klatchat2021 +language: + boost: false + core_lang: en-us + translation_module: libretranslate_plug + detection_module: libretranslate_detection_plug + libretranslate: + libretranslate_host: https://translate.neon.ai:5000 +signal: + use_signal_files: true + max_wait_seconds: 300 + diff --git a/docker_overlay/config/ngi_local_conf.yml b/docker_overlay/config/ngi_local_conf.yml index 72fb1ba4b..6746c1ab6 100644 --- a/docker_overlay/config/ngi_local_conf.yml +++ b/docker_overlay/config/ngi_local_conf.yml @@ -2,6 +2,13 @@ skills: essential_skills: [] default_skills: https://raw.githubusercontent.com/NeonGeckoCom/neon_skills/master/skill_lists/DEFAULT-SKILLS-DOCKER skill_manager: osm - dirVars: - skillsDir: /root/.local/share/neon/skills \ No newline at end of file + skillsDir: /root/.local/share/neon/skills + +websocket: + host: neon-messagebus + port: 8181 + +gui: + host: neon-gui + port: 18181 diff --git a/docker_overlay/root/.config/neon/skills/skill-demo.neongeckocom/settings.json b/docker_overlay/root/.config/neon/skills/skill-demo.neongeckocom/settings.json new file mode 100644 index 000000000..22f9c1b53 --- /dev/null +++ b/docker_overlay/root/.config/neon/skills/skill-demo.neongeckocom/settings.json @@ -0,0 +1,4 @@ +{ + "prompt_on_start": false, + "__mycroft_skill_firstrun": false +} \ No newline at end of file diff --git a/docker_overlay/root/.local/share/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json b/docker_overlay/root/.config/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json similarity index 88% rename from docker_overlay/root/.local/share/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json rename to docker_overlay/root/.config/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json index 21bbcf5e0..c17a34998 100644 --- a/docker_overlay/root/.local/share/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json +++ b/docker_overlay/root/.config/neon/skills/skill-ovos-homescreen.openvoiceos/settings.json @@ -3,5 +3,6 @@ "datetime_skill": "skill-date_time.neongeckocom", "examples_skill": "skill-about.neongeckocom", "wallpaper": "default.jpg", + "examples_enabled": true, "__mycroft_skill_firstrun": false } diff --git a/neon_core/launcher.py b/neon_core/launcher.py index 754bd7956..f6fddf9e5 100644 --- a/neon_core/launcher.py +++ b/neon_core/launcher.py @@ -25,12 +25,13 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from mycroft.lock import Lock from mycroft.util import wait_for_exit_signal, reset_sigint_handler from neon_messagebus.service import NeonBusService from neon_core.skills.service import NeonSkillService from neon_gui.service import NeonGUIService -from time import sleep +from neon_speech.service import NeonSpeechClient reset_sigint_handler() @@ -51,9 +52,16 @@ skills = NeonSkillService() skills.start() +speech = NeonSpeechClient() +speech.start() + wait_for_exit_signal() +speech.shutdown() skills.shutdown() gui.shutdown() bus.shutdown() -lock.delete() \ No newline at end of file + +# TODO: Add audio service when implemented DM + +lock.delete() diff --git a/neon_core/util/skill_utils.py b/neon_core/util/skill_utils.py index ec7b63918..c300517ef 100644 --- a/neon_core/util/skill_utils.py +++ b/neon_core/util/skill_utils.py @@ -84,6 +84,7 @@ def install_skills_from_list(skills_to_install: list, config: dict = None): if config.get("neon_token"): token_set = True set_github_token(config["neon_token"]) + LOG.info(f"Added token to request headers: {config.get('neon_token')}") for url in skills_to_install: try: normalized_url = normalize_github_url(url) @@ -97,6 +98,7 @@ def install_skills_from_list(skills_to_install: list, config: dict = None): else: LOG.warning(f"Requested Skill not in Neon skill store ({url})") entry = osm.skill_entry_from_url(url) + LOG.debug(entry.json) osm.install_skill(entry, skill_dir) if not os.path.isdir(os.path.join(skill_dir, entry.uuid)): @@ -116,7 +118,6 @@ def install_skills_default(config: dict = None): """ config = config or get_neon_skills_config() skills_list = config["default_skills"] - set_github_token(config.get("neon_token")) if isinstance(skills_list, str): skills_list = get_remote_entries(skills_list) assert isinstance(skills_list, list) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a78982663..c2e0fca38 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,12 +1,12 @@ # mycroft ovos-core[skills_lgpl]~=0.0.2a10 +# TODO: Update to stable version # utils -neon-utils~=0.12,>=0.15.1a16 +neon-utils~=0.16 ovos_utils~=0.0.19,>=0.0.19a3 -ovos-skills-manager~=0.0.10a20 -ovos-plugin-manager~=0.0.4,>=0.0.7a0 -# TODO: Update to stable versions +ovos-skills-manager~=0.0.10 +ovos-plugin-manager~=0.0.16 # default plugins neon-lang-plugin-libretranslate~=0.1,>=0.1.2 From d234679d18ad5eae1be727d07f821a3bd1086bbe Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 27 Apr 2022 17:17:34 +0000 Subject: [PATCH 65/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 607f3af76..997947fb1 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a14" +__version__ = "21.10.3a15" From 2253f89ebf0ce34d24c6d2b0d5055e18cc11adb3 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 9 May 2022 16:33:30 -0700 Subject: [PATCH 66/70] Bump ovos-utils and ovos-core dependency specs (#229) * Bump ovos-utils and ovos-core dependency specs * Update ovos-core dependency spec to release version Co-authored-by: Daniel McKnight --- requirements/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c2e0fca38..e55410fe2 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,10 +1,10 @@ # mycroft -ovos-core[skills_lgpl]~=0.0.2a10 +ovos-core[skills_lgpl]~=0.0.3 # TODO: Update to stable version # utils neon-utils~=0.16 -ovos_utils~=0.0.19,>=0.0.19a3 +ovos_utils~=0.0.20 ovos-skills-manager~=0.0.10 ovos-plugin-manager~=0.0.16 From 955bf2e8ed90206e65c503f00bd344579e53f854 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Mon, 9 May 2022 23:33:52 +0000 Subject: [PATCH 67/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 997947fb1..9f7b7116c 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a15" +__version__ = "21.10.3a16" From 2dff62a865b44288ce92efe7a99818ffbe0680f5 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 9 May 2022 20:23:50 -0700 Subject: [PATCH 68/70] Update core module version specs (#230) Update Docker overlay files Co-authored-by: Daniel McKnight --- docker/docker-compose.yml | 2 +- docker/ngi_local_conf.yml | 3 ++- requirements/core_modules.txt | 14 +++++--------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 73278e959..486cd4165 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,7 +4,7 @@ networks: volumes: config: driver_opts: - type: none-config + type: config o: bind device: ./ services: diff --git a/docker/ngi_local_conf.yml b/docker/ngi_local_conf.yml index 1231b9923..e845a2c29 100755 --- a/docker/ngi_local_conf.yml +++ b/docker/ngi_local_conf.yml @@ -121,13 +121,14 @@ session: ttl: 180 tts: module: coqui - package_spec: neon-tts-plugin-coqui + package_spec: neon-tts-plugin-coqui~=0.1 mozilla: {request_url: http://0.0.0.0:5002/api/tts?} mozilla_remote: {api_url: https://mtts.2022.us/api/tts} mimic: {voice: ap} mimic2: {lang: en-us, url: https://mimic-api.mycroft.ai/synthesize?text=} stt: module: deepspeech_stream_local + package_spec: neon-stt-plugin-deepspeech_stream_local~=1.0 logs: blacklist: [enclosure.mouth.viseme, enclosure.mouth.display] device: diff --git a/requirements/core_modules.txt b/requirements/core_modules.txt index 7b0442282..3cf98184d 100644 --- a/requirements/core_modules.txt +++ b/requirements/core_modules.txt @@ -1,14 +1,10 @@ -# Patching compat with extra-stt -# ovos-core[all]~=0.0.2a5 -ovos-core[audio-backend,PHAL,tts,skills_lgpl,gui,bus]~=0.0.2a5 +ovos-core[audio-backend,PHAL,tts,skills_lgpl,gui,bus]~=0.0.3 SpeechRecognition~=3.8.1 PyAudio~=0.2.11 ovos-ww-plugin-pocketsphinx~=0.1.2 # neon core modules -neon_enclosure~=0.1,>=0.1.2 -neon_messagebus -neon_speech~=0.3,>=0.3.3a7 -neon_audio~=0.4,>=0.4.3a4 -neon_gui -# TODO: Version spec GUI \ No newline at end of file +neon_messagebus~=0.1 +neon_speech~=1.0 +neon_audio~=1.0 +neon_gui~=0.1 \ No newline at end of file From 4a5ba5200fd1c3c49cd1c2c0e631097f2b724f3a Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Tue, 10 May 2022 03:24:12 +0000 Subject: [PATCH 69/70] Increment Version --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 9f7b7116c..4e6d3716f 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a16" +__version__ = "21.10.3a17" From e6f68828cad09c74105f3dc1ffe5f64a5e4414f4 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Mon, 9 May 2022 22:37:45 -0700 Subject: [PATCH 70/70] Increment to 22.05.0 (#231) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 4e6d3716f..a5cb5db4b 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "21.10.3a17" +__version__ = "22.05.0"