many many many fixes

README.md

@@ -60,6 +60,7 @@ To run the service locally and play with the code just do the following:
 export GCP_PROJECT_NAME=<name of your gcp_project>
 export GITHUB_ACTION_TOKEN=ghp_<github_token>
 export GITHUB_REPOSITORY=<your_fork>/Trolley-Management
+export EMAIL_AUTHENTICATION=Yes
 export GMAIL_PASSWORD=<password>
 export GMAIL_USER=<user>@gmail.com
 export KUBECONFIG=/.kube/config
@@ -276,7 +277,8 @@ forked repository.
 Notice that without the correct credentials you will not be able to access the Build/Manage pages for these cloud
 providers!
 After the providers were properly configured a discovery service will be triggered and will start to scan the cloud
-services for the accounts that were provided.
+services for the accounts that were provided. 
+Please allow a few minutes after initial scan is completed and all the Regions and Zones will appear in the GUI
 
 ### Building AKS/GKE/EKS clusters
 

agents/aws_agent/main.py

@@ -15,17 +15,26 @@
     AWSObject
 from web.variables.variables import AWS
 
-file_name = 'server.log'
-log_file_path = f'{os.getcwd()}/{file_name}'
+DOCKER_ENV = os.getenv('DOCKER_ENV', False)
+
+log_file_name = 'server.log'
+if DOCKER_ENV:
+    log_file_path = f'{os.getcwd()}/web/{log_file_name}'
+else:
+    log_file_path = f'{os.getcwd()}/{log_file_name}'
+
+logger = logging.getLogger(__name__)
+
+file_handler = logging.FileHandler(filename=log_file_path)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+handlers = [file_handler, stdout_handler]
 
 logging.basicConfig(
     level=logging.INFO,
-    format="%(asctime)s [%(levelname)s] %(message)s",
-    handlers=[
-        logging.FileHandler(log_file_path),
-        logging.StreamHandler(sys.stdout)
-    ]
+    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+    handlers=handlers
 )
+
 FETCH_INTERVAL = int(os.environ.get('FETCH_INTERVAL', "30"))
 EC2_CLIENT = boto3.client('ec2')
 S3_CLIENT = boto3.client('s3')

docker-compose.yml

@@ -20,7 +20,10 @@ services:
       AWS_SHARED_CREDENTIALS_FILE: /home/app/.aws/credentials # Substitute with your AWS credentials files if you want it to get pulled locally
       AWS_DEFAULT_REGION: us-east-1
       GCP_CREDENTIALS_FILE: /home/app/.gcp/gcp_credentials # Substitute with your GCP credentials files if you want it to get pulled locally
-      TROLLEY_URL: http://192.168.1.204 # Substitute with the IP (VM, Metal etc..) on which your Trolley installation will run. For local run please use http://localhost
+      TROLLEY_URL: http://localhost # Substitute with the IP (VM, Metal etc..) on which your Trolley installation will run. For local run please use http://localhost
+      EMAIL_AUTHENTICATION: Yes # Should the authentication be done via Email
+      GMAIL_USER: user # Substitute with your Gmail user. It will be needed for registration purposes
+      GMAIL_PASSWORD: password # Substitute with the your Gmail password. It will be needed for the registration purposes
       DOCKER_ENV: Yes
     networks:
       - frontend

web/cluster_operations.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import platform
+import sys
 from dataclasses import asdict
 
 import requests
@@ -20,30 +21,32 @@
     log_file_path = f'{os.getcwd()}/{log_file_name}'
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
 
-handler = logging.FileHandler(log_file_name)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-handler.setFormatter(formatter)
-logger.addHandler(handler)
+file_handler = logging.FileHandler(filename=log_file_path)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+handlers = [file_handler, stdout_handler]
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+    handlers=handlers
+)
 
 project_folder = os.path.expanduser(os.getcwd())
 load_dotenv(os.path.join(project_folder, '.env'))
-logger.info(f'project_folder is: {project_folder}')
 
 # horrible hack to solve the Dockerfile issues. Please find a better solution
 run_env = 'not_github'
 try:
     github_env_something = os.getenv('GITHUB_ENV')
-    logger.info(github_env_something)
     if github_env_something is not None:
         run_env = 'github'
         logger.info('this runs on github')
     else:
         logger.info('this does not run on github')
 except:
     run_env = 'not github'
-    logger.error('this does not run on github')
+    logger.info('this does not run on github')
 
 if 'Darwin' in platform.system() or run_env == 'github':
     AWS_CREDENTIALS_PATH = f'/Users/{getpass.getuser()}/.aws/credentials'

web/cluster_operations__.py

@@ -26,7 +26,6 @@
 
 project_folder = os.path.expanduser(os.getcwd())
 load_dotenv(os.path.join(project_folder, '.env'))
-logger.info(f'project_folder is: {project_folder}')
 
 # horrible hack to solve the Dockerfile issues. Please find a better solution
 run_env = 'not_github'
@@ -40,7 +39,7 @@
         logger.info('this does not run on github')
 except:
     run_env = 'not github'
-    logger.error('this does not run on github')
+    logger.info('this does not run on github')
 
 if 'Darwin' in platform.system() or run_env == 'github':
     AWS_CREDENTIALS_PATH = f'/Users/{getpass.getuser()}/.aws/credentials'

web/deployment.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 
 from dotenv import load_dotenv
 
@@ -12,36 +13,33 @@
     log_file_path = f'{os.getcwd()}/{log_file_name}'
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
 
-handler = logging.FileHandler(log_file_path)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-handler.setFormatter(formatter)
-logger.addHandler(handler)
+file_handler = logging.FileHandler(filename=log_file_path)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+handlers = [file_handler, stdout_handler]
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+    handlers=handlers
+)
 
 project_folder = os.path.expanduser(os.getcwd())
 load_dotenv(os.path.join(project_folder, '.env'))
-logger.info(f'project_folder is: {project_folder}')
 
 TROLLEY_URL = os.getenv('TROLLEY_URL', "https://trolley.com")
 
 
 def main():
     with open('web/static/pre_script.js', 'r') as f:
         lines = f.readlines()
-        print(f'reading the pre_script.js {lines}')
-
     with open('web/static/script.js', 'w') as f:
-        print(f'Current directory is: {os.getcwd()}')
-        print('successfully opened the static.js file')
         logger.info('successfully opened the static.js file')
         for line in lines:
-            logger.info(f'line value is: {line}')
             if "trolley_url = " in line:
                 line = f"    let trolley_url = '{TROLLEY_URL}';\n"
             try:
                 f.write(line)
-                logger.info(f'Writing of the file succeeded')
             except Exception as e:
                 logger.error(f'Writing of the file failed because: {e}')
 

web/mail_handler.py

@@ -1,10 +1,32 @@
+import logging
 import os
 import smtplib
 import ssl
+import sys
 from email.message import EmailMessage
 
 GMAIL_USER = os.getenv('GMAIL_USER', "trolley_user")
 GMAIL_PASSWORD = os.getenv('GMAIL_PASSWORD', "trolley_password")
+DOCKER_ENV = os.getenv('DOCKER_ENV', False)
+
+log_file_name = 'server.log'
+if DOCKER_ENV:
+    log_file_path = f'{os.getcwd()}/web/{log_file_name}'
+else:
+    log_file_path = f'{os.getcwd()}/{log_file_name}'
+
+logger = logging.getLogger(__name__)
+
+file_handler = logging.FileHandler(filename=log_file_path)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+handlers = [file_handler, stdout_handler]
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+    handlers=handlers
+)
 
 
 class MailSender:
@@ -26,9 +48,13 @@ def send_confirmation_mail(self):
         em.set_content(body)
 
         context = ssl.create_default_context()
-        with smtplib.SMTP_SSL('smtp.gmail.com', 465, context=context) as smtp:
-            smtp.login(GMAIL_USER, GMAIL_PASSWORD)
-            smtp.sendmail(GMAIL_USER, self.user_email, em.as_string())
+        logger.info(f'Sending an email to: {self.user_email} with {GMAIL_USER} user and {GMAIL_PASSWORD}')
+        try:
+            with smtplib.SMTP_SSL('smtp.gmail.com', 465, context=context) as smtp:
+                smtp.login(GMAIL_USER, GMAIL_PASSWORD)
+                smtp.sendmail(GMAIL_USER, self.user_email, em.as_string())
+        except Exception as e:
+            logger.error(f'Error sending out an email with {e} error')
 
     def send_invitation_mail(self):
         subject = "You have been invited to Trolley!"

web/main.py

@@ -1,4 +1,5 @@
 import codecs
+import sys
 import time
 
 from dotenv import load_dotenv
@@ -11,7 +12,6 @@
 from threading import Thread
 
 from dataclasses import asdict
-from distutils import util
 
 from cryptography.fernet import Fernet
 from flask import request, Response, Flask, session, redirect, url_for, render_template, jsonify
@@ -32,17 +32,23 @@
     log_file_path = f'{os.getcwd()}/{log_file_name}'
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
 
-handler = logging.FileHandler(log_file_path)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-handler.setFormatter(formatter)
-logger.addHandler(handler)
+file_handler = logging.FileHandler(log_file_path)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+handlers = [file_handler, stdout_handler]
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+    handlers=handlers
+)
+
 
 project_folder = os.path.expanduser(os.getcwd())
 load_dotenv(os.path.join(project_folder, '.env'))
-logger.info(f'project_folder is: {project_folder}')
 
+EMAIL_AUTHENTICATION = os.getenv('MAIL_AUTHENTICATION', False)
 GMAIL_USER = os.getenv('GMAIL_USER', "trolley_user")
 GMAIL_PASSWORD = os.getenv('GMAIL_PASSWORD', "trolley_password")
 PROJECT_NAME = os.getenv('PROJECT_NAME', "trolley-dev")
@@ -53,7 +59,7 @@
 from mongo_handler.mongo_objects import UserObject, GithubObject, DeploymentYAMLObject, ProviderObject
 from variables.variables import POST, GET, EKS, \
     APPLICATION_JSON, CLUSTER_TYPE, GKE, AKS, DELETE, USER_NAME, REGIONS_LIST, \
-    ZONES_LIST, HELM_INSTALLS_LIST, GKE_VERSIONS_LIST, GKE_IMAGE_TYPES, HELM, LOCATIONS_DICT, \
+    ZONES_LIST, GKE_VERSIONS_LIST, GKE_IMAGE_TYPES, LOCATIONS_DICT, \
     CLUSTER_NAME, AWS, PROVIDER, GCP, AZ, PUT, OK, FAILURE, OBJECT_TYPE, CLUSTER, INSTANCE, TEAM_NAME, ZONE_NAMES, \
     NAMES, REGION_NAME, CLIENT_NAME, AVAILABILITY, GCP_PROJECT_ID
 
@@ -497,7 +503,6 @@ def insert_cluster_data():
 
 @app.route('/settings', methods=[GET, POST])
 @login_required
-@cache.cached(timeout=180)
 def settings():
     """
     This endpoint saves Trolley settings
@@ -694,7 +699,7 @@ def index():
 
 @app.route('/fetch_regions', methods=[GET])
 @login_required
-@cache.cached(timeout=180)
+# @cache.cached(timeout=180)
 def fetch_regions():
     cluster_type = request.args.get(CLUSTER_TYPE)
     logger.info(f'A request to fetch regions for {cluster_type} has arrived')
@@ -714,7 +719,7 @@ def fetch_regions():
 
 @app.route('/fetch_machine_series', methods=[GET])
 @login_required
-@cache.cached(timeout=180)
+# @cache.cached(timeout=180)
 def fetch_machine_series():
     cluster_type = request.args.get(CLUSTER_TYPE)
     region_name = request.args.get(REGION_NAME.lower())
@@ -726,7 +731,7 @@ def fetch_machine_series():
 
 @app.route('/fetch_machine_types', methods=[GET])
 @login_required
-@cache.cached(timeout=180)
+# @cache.cached(timeout=180)
 def fetch_machine_types():
     cluster_type = request.args.get(CLUSTER_TYPE)
     machine_series = request.args.get('machine_series')
@@ -868,20 +873,21 @@ def register():
         else:
             if not mongo_handler.mongo_utils.retrieve_user(user_email):
                 token = generate_confirmation_token(user_email)
+                logger.info(f'token is: {token}')
                 confirmation_url = str(url_for('confirmation_email', token=token, _external=True))
-                print(f'confirmation_url is: {confirmation_url}')
-
-                mail_message = MailSender(user_email, confirmation_url)
-                try:
-                    mail_message.send_confirmation_mail()
-                except Exception as e:
-                    logger.error(f'Failed to send an email due to {e} error to {user_email}')
+                logger.info(f'confirmation_url is: {confirmation_url}')
+                if EMAIL_AUTHENTICATION:
+                    mail_message = MailSender(user_email, confirmation_url)
+                    try:
+                        mail_message.send_confirmation_mail()
+                    except Exception as e:
+                        logger.error(f'Failed to send an email due to {e} error to {user_email}')
 
                 if user_registration(first_name, last_name, password, user_email, user_type, team_name, image_file_name,
                                      confirmation_url, registration_status='pending'):
-                    return render_template('confirmation.html', email=user_email)
+                    return render_template('confirmation.html', email=user_email, url=confirmation_url)
                 else:
-                    return render_template('confirmation.html', email=user_email)
+                    return render_template('confirmation.html', email=user_email, url=confirmation_url)
             else:
                 return render_template('register.html',
                                        error_message=f'Dear {first_name}, your email was already registered. '