chg: [favicon] add favicons objects + correlation

bin/lib/objects/Favicons.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
+import base64
 
 import mmh3
 import os
 import sys
 
-from flask import url_for
+from io import BytesIO
 
+from flask import url_for
 from pymisp import MISPObject
 
 sys.path.append(os.environ['AIL_BIN'])
@@ -18,6 +20,7 @@
 
 config_loader = ConfigLoader()
 r_objects = config_loader.get_db_conn("Kvrocks_Objects")
+FAVICON_FOLDER = config_loader.get_files_directory('favicons')
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None
 
@@ -40,10 +43,6 @@ def delete(self):
         # # TODO:
         pass
 
-    def get_content(self, r_type='str'):
-        if r_type == 'str':
-            return self._get_field('content')
-
     def get_link(self, flask_context=False):
         if flask_context:
             url = url_for('correlation.show_correlation', type=self.type, id=self.id)
@@ -53,7 +52,24 @@ def get_link(self, flask_context=False):
 
     # TODO # CHANGE COLOR
     def get_svg_icon(self):
-        return {'style': 'fas', 'icon': '\uf20a', 'color': '#1E88E5', 'radius': 5}  # f0c8 f45c
+        return {'style': 'fas', 'icon': '\uf089', 'color': '#E1F5D0', 'radius': 5}  # f0c8 f45c f089
+
+    def get_rel_path(self): # TODO USE MUMUR HASH
+        rel_path = os.path.join(self.id[0:1], self.id[1:2], self.id[2:3], self.id[3:4], self.id[4:5], self.id[5:6], self.id[6:])
+        return rel_path
+
+    def get_filepath(self):
+        filename = os.path.join(FAVICON_FOLDER, self.get_rel_path())
+        return os.path.realpath(filename)
+
+    def get_file_content(self):
+        filepath = self.get_filepath()
+        with open(filepath, 'rb') as f:
+            file_content = BytesIO(f.read())
+        return file_content
+
+    def get_content(self, r_type='str'):
+        return self.get_file_content()
 
     def get_misp_object(self):
         obj_attrs = []
@@ -69,7 +85,7 @@ def get_misp_object(self):
                 f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('favicon-mmh3', value=self.id))
-        obj_attrs.append(obj.add_attribute('favicon', value=self.get_content(r_type='bytes')))
+        obj_attrs.append(obj.add_attribute('favicon', value=self.get_content()))
         for obj_attr in obj_attrs:
             for tag in self.get_tags():
                 obj_attr.add_tag(tag)
@@ -78,29 +94,32 @@ def get_misp_object(self):
     def get_meta(self, options=set()):
         meta = self._get_meta(options=options)
         meta['id'] = self.id
+        meta['img'] = self.id
         meta['tags'] = self.get_tags(r_list=True)
         if 'content' in options:
             meta['content'] = self.get_content()
+        if 'tags_safe' in options:
+            meta['tags_safe'] = self.is_tags_safe(meta['tags'])
         return meta
 
-    # def get_links(self):
-    #     # TODO GET ALL URLS FROM CORRELATED ITEMS
-
-    def create(self, content, _first_seen=None, _last_seen=None):
-        if not isinstance(content, str):
-            content = content.decode()
-        self._set_field('content', content)
+    def create(self, content):  # TODO first seen / last seen options
+        filepath = self.get_filepath()
+        dirname = os.path.dirname(filepath)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filepath, 'wb') as f:
+            f.write(content)
         self._create()
 
-
-def create_favicon(content, url=None):  # TODO URL ????
-    if isinstance(content, str):
-        content = content.encode()
-    favicon_id = mmh3.hash_bytes(content)
+def create(b_content, size_limit=5000000, b64=False, force=False):
+    if isinstance(b_content, str):
+        b_content = b_content.encode()
+    b64 = base64.encodebytes(b_content)  # newlines inserted after every 76 bytes of output
+    favicon_id = str(mmh3.hash(b64))
     favicon = Favicon(favicon_id)
     if not favicon.exists():
-        favicon.create(content)
-
+        favicon.create(b_content)
+    return favicon
 
 class Favicons(AbstractDaterangeObjects):
     """

configs/core.cfg.sample

@@ -7,6 +7,7 @@ crawled = crawled
 har = CRAWLED_SCREENSHOT
 screenshot = CRAWLED_SCREENSHOT/screenshot
 images = IMAGES
+favicons = FAVICONS
 
 wordtrending_csv = var/www/static/csv/wordstrendingdata
 wordsfile = files/wordfile

var/www/Flask_server.py

@@ -52,6 +52,7 @@
 from blueprints.objects_hhhash import objects_hhhash
 from blueprints.chats_explorer import chats_explorer
 from blueprints.objects_image import objects_image
+from blueprints.objects_favicon import objects_favicon
 
 Flask_dir = os.environ['AIL_FLASK']
 
@@ -111,6 +112,7 @@
 app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
 app.register_blueprint(chats_explorer, url_prefix=baseUrl)
 app.register_blueprint(objects_image, url_prefix=baseUrl)
+app.register_blueprint(objects_favicon, url_prefix=baseUrl)
 
 # =========       =========#
 

var/www/blueprints/correlation.py

@@ -93,6 +93,9 @@ def show_correlation():
         correl_option = request.form.get('EtagCheck')
         if correl_option:
             filter_types.append('etag')
+        correl_option = request.form.get('FaviconCheck')
+        if correl_option:
+            filter_types.append('favicon')
         correl_option = request.form.get('CveCheck')
         if correl_option:
             filter_types.append('cve')

var/www/blueprints/objects_favicon.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects import Favicons
+from packages import Date
+
+# ============ BLUEPRINT ============
+objects_favicon = Blueprint('objects_favicon', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/favicon'))
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+
+# ============ FUNCTIONS ============
+@objects_favicon.route('/favicon/<path:filename>')
+@login_required
+@login_read_only
+@no_cache
+def favicon(filename):
+    if not filename:
+        abort(404)
+    if not 9 <= len(filename) <= 11:
+        abort(404)
+    filename = filename.replace('/', '')
+    fav = Favicons.Favicon(filename)
+    return send_from_directory(Favicons.FAVICON_FOLDER, fav.get_rel_path(), as_attachment=False, mimetype='image')
+
+
+@objects_favicon.route("/objects/favicons", methods=['GET'])
+@login_required
+@login_read_only
+def objects_favicons():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    show_objects = request.args.get('show_objects')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+
+    if show_objects:
+        dict_objects = Favicons.Favicons().api_get_meta_by_daterange(date_from, date_to)
+    else:
+        dict_objects = {}
+
+    print(dict_objects)
+
+    return render_template("FaviconDaterange.html", date_from=date_from, date_to=date_to,
+                           dict_objects=dict_objects, show_objects=show_objects)
+
+
+@objects_favicon.route("/objects/favicons/post", methods=['POST'])
+@login_required
+@login_read_only
+def objects_favicons_post():
+    date_from = request.form.get('date_from')
+    date_to = request.form.get('date_to')
+    show_objects = request.form.get('show_objects')
+    return redirect(url_for('objects_favicon.objects_favicons', date_from=date_from, date_to=date_to, show_objects=show_objects))
+
+
+@objects_favicon.route("/objects/favicons/range/json", methods=['GET'])
+@login_required
+@login_read_only
+def objects_favicons_range_json():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+    return jsonify(Favicons.Favicons().api_get_chart_nb_by_daterange(date_from, date_to))
+
+# ============= ROUTES ==============
+