From 40ca3ed9bc20b02cd49c4704279b386e7123e305 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 20 Feb 2024 01:13:20 +0100
Subject: [PATCH] new: easy way to find other captures with same favicon

---
 lookyloo/indexing.py                     |  3 ++
 website/web/__init__.py                  | 20 +++++++++
 website/web/templates/tree.html          | 28 +++++++++++++
 website/web/templates/tree_favicons.html | 53 ++++++++++++++++++++++++
 4 files changed, 104 insertions(+)
 create mode 100644 website/web/templates/tree_favicons.html

diff --git a/lookyloo/indexing.py b/lookyloo/indexing.py
index 59a02399..252915a4 100644
--- a/lookyloo/indexing.py
+++ b/lookyloo/indexing.py
@@ -339,6 +339,9 @@ def get_captures_hostname(self, hostname: str) -> set[str]:
     def favicons(self) -> list[tuple[str, float]]:
         return self.redis.zrevrange('favicons', 0, 200, withscores=True)
 
+    def favicon_frequency(self, favicon_sha512: str) -> float | None:
+        return self.redis.zscore('favicons', favicon_sha512)
+
     def favicon_number_captures(self, favicon_sha512: str) -> int:
         return self.redis.scard(f'favicons|{favicon_sha512}|captures')
 
diff --git a/website/web/__init__.py b/website/web/__init__.py
index 8dda2eca..8a68554e 100644
--- a/website/web/__init__.py
+++ b/website/web/__init__.py
@@ -5,6 +5,7 @@
 import base64
 import calendar
 import functools
+import hashlib
 import http
 import json
 import logging
@@ -833,6 +834,25 @@ def mark_as_legitimate(tree_uuid: str) -> Response:
     return jsonify({'message': 'Legitimate entry added.'})
 
 
+@app.route('/tree/<string:tree_uuid>/favicons', methods=['GET'])
+def tree_favicons(tree_uuid: str) -> str:
+    favicons = []
+    favicons_zip = lookyloo.get_potential_favicons(tree_uuid, all_favicons=True, for_datauri=False)
+    with ZipFile(favicons_zip, 'r') as myzip:
+        for name in myzip.namelist():
+            if not name.endswith('.ico'):
+                continue
+            favicon = myzip.read(name)
+            if not favicon:
+                continue
+            favicon_sha512 = hashlib.sha512(favicon).hexdigest()
+            frequency = lookyloo.indexing.favicon_frequency(favicon_sha512)
+            number_captures = lookyloo.indexing.favicon_number_captures(favicon_sha512)
+            b64_favicon = base64.b64encode(favicon).decode()
+            favicons.append((favicon_sha512, frequency, number_captures, b64_favicon))
+    return render_template('tree_favicons.html', tree_uuid=tree_uuid, favicons=favicons)
+
+
 @app.route('/tree/<string:tree_uuid>/body_hashes', methods=['GET'])
 def tree_body_hashes(tree_uuid: str) -> str:
     body_hashes = lookyloo.get_all_body_hashes(tree_uuid)
diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html
index 83a40068..4cfe590f 100644
--- a/website/web/templates/tree.html
+++ b/website/web/templates/tree.html
@@ -84,6 +84,13 @@
   });
   </script>
   <script>
+  $('#faviconsModal').on('show.bs.modal', function(e) {
+      var button = $(e.relatedTarget);
+      var modal = $(this);
+      modal.find('.modal-body').load(button.data("remote"));
+  });
+  </script>
+  <script>
   $('#bodyHashesModal').on('show.bs.modal', function(e) {
       var button = $(e.relatedTarget);
       var modal = $(this);
@@ -287,6 +294,10 @@
               <a href="#bodyHashesModal" data-remote="{{ url_for('tree_body_hashes', tree_uuid=tree_uuid) }}"
                  data-bs-toggle="modal" data-bs-target="#bodyHashesModal" role="button">Ressources Capture</a>
             </li>
+            <li class="list-group-item">
+              <a href="#faviconsModal" data-remote="{{ url_for('tree_favicons', tree_uuid=tree_uuid) }}"
+                 data-bs-toggle="modal" data-bs-target="#faviconsModal" role="button">Favicons Capture</a>
+            </li>
           </ul>
         </div>
         {% if current_user.is_authenticated %}
@@ -509,6 +520,23 @@ <h5 class="modal-title" id="statsModalLabel">Statistics</h5>
   </div>
 </div>
 
+<div class="modal fade" id="faviconsModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="faviconsModalLabel">Favicons in tree</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+              ... loading favicons ...
+          </div>
+          <div class="modal-footer">
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+  </div>
+</div>
+
 <div class="modal fade" id="bodyHashesModal" tabindex="-1" role="dialog">
   <div class="modal-dialog modal-xl" role="document">
       <div class="modal-content">
diff --git a/website/web/templates/tree_favicons.html b/website/web/templates/tree_favicons.html
new file mode 100644
index 00000000..b0342e56
--- /dev/null
+++ b/website/web/templates/tree_favicons.html
@@ -0,0 +1,53 @@
+<script type="text/javascript">
+    $('#faviconsTable')
+        .on('order.dt', () =>  {
+            $(function () {
+              $('[data-bs-toggle="tooltip"]').tooltip()
+            })
+        })
+        .on('search.dt', () => {
+            $(function () {
+              $('[data-bs-toggle="tooltip"]').tooltip()
+            })
+        })
+        .on('page.dt', () => {
+            $(function () {
+              $('[data-bs-toggle="tooltip"]').tooltip()
+            })
+        })
+        .DataTable( {
+          "order": [[ 1, "desc" ]],
+          "pageLength": 50
+        });
+</script>
+
+{% if from_popup %}
+<script type="text/javascript">
+    function openTreeInNewTab(treeUUID) {
+        window.opener.openTreeInNewTab(treeUUID);
+    };
+</script>
+{% endif %}
+
+<table id="faviconsTable" class="table w-auto">
+<thead>
+ <tr>
+   <th>Favicon</th>
+   <th>Frequency</th>
+   <th>Number of captures</th>
+ </tr>
+</thead>
+<tbody>
+  {% for favicon_sha512, freq, number_captures, b64_favicon in favicons %}
+  <tr>
+    <td>
+      <a href="{{ url_for('favicon_detail', favicon_sha512=favicon_sha512) }}">
+        <img src="data:image/ico;base64,{{ b64_favicon }}" style="width:32px;height:32px;"/>
+      </a>
+    </td>
+    <td>{{ freq }}</td>
+    <td>{{ number_captures }}</td>
+  </tr>
+  {% endfor %}
+</tbody>
+</table>