Improve ASI detector support and reduce excessive logging

THANKS.md

@@ -1,7 +1,7 @@
 Code contributors
 -----------------
 
-See [Github contributors list](https://github.com/nipy/nipype/graphs/contributors)
+See [Github contributors list](https://github.com/instamatic-dev/instamatic/graphs/contributors)
 
 Special thanks
 --------------

src/instamatic/camera/camera_serval.py

@@ -2,13 +2,11 @@
 
 import atexit
 import logging
-from pathlib import Path
 from typing import Tuple
 
 import numpy as np
 from serval_toolkit.camera import Camera as ServalCamera
 
-from instamatic import config
 from instamatic.camera.camera_base import CameraBase
 
 logger = logging.getLogger(__name__)
@@ -27,14 +25,22 @@ class CameraServal(CameraBase):
     def __init__(self, name='serval'):
         """Initialize camera module."""
         super().__init__(name)
-
         self.establish_connection()
-
-        msg = f'Camera {self.get_name()} initialized'
-        logger.info(msg)
-
+        self.exposure_cooldown = (
+            self.detector_config['TriggerPeriod'] - self.detector_config['ExposureTime']
+        )
+        logger.info(f'Camera {self.get_name()} initialized')
         atexit.register(self.release_connection)
 
+    def _validate_exposure(self, exposure: float) -> float:
+        if exposure < 0.001:
+            logger.warning(f'Exposure {exposure} too low. Adjusting to 0.001s')
+            exposure = 0.001
+        elif exposure > 10:
+            logger.warning(f'Exposure {exposure} too high. Adjusting to 10s')
+            exposure = 10
+        return exposure
+
     def get_image(self, exposure=None, binsize=None, **kwargs) -> np.ndarray:
         """Image acquisition routine. If the exposure and binsize are not
         given, the default values are read from the config file.
@@ -46,12 +52,12 @@ def get_image(self, exposure=None, binsize=None, **kwargs) -> np.ndarray:
         """
         if exposure is None:
             exposure = self.default_exposure
-        if not binsize:
-            binsize = self.default_binsize
+        exposure = self._validate_exposure(exposure)
 
         # Upload exposure settings (Note: will do nothing if no change in settings)
         self.conn.set_detector_config(
-            ExposureTime=exposure, TriggerPeriod=exposure + 0.00050001
+            ExposureTime=exposure,
+            TriggerPeriod=exposure + self.exposure_cooldown,
         )
 
         # Check if measurement is running. If not: start
@@ -82,6 +88,7 @@ def get_movie(self, n_frames, exposure=None, binsize=None, **kwargs):
             exposure = self.default_exposure
         if not binsize:
             binsize = self.default_binsize
+        exposure = self._validate_exposure(exposure)
 
         self.conn.set_detector_config(TriggerMode='CONTINUOUS')
 
@@ -111,21 +118,14 @@ def establish_connection(self) -> None:
             bpc_file_path=self.bpc_file_path, dacs_file_path=self.dacs_file_path
         )
         self.conn.set_detector_config(**self.detector_config)
-        # Check pixel depth. If 24 bit mode is used, the pgm format does not work
-        # (has support up to 16 bits) so use tiff in that case. In other cases (1, 6, 12 bits)
-        # use pgm since it is more efficient
-        self.pixel_depth = self.conn.detector_config['PixelDepth']
-        if self.pixel_depth == 24:
-            file_format = 'tiff'
-        else:
-            file_format = 'pgm'
+
         self.conn.destination = {
             'Image': [
                 {
                     # Where to place the preview files (HTTP end-point: GET localhost:8080/measurement/image)
                     'Base': 'http://localhost',
                     # What (image) format to provide the files in.
-                    'Format': file_format,
+                    'Format': 'tiff',
                     # What data to build a frame from
                     'Mode': 'count',
                 }

src/instamatic/camera/videostream.py

@@ -126,10 +126,7 @@ def setup_grabber(self) -> ImageGrabber:
         return grabber
 
     def get_image(self, exposure=None, binsize=None):
-        current_frametime = self.grabber.frametime
-
-        # set to 0 to prevent it lagging data acquisition
-        self.grabber.frametime = 0
+        self.block()  # Stop the passive collection during single-frame acquisition
         if exposure:
             self.grabber.exposure = exposure
         if binsize:
@@ -144,7 +141,7 @@ def get_image(self, exposure=None, binsize=None):
         self.grabber.lock.release()
 
         self.grabber.acquireCompleteEvent.clear()
-        self.grabber.frametime = current_frametime
+        self.unblock()  # Resume the passive collection
         return frame
 
     def update_frametime(self, frametime):

src/instamatic/config/camera/serval.yaml

@@ -1,21 +1,36 @@
 camera_rotation_vs_stage_xy: 0.0
 default_binsize: 1
 default_exposure: 0.02
+
+# Double-check that your dimensions match the ones from your detector.
 dimensions: [512, 512]
+
 dynamic_range: 11800
 interface: serval
+
 physical_pixelsize: 0.055
 possible_binsizes: [1]
+
 stretch_amplitude: 0.0
 stretch_azimuth: 0.0
+
+# This configuration can be tuned based on your needs.
+# Please refer to the Serval manual.
+# For Serval 3.3.x, it is in section 4.5.7 - Detector Config JSON structure.
 detector_config:
   BiasVoltage: 100
   BiasEnabled: True
   TriggerMode: SOFTWARESTART_TIMERSTOP
   ExposureTime: 1.0
-  TriggerPeriod: 1.001
+  TriggerPeriod: 1.002  # Try 1.0005 for Medipix3
   nTriggers: 1000000000
-  GainMode: HGM
+  # Disregard the following lines if you are using CheeTah T3 variants (timepix3),
+  # as they do not support these settings.
+  GainMode: HGM # Only for Medipix3
+  PixelDepth: 24 # Set to 12 for 12-bit (normal) mode - only Medipix3
+  BothCounters: False # Set to True for 12-bit mode - only Medipix3
+
+# Change this to the location of your actual factory settings
 bpc_file_path: '/home/asi/Desktop/Factory_settings/SPM-HGM/config.bpc'
 dacs_file_path: '/home/asi/Desktop/Factory_settings/SPM-HGM/config.dacs'
 url: 'http://localhost:8080'

src/instamatic/experiments/cred/experiment.py

@@ -83,7 +83,8 @@ def __init__(
         self.unblank_beam = unblank_beam
         self.logger = log
         self.mode = mode
-        if ctrl.cam.name == 'simulate':
+        # we need to be able to simulate using the serval camera as well - add "simulate" to part of the name.
+        if 'simulate' in ctrl.cam.name:
             self.mode = 'simulate'
         self.stopEvent = stop_event
         self.flatfield = flatfield

src/instamatic/gui/about_frame.py

@@ -4,6 +4,7 @@
 from tkinter import Label as tkLabel
 from tkinter.font import Font, nametofont
 from tkinter.ttk import *
+from typing import Callable, Optional
 
 import instamatic
 
@@ -29,6 +30,16 @@ def get_background_of_widget(widget):
     return background
 
 
+class Copyable(Text):
+    def __init__(self, master=None, text: str = '', **kwargs) -> None:
+        kwargs['bg'] = kwargs.get('bg', get_background_of_widget(master))
+        kwargs['relief'] = kwargs.get('relief', 'flat')
+        kwargs['width'] = kwargs.get('width', len(text))
+        super().__init__(master, **kwargs)
+        self.insert(1.0, text)
+        self.configure(state='disabled', font=nametofont('TkDefaultFont'))
+
+
 class Link_Button(tkLabel):
     """http://code.activestate.com/recipes/580774-tkinter-link-or-hyperlink-
     button/"""
@@ -110,45 +121,43 @@ def __init__(self, parent):
 
         Label(frame, text='').grid(row=0, column=0, sticky='W')
         Label(frame, text='Contact:').grid(row=1, column=0, sticky='W', padx=10)
-        Label(frame, text=f'{instamatic.__author__} ({instamatic.__author_email__}').grid(
-            row=1, column=1, sticky='W'
-        )
+        auth = f'{instamatic.__author__}, {instamatic.__author_email__}'
+        Copyable(frame, text=auth, height=1).grid(row=1, column=1, sticky='W')
         Label(frame, text='').grid(row=5, column=0, sticky='W')
 
         Label(frame, text='Source code:').grid(row=10, column=0, sticky='W', padx=10)
-        link = Link_Button(frame, text=instamatic.__url__, action=self.link_github)
+        action = self.link_action_factory(url=instamatic.__url__)
+        link = Link_Button(frame, text=instamatic.__url__, action=action)
         link.grid(row=10, column=1, sticky='W')
         Label(frame, text='').grid(row=12, column=0, sticky='W')
 
         Label(frame, text='Docs:').grid(row=20, column=0, sticky='W', padx=10)
-        link = Link_Button(frame, text=instamatic.__docs__, action=self.link_github)
+        action = self.link_action_factory(url=instamatic.__docs__)
+        link = Link_Button(frame, text=instamatic.__docs__, action=action)
         link.grid(row=20, column=1, sticky='W')
         Label(frame, text='').grid(row=22, column=0, sticky='W')
 
         Label(frame, text='Bugs:').grid(row=30, column=0, sticky='W', padx=10)
-        link = Link_Button(frame, text=instamatic.__issues__, action=self.link_github)
+        action = self.link_action_factory(url=instamatic.__issues__)
+        link = Link_Button(frame, text=instamatic.__issues__, action=action)
         link.grid(row=30, column=1, sticky='W')
         Label(frame, text='').grid(row=32, column=0, sticky='W')
 
         Label(frame, text='If you found this software useful, please cite:').grid(
             row=40, column=0, sticky='W', columnspan=2, padx=10
         )
-        txt = Message(frame, text=instamatic.__citation__, width=320, justify=LEFT)
-        txt.grid(row=41, column=1, sticky='W')
-
-        Label(frame, text='').grid(row=41, column=0, sticky='W', padx=10)
+        txt = Copyable(frame, text=instamatic.__citation__, height=1)
+        txt.grid(row=41, column=0, columnspan=2, sticky='W', padx=10)
 
         frame.pack(side='top', fill='x')
 
-    def link_github(self, event=None):
-        import webbrowser
-
-        webbrowser.open_new(instamatic.__url__)
+    def link_action_factory(self, url: str) -> Callable:
+        def link_action(event=None) -> None:
+            import webbrowser
 
-    def link_manual(self, event=None):
-        import webbrowser
+            webbrowser.open_new(url)
 
-        webbrowser.open_new(instamatic.__url__)
+        return link_action
 
 
 module = BaseModule(name='about', display_name='about', tk_frame=AboutFrame, location='bottom')
@@ -157,5 +166,5 @@ def link_manual(self, event=None):
 
 if __name__ == '__main__':
     root = Tk()
-    About(root).pack(side='top', fill='both', expand=True)
+    AboutFrame(root).pack(side='top', fill='both', expand=True)
     root.mainloop()

src/instamatic/main.py

@@ -117,6 +117,14 @@ def main():
         help='Show info about the current instamatic installation.',
     )
 
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='store_true',
+        dest='verbose',
+        help="Show imported packages' DEBUG records in log",
+    )
+
     parser.set_defaults(
         script=None,
         acquire_at_items=False,
@@ -155,11 +163,21 @@ def main():
     date = datetime.datetime.now().strftime('%Y-%m-%d')
     logfile = config.locations['logs'] / f'instamatic_{date}.log'
 
-    logging.basicConfig(
-        format='%(asctime)s | %(module)s:%(lineno)s | %(levelname)s | %(message)s',
-        filename=logfile,
-        level=logging.DEBUG,
-    )
+    def filter_out_imported_package_debug_records(r: logging.LogRecord) -> bool:
+        return (
+            r.name.startswith('instamatic')
+            or r.name == '__main__'
+            or r.levelno > logging.DEBUG
+            or options.verbose
+        )
+
+    log_main = logging.getLogger()
+    log_main.setLevel(logging.DEBUG)
+    log_format = '%(asctime)s | %(module)s:%(lineno)s | %(levelname)s | %(message)s'
+    log_handler = logging.FileHandler(logfile)
+    log_handler.setFormatter(logging.Formatter(log_format))
+    log_handler.addFilter(filter_out_imported_package_debug_records)
+    log_main.addHandler(log_handler)
 
     logging.captureWarnings(True)
     log = logging.getLogger(__name__)

src/instamatic/server/tem_server.py

@@ -1,18 +1,15 @@
 from __future__ import annotations
 
 import datetime
-import json
 import logging
-import pickle
 import queue
 import socket
 import threading
 import traceback
 
 from instamatic import config
 from instamatic.microscope import get_microscope
-
-from .serializer import dumper, loader
+from instamatic.server.serializer import dumper, loader
 
 condition = threading.Condition()
 box = []