Port ndi.fun from Matlab to Python

comparison_table.md

@@ -353,59 +353,59 @@
 | ndi.file.pfilemirror | No | No |
 | ndi.file.temp_fid | No | No |
 | ndi.file.temp_name | No | No |
-| ndi.fun.calc.stimulus_tuningcurve_log | No | No |
-| ndi.fun.data.mat2ngrid | No | No |
-| ndi.fun.data.readImageStack | No | No |
-| ndi.fun.data.readngrid | No | No |
-| ndi.fun.data.writengrid | No | No |
-| ndi.fun.dataset.diff | No | No |
-| ndi.fun.doc.probe.probeLocations4probes | No | No |
-| ndi.fun.doc.subject.makeSpeciesStrainSex | No | No |
-| ndi.fun.doc.allTypes | No | No |
-| ndi.fun.doc.diff | No | No |
-| ndi.fun.doc.findFuid | No | No |
-| ndi.fun.doc.getDocTypes | No | No |
-| ndi.fun.doc.ontologyTableRowDoc2Table | No | No |
-| ndi.fun.doc.ontologyTableRowVars | No | No |
-| ndi.fun.doc.t0_t1cell2array | No | No |
-| ndi.fun.docTable.docCellArray2Table | No | No |
-| ndi.fun.docTable.element | No | No |
-| ndi.fun.docTable.epoch | No | No |
-| ndi.fun.docTable.openminds | No | No |
-| ndi.fun.docTable.probe | No | No |
-| ndi.fun.docTable.subject | No | No |
-| ndi.fun.docTable.treatment | No | No |
-| ndi.fun.epoch.epochid2element | No | No |
-| ndi.fun.epoch.filename2epochid | No | No |
-| ndi.fun.file.MD5 | No | No |
-| ndi.fun.file.dateCreated | No | No |
-| ndi.fun.file.dateUpdated | No | No |
-| ndi.fun.plot.bar3 | No | No |
-| ndi.fun.plot.multichan | No | No |
-| ndi.fun.probe.location | No | No |
-| ndi.fun.session.diff | No | No |
-| ndi.fun.stimulus.f0_f1_responses | No | No |
-| ndi.fun.stimulus.findMixtureName | No | No |
-| ndi.fun.stimulus.tuning_curve_to_response_type | No | No |
-| ndi.fun.table.identifyMatchingRows | No | No |
-| ndi.fun.table.identifyValidRows | No | No |
-| ndi.fun.table.join | No | No |
-| ndi.fun.table.moveColumnsLeft | No | No |
-| ndi.fun.table.vstack | No | No |
+| ndi.fun.calc.stimulus_tuningcurve_log | Yes | Yes |
+| ndi.fun.data.mat2ngrid | Yes | Yes |
+| ndi.fun.data.readImageStack | Yes | Yes |
+| ndi.fun.data.readngrid | Yes | Yes |
+| ndi.fun.data.writengrid | Yes | Yes |
+| ndi.fun.dataset.diff | Yes | Yes |
+| ndi.fun.doc.probe.probeLocations4probes | Yes | Yes |
+| ndi.fun.doc.subject.makeSpeciesStrainSex | Yes | Yes |
+| ndi.fun.doc.allTypes | Yes | Yes |
+| ndi.fun.doc.diff | Yes | Yes |
+| ndi.fun.doc.findFuid | Yes | Yes |
+| ndi.fun.doc.getDocTypes | Yes | Yes |
+| ndi.fun.doc.ontologyTableRowDoc2Table | Yes | Yes |
+| ndi.fun.doc.ontologyTableRowVars | Yes | Yes |
+| ndi.fun.doc.t0_t1cell2array | Yes | Yes |
+| ndi.fun.docTable.docCellArray2Table | Yes | Yes |
+| ndi.fun.docTable.element | Yes | Yes |
+| ndi.fun.docTable.epoch | Yes | Yes |
+| ndi.fun.docTable.openminds | Yes | Yes |
+| ndi.fun.docTable.probe | Yes | Yes |
+| ndi.fun.docTable.subject | Yes | Yes |
+| ndi.fun.docTable.treatment | Yes | Yes |
+| ndi.fun.epoch.epochid2element | Yes | Yes |
+| ndi.fun.epoch.filename2epochid | Yes | Yes |
+| ndi.fun.file.MD5 | Yes | Yes |
+| ndi.fun.file.dateCreated | Yes | Yes |
+| ndi.fun.file.dateUpdated | Yes | Yes |
+| ndi.fun.plot.bar3 | Yes | Yes |
+| ndi.fun.plot.multichan | Yes | Yes |
+| ndi.fun.probe.location | Yes | Yes |
+| ndi.fun.session.diff | Yes | Yes |
+| ndi.fun.stimulus.f0_f1_responses | Yes | Yes |
+| ndi.fun.stimulus.findMixtureName | Yes | Yes |
+| ndi.fun.stimulus.tuning_curve_to_response_type | Yes | Yes |
+| ndi.fun.table.identifyMatchingRows | Yes | Yes |
+| ndi.fun.table.identifyValidRows | Yes | Yes |
+| ndi.fun.table.join | Yes | Yes |
+| ndi.fun.table.moveColumnsLeft | Yes | Yes |
+| ndi.fun.table.vstack | Yes | Yes |
 | ndi.fun.assertAddonOnPath | No | No |
-| ndi.fun.channelname2prefixnumber | No | No |
+| ndi.fun.channelname2prefixnumber | Yes | Yes |
 | ndi.fun.check_Matlab_toolboxes | No | No |
-| ndi.fun.console | No | No |
+| ndi.fun.console | Yes | Yes |
 | ndi.fun.convertoldnsd2ndi | No | No |
-| ndi.fun.debuglog | No | No |
-| ndi.fun.errlog | No | No |
-| ndi.fun.find_calc_directories | No | No |
-| ndi.fun.name2variableName | No | No |
+| ndi.fun.debuglog | Yes | Yes |
+| ndi.fun.errlog | Yes | Yes |
+| ndi.fun.find_calc_directories | Yes | Yes |
+| ndi.fun.name2variableName | Yes | Yes |
 | ndi.fun.plot_extracellular_spikeshapes | No | No |
-| ndi.fun.pseudorandomint | No | No |
+| ndi.fun.pseudorandomint | Yes | Yes |
 | ndi.fun.run_Linux_checks | No | No |
-| ndi.fun.stimulustemporalfrequency | No | No |
-| ndi.fun.syslog | No | No |
+| ndi.fun.stimulustemporalfrequency | Yes | Yes |
+| ndi.fun.syslog | Yes | Yes |
 | ndi.fun.timestamp | Yes | Yes |
 | ndi.gui.component.abstract.ProgressMonitor | No | No |
 | ndi.gui.component.internal.event.MessageUpdatedEventData | No | No |

src/ndi/common/__init__.py

@@ -0,0 +1 @@
+from .path_constants import PathConstants

src/ndi/common/path_constants.py

@@ -0,0 +1,18 @@
+import os
+
+class PathConstants:
+    @staticmethod
+    def root_folder():
+        """
+        Returns the root folder of the NDI distribution.
+        """
+        # Assuming this file is in src/ndi/common/path_constants.py
+        # Root is src/ndi/
+        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+    @staticmethod
+    def common_folder():
+        """
+        Returns the path to the package ndi_common resources.
+        """
+        return os.path.join(PathConstants.root_folder(), 'resources', 'ndi_common')

src/ndi/document.py

@@ -1,6 +1,6 @@
 from did.document import Document
 from .ido import Ido
-import ndi.fun.timestamp
+import ndi.fun
 from .util.vlt import data as vlt_data
 import json
 import os
@@ -15,7 +15,7 @@ def __init__(self, document_type, **kwargs):
             self.document_properties = self.read_blank_definition(document_type)
             ido = Ido()
             self.document_properties['base']['id'] = ido.id()
-            self.document_properties['base']['datestamp'] = ndi.fun.timestamp.timestamp()
+            self.document_properties['base']['datestamp'] = ndi.fun.timestamp()
 
             for key, value in kwargs.items():
                 keys = key.split('.')

src/ndi/fun/__init__.py

@@ -0,0 +1,10 @@
+from .channel_name_to_prefix_number import channel_name_to_prefix_number
+from .name_to_variable_name import name_to_variable_name
+from .timestamp import timestamp
+from .pseudorandomint import pseudorandomint
+from .stimulus_temporal_frequency import stimulus_temporal_frequency
+from .find_calc_directories import find_calc_directories
+from .console import console
+from .debug_log import debug_log
+from .err_log import err_log
+from .sys_log import sys_log

src/ndi/fun/calc/__init__.py

@@ -0,0 +1 @@
+from .stimulus_tuning_curve_log import stimulus_tuning_curve_log

src/ndi/fun/calc/stimulus_tuning_curve_log.py

@@ -0,0 +1,31 @@
+from did.query import Query
+
+def stimulus_tuning_curve_log(S, doc):
+    """
+    Retrieve stimulus_tuningcurve log string from dependent document.
+
+    Args:
+        S: ndi.session object.
+        doc: ndi.document object.
+
+    Returns:
+        str: The log string.
+    """
+    log_str = ''
+
+    try:
+        stim_tune_doc_id = doc.dependency_value('stimulus_tuningcurve_id')
+    except Exception:
+        # If dependency not found
+        return log_str
+
+    q1 = Query('base.id', 'exact_string', stim_tune_doc_id)
+    q2 = Query('', 'isa', 'tuningcurve_calc')
+
+    stim_tune_doc = S.database_search(q1 & q2)
+
+    if stim_tune_doc:
+        props = stim_tune_doc[0].document_properties.get('tuningcurve_calc', {})
+        log_str = props.get('log', '')
+
+    return log_str

src/ndi/fun/console.py

@@ -0,0 +1,58 @@
+import sys
+import platform
+import os
+import subprocess
+import tempfile
+
+def console(filename):
+    """
+    Pops up an external terminal window that displays a log file.
+
+    CONSOLE(FILENAME)
+
+    Pops up a console window that displays a log file.
+    Right now, only MacOS is supported.
+    """
+
+    system = platform.system()
+
+    if system == 'Darwin': # MacOS
+        # Create temporary applescript file
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.scpt', delete=False) as f:
+            script_content = f"""tell application "Terminal"
+    activate
+    do script "tail -f {filename}"
+end tell"""
+            f.write(script_content)
+            temp_filename = f.name
+
+        try:
+            subprocess.run(['osascript', temp_filename], check=True)
+        finally:
+            os.remove(temp_filename)
+
+    elif system == 'Linux':
+        # Try some common terminal emulators
+        terminals = ['gnome-terminal', 'xterm', 'konsole']
+        launched = False
+        for term in terminals:
+            try:
+                # This is a very basic attempt and might need adjustment for specific terminals
+                if term == 'gnome-terminal':
+                    subprocess.run([term, '--', 'tail', '-f', filename])
+                elif term == 'xterm':
+                    subprocess.run([term, '-e', f'tail -f {filename}'])
+                elif term == 'konsole':
+                    subprocess.run([term, '-e', 'tail', '-f', filename])
+                launched = True
+                break
+            except FileNotFoundError:
+                continue
+
+        if not launched:
+             raise NotImplementedError("Linux terminal launch not fully supported yet.")
+
+    elif system == 'Windows':
+         raise NotImplementedError("Windows not supported yet.")
+    else:
+         raise NotImplementedError(f"{system} not supported yet.")

src/ndi/fun/data/__init__.py

@@ -0,0 +1,4 @@
+from .read_ngrid import read_ngrid
+from .write_ngrid import write_ngrid
+from .read_image_stack import read_image_stack
+from .mat_to_ngrid import mat_to_ngrid

src/ndi/fun/data/mat_to_ngrid.py

@@ -0,0 +1,2 @@
+def mat_to_ngrid():
+    raise NotImplementedError("Not yet ported.")

src/ndi/fun/data/read_image_stack.py

@@ -0,0 +1,5 @@
+def read_image_stack(session, doc, fmt):
+    """
+    Read image stack or video.
+    """
+    raise NotImplementedError("This function depends on imageio or opencv which are not yet in dependencies.")

src/ndi/fun/data/read_ngrid.py

@@ -0,0 +1,51 @@
+import numpy as np
+import os
+
+def read_ngrid(filename_or_fileobj, data_size, data_type='double'):
+    """
+    Read an n-dimensional matrix from a binary file.
+
+    Args:
+        filename_or_fileobj (str or fileobj): Path to file or file object.
+        data_size (list of int): Dimensions of matrix.
+        data_type (str): 'double', 'single', 'int8', 'uint8', etc.
+
+    Returns:
+        np.ndarray: N-dimensional matrix.
+    """
+    dtype_map = {
+        'double': np.float64,
+        'single': np.float32,
+        'int8': np.int8,
+        'uint8': np.uint8,
+        'int16': np.int16,
+        'uint16': np.uint16,
+        'int32': np.int32,
+        'uint32': np.uint32,
+        'int64': np.int64,
+        'uint64': np.uint64,
+        'char': np.char, # ?
+        'logical': np.bool_
+    }
+
+    np_dtype = dtype_map.get(data_type, np.float64)
+
+    if isinstance(filename_or_fileobj, str):
+        if not os.path.isfile(filename_or_fileobj):
+            raise FileNotFoundError(f"File not found: {filename_or_fileobj}")
+
+        with open(filename_or_fileobj, 'rb') as f:
+            # Numpy reads in C-order by default, but Matlab writes in F-order (column-major)
+            # data_size is (rows, cols, depth...)
+            # We should read as flat then reshape in 'F' order
+            count = np.prod(data_size)
+            x = np.fromfile(f, dtype=np_dtype, count=count)
+            x = x.reshape(data_size, order='F')
+
+    else: # file object
+        # Assuming read method
+        count = np.prod(data_size)
+        x = np.fromfile(filename_or_fileobj, dtype=np_dtype, count=count)
+        x = x.reshape(data_size, order='F')
+
+    return x

src/ndi/fun/data/write_ngrid.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+def write_ngrid(x, file_path, data_type='double'):
+    """
+    Write an n-dimensional matrix to a binary file.
+
+    Args:
+        x (np.ndarray): Data to write.
+        file_path (str): Path to output file.
+        data_type (str): Data type to write as.
+    """
+    dtype_map = {
+        'double': np.float64,
+        'single': np.float32,
+        'int8': np.int8,
+        'uint8': np.uint8,
+        'int16': np.int16,
+        'uint16': np.uint16,
+        'int32': np.int32,
+        'uint32': np.uint32,
+        'int64': np.int64,
+        'uint64': np.uint64,
+        'logical': np.bool_
+    }
+
+    np_dtype = dtype_map.get(data_type, np.float64)
+
+    # Ensure x is numpy array
+    if not isinstance(x, np.ndarray):
+        x = np.array(x)
+
+    # Cast to type
+    x_casted = x.astype(np_dtype)
+
+    with open(file_path, 'wb') as f:
+        # Write in Fortran order (column-major) to match Matlab
+        # To do this correctly:
+        # 1. Flatten in F order
+        # 2. Write to file
+        x_casted.flatten(order='F').tofile(f)

src/ndi/fun/dataset/__init__.py

@@ -0,0 +1 @@
+from .diff import diff

src/ndi/fun/dataset/diff.py

@@ -0,0 +1,9 @@
+def diff(dataset1, dataset2):
+    """
+    Compares two NDI datasets.
+
+    This functionality is similar to ndi.fun.session.diff, as dataset often inherits from session or behaves similarly.
+    """
+    # Reuse session diff
+    from ndi.fun.session.diff import diff as session_diff
+    return session_diff(dataset1, dataset2)

src/ndi/fun/debug_log.py

@@ -0,0 +1 @@
+def debug_log(): raise NotImplementedError("This function depends on ndi.common.getLogger which is not yet ported.")

src/ndi/fun/doc/__init__.py

@@ -0,0 +1,9 @@
+from .all_types import all_types
+from .diff import diff
+from .find_fuid import find_fuid
+from .get_doc_types import get_doc_types
+from .ontology_table_row_doc_to_table import ontology_table_row_doc_to_table
+from .ontology_table_row_vars import ontology_table_row_vars
+from .t0_t1_cell_to_array import t0_t1_cell_to_array
+from . import probe
+from . import subject