diff --git a/.gitignore b/.gitignore index bfc40973..c3bd3eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,27 @@ test_*.py *.csv .vscode/* test_delsys_api.py -resources/ \ No newline at end of file +resources/ +*.csv +*.txt +ContinuousTransitions/* +FORS-EMG/* +MyoDisCo/* +NinaProDB1/* +*.zip +libemg/_datasets/__pycache__/* +CIILData/* +EMGEPN612.pkl +OneSubjectMyoDataset/ +_3DCDataset/ +ContractionIntensity/ +CIILData/ +*.pkl +LimbPosition/ +CNN.py +MLP.py +__pycache__/ +MLPR.py +docs/Makefile +sifibridge-* +*.pyc diff --git a/docs/source/conf.py b/docs/source/conf.py index b1b1b756..54a44676 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -22,7 +22,7 @@ author = 'Ethan Eddy, Evan Campbell, Angkoon Phinyomark, Scott Bateman, and Erik Scheme' # The full version, including alpha/beta/rc tags -release = '1.0.0' +release = '2.0.0' # -- General configuration --------------------------------------------------- diff --git a/docs/source/doc.md b/docs/source/doc.md index b378126b..1243a575 100644 --- a/docs/source/doc.md +++ b/docs/source/doc.md @@ -20,6 +20,10 @@ The goal of this library is to provide an easy to use and feature-rich API for d $ pip install libemg ``` +# Interactive API Walkthrough + +Interactive offline and online walkthroughs were created as part of a `LibEMG` workshop we presented at [MEC24](https://www.unb.ca/ibme/mec/index.html), hosted by the Institute of Biomedical Engineering. These walkthroughs step through much of the core functionality of `LibEMG`, and offer visuals to explain pieces of the API. The offline example is formatted as a jupyter notebook, allowing you to see visuals for each code snippet without running the code yourself. Please check out out the workshop GitHub repository to see these interactive API walkthroughs: . + # Questions Check out our discord server if you have questions, comments, or feature requests: https://discord.gg/NeqTTXmM4F diff --git a/docs/source/documentation/adaptation/adaptation.rst b/docs/source/documentation/adaptation/adaptation.rst new file mode 100644 index 00000000..ebf4fd85 --- /dev/null +++ b/docs/source/documentation/adaptation/adaptation.rst @@ -0,0 +1,4 @@ +Online Adaptation +------------------------------ +.. include:: adaptation_doc.md + :parser: myst_parser.sphinx_ diff --git a/docs/source/documentation/adaptation/adaptation_doc.md b/docs/source/documentation/adaptation/adaptation_doc.md new file mode 100644 index 00000000..e71464b2 --- /dev/null +++ b/docs/source/documentation/adaptation/adaptation_doc.md @@ -0,0 +1,160 @@ +[View Source Code](https://github.com/ECEEvanCampbell/CoAdaptUnderCurriculum) + +The offline performance of a myoelectric model (e.g., $R^2$, mean absolute error on screen-guided training data) does not necessarily reflect how usable it is once a person is actually in the loop. A model that looks excellent on a calibration set can still drift or feel unresponsive online. `LibEMG` provides an **adaptation suite** that keeps improving a model *while the user operates it*, using the ongoing interaction as a source of labels. This is the mechanism behind context-informed incremental learning (see [Context-Informed Incremental Learning Improves Throughput and Reduces Drift in Regression-Based Myoelectric Control, Morrell et al. 2025](https://github.com/ECEEvanCampbell/CoAdaptUnderCurriculum)). + +This tutorial builds the minimal pipeline: a **within-subject initialization** from screen-guided training (SGT) data, followed by an **online environment that adapts the model** as the user plays through it. + +# Architecture +Adaptation runs as **four cooperating processes** that communicate over shared memory. The `libemg.adaptation._base.get_edil_adaptation_objects` helper pre-builds every shared-memory item and `OutputWriter` needed to wire them together (each modality buffer and its sample counter are paired on a shared lock so readers always see a consistent snapshot), so you never allocate them by hand. + +1. **`OnlineEMGRegressor`** — the live model. It windows the incoming EMG, extracts features, predicts, and publishes each feature vector (`model_input`) with a timestamp to shared memory. It also watches an `adapt_flag`; when the adaptation manager raises it, the regressor hot-swaps in the newly adapted model. +2. **Environment** (`CurricularFitts`) — the task the user performs. Every frame it turns the current cursor/target geometry into a *pseudo-label* through a `feedback_handle`, and writes that label (`environment_feedback`) with the same timestamp and trial number. +3. **`MemoryManager`** — joins each `environment_feedback` row to the `model_input` row with the matching timestamp, appends the pair to a `Memory`, and saves one memory slice (`memory_.pkl`) at the end of every trial. +4. **`AdaptationManager`** — loads memory slices (seeded by the SGT data for stability), calls `model.adapt(memory)`, saves the updated model (`mdl.pkl`), and — if `notify=True` — sets the `adapt_flag` so the live model reloads it. + +The loop is therefore: **predict → act → pseudo-label → remember → adapt → reload → predict …**, all without pausing the user. + +**You supply three things.** The suite is model-agnostic; it only assumes: +- a **model** with `predict(features)`, `adapt(memory)`, `save(path)`, and `load(path)` (the reference repository uses a small MLP / Transformer), +- a **memory** subclassing `libemg.adaptation.memory.Memory` (implementing `append`, `reset`, `save`, `load`, and `__add__`), +- a **feedback function** mapping game state to a pseudo-label — `libemg.adaptation._base.produce_tciil_feedback` is a ready-made regression example. + +**Note:** The snippets below use the `Myo Armband` and a 2-DOF wrist regressor (flexion/extension and radial/ulnar deviation). Any hardware works by switching the `streamer`, `window_size`, and `window_increment`. + +# Step 1 — Within-Subject Initialization +First, record a short screen-guided training session and fit an initial model to it. This is the same collection flow used elsewhere in `LibEMG`; for regression we prompt the four wrist directions and record continuous labels. + +```Python +import libemg +from libemg.streamers import myo_streamer + +# Stream from the device and collect calibration (SGT) data. +streamer, sm_items = myo_streamer() +odh = libemg.data_handler.OnlineDataHandler(shared_memory_items=sm_items) + +gui = libemg.gui.GUI(odh, args={ + 'media_folder': 'media/', # regression prompts (wrist flexion/extension, radial/ulnar) + 'data_folder': 'data/sgt/', + 'num_reps': 5, + 'rep_time': 5, + 'auto_advance': True, +}) +gui.start_gui() +streamer.stop() +``` + +Next, parse those recordings with an `OfflineDataHandler`, fit your model, and — crucially — save the SGT data **as a memory slice**. That slice seeds the adaptation manager so early online updates don't wander away from a known-good starting point. + +```Python +# Parse the SGT recordings. +offdh = libemg.data_handler.OfflineDataHandler() +offdh.get_data('data/sgt/', regex_filters, metadata_fetchers, delimiter=',') + +# Fit the initial within-subject model (implements predict/adapt/save/load). +model = MyModel(...) +model.calibrate(offdh) + +# Seed the adaptation with the SGT data as the first memory slice. +initial_memory = MyMemory(...) # subclass of libemg.adaptation.memory.Memory +initial_memory.load_from_odh(offdh) +initial_memory.save('data/adapt/sgt_memory.pkl') +``` + +The same `model` object is handed to both the live regressor and the adaptation manager below; because they run in separate processes each gets its own copy. + +# Step 2 — Launch the Online Adaptive Environment +Now assemble the four processes. Start by requesting the pre-wired shared-memory items and output writers. + +```Python +from libemg.adaptation._base import get_edil_adaptation_objects, produce_tciil_feedback +from libemg.adaptation.managers import MemoryManager, AdaptationManager +from libemg.environments.controllers import RegressorController +from libemg.environments.curricular_fitts import ( + CurricularFitts, CurricularFittsConfig, RadiusTargetGenerator, +) + +NUM_FEATURES, NUM_DOFS, NUM_TRIALS = 64, 2, 80 +ADAPT_DIR = 'data/adapt/' # adapted models (mdl.pkl) AND memory slices (memory_.pkl) + +(model_smi, model_ow, + environment_smi, environment_ow, + adaptation_manager_smi, adaptation_manager_ow, + memory_manager_smi, memory_manager_ow) = get_edil_adaptation_objects( + num_features=NUM_FEATURES, num_outputs=NUM_DOFS) +``` + +**The live model.** Pass `model_ow`/`model_smi` so it publishes `model_input` and exposes the `active_flag`/`adapt_flag`. Its `file_path` is where it reloads adapted models from. + +```Python +streamer, sm_items = myo_streamer() +odh = libemg.data_handler.OnlineDataHandler(shared_memory_items=sm_items) + +emg_regressor = libemg.emg_predictor.EMGRegressor(model) # the SGT-trained model +online_regressor = libemg.emg_predictor.OnlineEMGRegressor( + offline_regressor = emg_regressor, + online_data_handler = odh, + window_size = 100, window_increment = 40, + features = ['WENG'], + output_writers = model_ow, # publishes model_input (features + timestamp) + smm = True, smm_items = model_smi, # exposes active_flag / adapt_flag / model_input + file_path = ADAPT_DIR, # reloads mdl.pkl from here when adapt_flag is set +) +online_regressor.run(block=False) +``` + +**The environment.** The `feedback_handle` converts cursor/target geometry into a pseudo-label; `environment_ow` writes it out for the memory manager. + +```Python +controller = RegressorController() +config = CurricularFittsConfig( + feedback_handle = produce_tciil_feedback, + num_trials = NUM_TRIALS, + controller_map = [1, -1], +) +env = CurricularFitts( + controller, config, + target_generator = RadiusTargetGenerator(config, F=0, P=0), + environment_ow = environment_ow, # writes environment_feedback (label + timestamp + trial) + save_file = ADAPT_DIR, +) +``` + +**The memory and adaptation managers.** Two path linkages must line up: the adaptation manager's `load_dir` equals the memory manager's `save_dir` (where slices are written), and its `save_dir` equals the online regressor's `file_path` (where adapted models are read back). Setting `notify=True` is what closes the loop by raising the `adapt_flag`. + +```Python +memory_manager = MemoryManager( + memory = MyMemory(...), # empty memory of the same type as the SGT seed + smi = memory_manager_smi, + ow = memory_manager_ow, + save_dir = ADAPT_DIR, # writes memory_.pkl +) + +adaptation_manager = AdaptationManager( + model = model, # adapts its own copy of the SGT model + smi = adaptation_manager_smi, + ow = adaptation_manager_ow, + initial_memory_loc = ADAPT_DIR + 'sgt_memory.pkl', # SGT seed for stability + load_dir = ADAPT_DIR, # reads memory_.pkl (== MemoryManager.save_dir) + save_dir = ADAPT_DIR, # writes mdl.pkl (== OnlineEMGRegressor.file_path) + stop_condition = lambda n: n >= NUM_TRIALS - 1, + notify = True, # raise adapt_flag so the live model hot-swaps +) +``` + +Finally, launch them. Run the environment and memory manager in the background and block on the adaptation manager; it returns once `stop_condition` is met. Then tear everything down. + +```Python +env.run_helper(block=False) # spawn the game +memory_manager.run_helper(block=False) # spawn the memory assembler +adaptation_manager.run_helper(block=True) # adapt until stop_condition; blocks here + +# Cleanup once adaptation ends. +env.process.join() +memory_manager.signal.set(); memory_manager.join() +online_regressor.odh.stop_all() +online_regressor.stop_running() +streamer.stop() +``` + +# Result +As the user completes trials, memory slices accumulate, the adaptation manager retrains on them (seeded by the SGT slice), and the live model is swapped out mid-session — so control quality improves *during* use rather than only between sessions. To make the model non-adaptive for a baseline comparison, build everything identically but pass `notify=False` to the `AdaptationManager`: memories are still collected, but the `adapt_flag` is never raised and the live model stays fixed. diff --git a/docs/source/documentation/animation/animation.rst b/docs/source/documentation/animation/animation.rst new file mode 100644 index 00000000..bada21c1 --- /dev/null +++ b/docs/source/documentation/animation/animation.rst @@ -0,0 +1,4 @@ +Animation +------------------------------ +.. include:: animation_doc.md + :parser: myst_parser.sphinx_ \ No newline at end of file diff --git a/docs/source/documentation/animation/animation_doc.md b/docs/source/documentation/animation/animation_doc.md new file mode 100644 index 00000000..e461defa --- /dev/null +++ b/docs/source/documentation/animation/animation_doc.md @@ -0,0 +1,75 @@ +## Use Case + +The `Animator` class offers some simple functionality to create animations in multiple video formats (e.g., .gif and .mp4). Specific `Animator` classes have been created to help create specific visual prompts, such as bar plots and cartesian plots. These prompts are primarily used for training of regression-based myoelectric control systems, but the `Animator` class can also be used to generate these prompts for any purpose. For fully custom animations, you can inherit from the `Animator` class and pass a set of frames to `save_video()` (see other `Animators` for examples). If you'd rather implement a custom plotting animation that isn't supported, you can inherit from `PlotAnimator` instead. + +For all further examples, we will use a generated set of coordinates to illustrate the difference between plots. + +```Python +import numpy as np + +fps = 24 +coordinates = np.concatenate(( + np.linspace(0, 1, num=fps), # each movement is 24 frames -> 1 second + np.ones(2 * fps), # steady state + np.linspace(1, -1, num=2 * fps), + np.ones(2 * fps) * -1, + np.linspace(-1, 0, num=fps) +)) +coordinates = np.hstack(( + np.expand_dims(coordinates, 1), + np.zeros((coordinates.shape[0], 1)) +)) +``` + +## Bar Plots + +One type of `Animator` built into `LibEMG` creates bar plot animations. Pass in an array of coordinates to the `plot_icon` method to create a bar plot visualization (see Figure 1). + +```Python +from libemg.animator import BarPlotAnimator + +animator = BarPlotAnimator(['Open', 'Close'], fps=fps) +animator.save_plot_video(coordinates) +``` + +![alt text](bar.gif) +

Figure 1: Simple bar plot animation.

+ +Additional information can also be shown during these animations, such as the next destination and a countdown for steady states (see Figure 2). + +```Python +animator = BarPlotAnimator(['Open', 'Close'], fps=fps, show_countdown=True, show_direction=True) +animator.save_plot_video(coordinates) +``` + +![alt text](bar-info.gif) +

Figure 2: Bar plot animation with added information.

+ +Parameters such as the time per unit distance, figure size, and more can also be modified. See the `BarPlotAnimator` API for more details. + +## Scatter Plots + +`LibEMG` also provides a helper class to animate scatter plots (see Figure 3). + +```Python +from libemg.animator import ScatterPlotAnimator + +animator = ScatterPlotAnimator(['Open', 'Close'], fps=fps) +animator.save_plot_video(coordinates) +``` + +![alt text](scatter.gif) +

Figure 3: Simple scatter plot animation.

+ +Similar to the bar plot animation, extra information can be added such as next destination, a countdown, and a unit circle boundary (see Figure 4). + +```Python + +animator = ScatterPlotAnimator(['Open', 'Close'], fps=fps, show_countdown=True, show_direction=True, show_boundary=True) +animator.save_plot_video(coordinates) +``` + +![alt text](scatter-info.gif) +

Figure 4: Scatter plot animation with added information.

+ +Parameters such as the time per unit distance, figure size, and more can also be modified. See the `ScatterPlotAnimator` API for more details. diff --git a/docs/source/documentation/animation/bar-info.gif b/docs/source/documentation/animation/bar-info.gif new file mode 100644 index 00000000..c43417d9 Binary files /dev/null and b/docs/source/documentation/animation/bar-info.gif differ diff --git a/docs/source/documentation/animation/bar.gif b/docs/source/documentation/animation/bar.gif new file mode 100644 index 00000000..97d39c8a Binary files /dev/null and b/docs/source/documentation/animation/bar.gif differ diff --git a/docs/source/documentation/animation/scatter-info.gif b/docs/source/documentation/animation/scatter-info.gif new file mode 100644 index 00000000..3ab0b7a0 Binary files /dev/null and b/docs/source/documentation/animation/scatter-info.gif differ diff --git a/docs/source/documentation/animation/scatter.gif b/docs/source/documentation/animation/scatter.gif new file mode 100644 index 00000000..2fa251c9 Binary files /dev/null and b/docs/source/documentation/animation/scatter.gif differ diff --git a/docs/source/documentation/classification/classification_doc.md b/docs/source/documentation/classification/classification_doc.md deleted file mode 100644 index 077d4480..00000000 --- a/docs/source/documentation/classification/classification_doc.md +++ /dev/null @@ -1,101 +0,0 @@ -# Classifiers -After recording, processing, and extracting features from a window of EMG data, it is passed to a machine learning algorithm for classification. These control systems have evolved in the prosthetics community for continuously classifying muscular contractions for enabling prosthesis control. Therefore, they are primarily limited to recognizing static contractions (e.g., hand open/close and wrist flexion/extension) as they have no temporal awareness. Currently, this is the form of recognition supported by LibEMG and is an initial step to explore EMG as an interaction opportunity for general-purpose use. This section highlights the machine-learning strategies that are part of LibEMG's pipeline. Additionally, a number of post-processing methods (i.e., techniques to improve performance after classification) are explored. - -## Statistical Models - -The statistical classifiers (i.e., traditional machine learning methods) implemented leverage the sklearn package. For most cases, the "base" classifiers use the default options, meaning that the pre-defined models are not necessarily optimal. However, the `parameters` attribute can be used when initializing the classifiers to pass in additional sklearn parameters in a dictionary. For example, looking at the `RandomForestClassifier` docs on sklearn: - -![Random Forest](random_forest.png) - -A classifier with any of those parameters using the `parameters` attribute. For example: -```Python -parameters = { - 'n_estimators': 99, - 'max_depth': 20, - 'random_state': 5, - 'max_leaf_nodes': 10 -} -classifier.fit(data_set, parameters=parameters) -``` - -Please reference the [sklearn docs](https://scikit-learn.org/stable/) for parameter options for each classifier. - -Additionally, custom classifiers can be created. Any custom classifier should be modeled after the sklearn classifiers and must have the `fit`, `predict`, and `predict_proba` functions to work correctly. - -```Python -from sklearn.ensemble import RandomForestClassifier -from libemg.predictor import EMGClassifier - -rf_custom_classifier = RandomForestClassifier(max_depth=5, random_state=0) -classifier = EMGClassifier(rf_custom_classifier) -classifier.fit(data_set) -``` - -### Linear Discriminant Analysis (LDA) -A linear classifier that uses common covariances for all classes and assumes a normal distribution. -```Python -classifier = EMGClassifier('LDA') -classifier.fit(data_set) -``` -Check out the LDA docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html) - -### K-Nearest Neighbour (KNN) -Discriminates between inputs using the K closest samples in feature space. The implemented version in the library defaults to k = 5. A commonly used classifier for EMG-based recognition. - -```Python -params = {'n_neighbors': 5} # Optional -classifier = EMGClassifier('KNN') -classifier.fit(data_set, parameters=params) -``` -Check out the KNN docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html) - -### Support Vector Machines (SVM) -A hyperplane that maximizes the distance between classes is used as the boundary for recognition. A commonly used classifier for EMG-based recognition. -```Python -classifier = EMGClassifier('SVM') -classifier.fit(data_set) -``` -Check out the SVM docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html) - -### Artificial Neural Networks (MLP) -A deep learning technique that uses human-like "neurons" to model data to help discriminate between inputs. Especially for this model, we **highly** recommend you create your own. -```Python -classifier = EMGClassifier('MLP') -classifier.fit(data_set) -``` -Check out the MLP docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html) - -### Random Forest (RF) -Uses a combination of decision trees to discriminate between inputs. -```Python -classifier = EMGClassifier('RF') -classifier.fit(data_set) -``` -Check out the RF docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html) - -### Quadratic Discriminant Analysis (QDA) -A quadratic classifier that uses class-specific covariances and assumes normally distributed classes. -```Python -classifier = EMGClassifier('QDA') -classifier.fit(data_set) -``` -Check out the QDA docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html) - -### Gaussian Naive Bayes (NB) -Assumes independence of all input features and normally distributed classes. -```Python -classifier = EMGClassifier('NB') -classifier.fit(data_set) -``` -Check out the NB docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html) - - - - -## Deep Learning (Pytorch) -Another available option is to use [pytorch](https://pytorch.org/) models (i.e., a library for deep learning) to train the classifier, although this involves making some custom code for preparing the dataset and the deep learning model. For a guide on how to use deep learning models, consult the deep learning example. \ No newline at end of file diff --git a/docs/source/documentation/data/data_doc.md b/docs/source/documentation/data/data_doc.md index daea40c3..a439592e 100644 --- a/docs/source/documentation/data/data_doc.md +++ b/docs/source/documentation/data/data_doc.md @@ -23,146 +23,1031 @@ This module has three data-related functions: **(1) Datasets**, **(2) Offline Da # Datasets Several validated datasets consisting of different gestures and recording technology are included in this library. These datasets can be used for exploring the library's capabilities and for future research. When using the packaged datasets for research purposes, we ask that you reference the original dataset contribution and not just this toolkit (the original dataset contributions might not be obvious since they are included for download with this toolkit). +## Classification + + +
OneSubjectMyoDataset +
+ +**Dataset Description:** +Simple one subject dataset. + | Attribute | Description | | ------------------ | ----------- | | **Num Subjects:** | 1 | | **Num Reps:** | 12 Reps (i.e., 6 Trials x 2 Reps)| -| **Time Per Rep:** | 3s | | **Classes:** |
  • 0 - Hand Open
  • 1 - Hand Close
  • 2 - No Movement
  • 3 - Wrist Extension
  • 4 - Wrist Flexion
| | **Device:** | Myo | -| **Sampling Rates:** | EMG (200 Hz) | +| **Sampling Rates:** | 200 Hz | +| **Auto Download:** | True | + **Using the Dataset:** ```Python -from libemg.datasets import OneSubjectMyoDataset -dataset = OneSubjectMyoDataset(redownload=False) +from libemg.datasets import * +dataset = get_dataset_list()['OneSubjectMyo']() odh = dataset.prepare_data() ``` +**Dataset Location** +https://github.com/LibEMG/OneSubjectEMaGerDataset + **References:** ``` -Work to be published... +@ARTICLE{libemg, + author={Eddy, Ethan and Campbell, Evan and Phinyomark, Angkoon and Bateman, Scott and Scheme, Erik}, + journal={IEEE Access}, + title={LibEMG: An Open Source Library to Facilitate the Exploration of Myoelectric Control}, + year={2023}, + volume={11}, + number={}, + pages={87380-87397}, + keywords={Electromyography;Prosthetics;Libraries;Human computer interaction;Feature extraction;Muscles;Control systems;Gesture recognition;Open source software;EMG;electromyography;toolkit;library;myoelectric control;gesture recognition}, + doi={10.1109/ACCESS.2023.3304544}} ``` -------------

+ +
3DCDatset +
+ +**Dataset Description:** +A relatively simple within session baseline. + | Attribute | Description | | ------------------ | ----------- | | **Num Subjects:** | 22 | | **Num Reps:** | 4 Training, 4 Testing | -| **Time Per Rep:** | 5s | | **Classes:** |
  • 0 - No Motion
  • 1 - Radial Deviaton
  • 2 - Wrist Flexion
  • 3 - Ulnar Deviaton
  • 4 - Wrist Extension
  • 5 - Supination
  • 6 - Pronation
  • 7 - Power Grip
  • 8- Open Hand
  • 9 - Chuck Grip
  • 10 - Pinch Grip
| | **Device:** | Delsys | -| **Sampling Rates:** | EMG (1000 Hz) | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | **Using the Dataset:** ```Python -from libemg.datasets import _3DCDataset -dataset = _3DCDataset(redownload=False) +from libemg.datasets import * +dataset = get_dataset_list()['3DC']() odh = dataset.prepare_data() ``` +**Dataset Location** +https://github.com/LibEMG/3DCDataset + **References:** ``` -@article{cote2019deep, title={Deep learning for electromyographic hand gesture signal classification using transfer learning}, author={C{^o}t{'e}-Allard, Ulysse and Fall, Cheikh Latyr and Drouin, Alexandre and Campeau-Lecours, Alexandre and Gosselin, Cl{'e}ment and Glette, Kyrre and Laviolette, Fran{\c{c}}ois and Gosselin, Benoit}, journal={IEEE transactions on neural systems and rehabilitation engineering}, volume={27}, number={4}, pages={760--771}, year={2019}, publisher={IEEE} } +@article{cote2019low, + title={A low-cost, wireless, 3-D-printed custom armband for sEMG hand gesture recognition}, + author={C{\^o}t{\'e}-Allard, Ulysse and Gagnon-Turcotte, Gabriel and Laviolette, Fran{\c{c}}ois and Gosselin, Benoit}, + journal={Sensors}, + volume={19}, + number={12}, + pages={2811}, + year={2019}, + publisher={MDPI} +} +``` +
+
+ + + + +
+CIIL_MinimalData + +
+ +**Dataset Description:** +The goal of this Myo dataset is to explore how well models perform when they have a limited amount of training data (1s per class). + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 11 | +| **Num Reps:** | 1 Train, 15 Test | +| **Classes:** |
  • 0 - Close
  • 1 - Open
  • 2 - Rest
  • 3 - Flexion
  • 4 - Extension
| +| **Device:** | Myo Armband | +| **Sampling Rates:** | 200 Hz | +| **Auto Download:** | True | + + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['CIIL_MinimalData']() +odh = dataset.prepare_data() +``` -@article{cote2020interpreting, title={Interpreting deep learning features for myoelectric control: A comparison with handcrafted features}, author={C{^o}t{'e}-Allard, Ulysse and Campbell, Evan and Phinyomark, Angkoon and Laviolette, Fran{\c{c}}ois and Gosselin, Benoit and Scheme, Erik}, journal={Frontiers in Bioengineering and Biotechnology}, volume={8}, pages={158}, year={2020}, publisher={Frontiers Media SA} } +**Dataset Location** +https://github.com/LibEMG/CIILData + +**References:** ``` -------------- +@inproceedings{ciil_md, + title={Leveraging task-specific context to improve unsupervised adaptation for myoelectric control}, + author={Eddy, Ethan and Campbell, Evan and Bateman, Scott and Scheme, Erik}, + booktitle={2023 IEEE International Conference on Systems, Man, and Cybernetics (SMC)}, + pages={4661--4666}, + year={2023}, + organization={IEEE} +} +``` +
+
-
+
-Nina Pro DB2 +CIIL_ElectrodeShift -
-The Ninapro DB2 is a dataset that can be used to test how algorithms perform for large gesture sets. The dataset contains 6 repetitions of 50 motion classes (plus optional rest) that were recorded using 12 Delsys Trigno electrodes around the forearm. +
-
-
+**Dataset Description:** +An electrode shift confounding factors dataset. -Note, this dataset will not be automatically downloaded. To download this dataset, please see [Nina DB2](http://ninapro.hevs.ch/node/17). Simply download the ZIPs and place them in a folder and LibEMG will handle the rest. All credit for this dataset should be given to the original authors. +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 21 | +| **Num Reps:** | 5 Train (Before Shift), 8 Test (After Shift) | +| **Classes:** |
  • 0 - Close
  • 1 - Open
  • 2 - Rest
  • 3 - Flexion
  • 4 - Extension
| +| **Device:** | Myo Armband | +| **Sampling Rates:** | 200 Hz | +| **Auto Download:** | True | -
+**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['CIIL_ElectrodeShift']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/CIILData + +**References:** +``` +@article{ciil_es, + title={Context-informed incremental learning improves both the performance and resilience of myoelectric control}, + author={Campbell, Evan and Eddy, Ethan and Bateman, Scott and C{\^o}t{\'e}-Allard, Ulysse and Scheme, Erik}, + journal={Journal of NeuroEngineering and Rehabilitation}, + volume={21}, + number={1}, + pages={70}, + year={2024}, + publisher={Springer} +} +``` + +
+
+ + + +
+CIIL_WeaklySupervised + +
+ +**Dataset Description:** +A weakly supervised environment with sparse supervised calibration. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 16 | +| **Num Reps:** | 30 min weakly supervised, 1 rep calibration, 14 reps test | +| **Classes:** |
  • 0 - Close
  • 1 - Open
  • 2 - Rest
  • 3 - Flexion
  • 4 - Extension
| +| **Device:** | OyMotion gForcePro+ EMG Armband | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('WEAKLYSUPERVISED')['CIIL_WeaklySupervised']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/WS_CIIL + +**References:** +``` +In Publication... +``` + +
+
+ + +
+ContinuousTransitions + +
+ +**Dataset Description:** +The testing set in this dataset has continuous transitions between classes, providing a more realistic offline evaluation standard for myoelectric control. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 43 | +| **Num Reps:** | 6 Training (Ramp), 42 Transitions (All combinations of Transitions) x 6 Reps | +| **Classes:** |
  • 0 - No Motion
  • 1 - Wrist Flexion
  • 2 - Wrist Extension
  • 3 - Wrist Pronation
  • 4 - Wrist Supination
  • 5 - Hand Close
  • 6 - Hand Open
| +| **Device:** | Delsys | +| **Sampling Rates:** | 2000 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['ContinuousTransitions']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://unbcloud-my.sharepoint.com/:f:/g/personal/ecampbe2_unb_ca/EjgjhM9ZHJxOglKoAf062ngBf4wFj2Mn2bORKY1-aMYGRw?e=WkZNwI + +**References:** +``` +@ARTICLE{transitions, + author={Raghu, Shriram Tallam Puranam and MacIsaac, Dawn and Scheme, Erik}, + journal={IEEE Journal of Biomedical and Health Informatics}, + title={Decision-Change Informed Rejection Improves Robustness in Pattern Recognition-Based Myoelectric Control}, + year={2023}, + volume={27}, + number={12}, + pages={6051-6061}, + doi={10.1109/JBHI.2023.3316599}} +``` + +
+
+ + +
+ContractionIntensity + +
+ +**Dataset Description:** +A contraction intensity dataset. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 10 | +| **Num Reps:** | 4 Ramp Reps (Train), 4 Reps x 20%, 30%, 40%, 50%, 60%, 70%, 80%, MVC (Test) | +| **Classes:** |
  • 0 - No Motion
  • 1 - Wrist Flexion
  • 2 - Wrist Extension
  • 3 - Wrist Pronation
  • 4 - Wrist Supination
  • 5 - Chuck Grip
  • 6 - Hand Open
| +| **Device:** | BE328 by Liberating Technologies, Inc | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['ContractionIntensity']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/ContractionIntensity + +**References:** +``` +@article{contraction_intensity, + title={Training strategies for mitigating the effect of proportional control on classification in pattern recognition--based myoelectric control}, + author={Scheme, Erik and Englehart, Kevin}, + journal={JPO: Journal of Prosthetics and Orthotics}, + volume={25}, + number={2}, + pages={76--83}, + year={2013}, + publisher={LWW} +} +``` + +
+
+ + +
+EMGEPN612 + +
+ +**Dataset Description:** +A large 612 user dataset for developing cross-user models. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 612 | +| **Num Reps:** | 50 Reps x 306 Users (Train), 25 Reps x 306 Users (Test) --> Cross User Split | +| **Classes:** |
  • 0 - No Movement
  • 1 - Hand Close
  • 2 - Flexion
  • 3 - Extension
  • 4 - Hand Open
  • 5 - Pinch
| +| **Device:** | Myo Armband | +| **Sampling Rates:** | 200 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['EMGEPN612']() # User Dependent +dataset = get_dataset_list(cross_user=True)['EMGEPN612']() # User Independent +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://unbcloud-my.sharepoint.com/:u:/g/personal/ecampbe2_unb_ca/EWf3sEvRxg9HuAmGoBG2vYkBLyFv6UrPYGwAISPDW9dBXw?e=vjCA14 + +**References:** +``` +@article{epn, + title={EMG-EPN-612 Dataset. 2020}, + author={Benalc{\'a}zar, M and Barona, L and Valdivieso, L and Aguas, X and Zea, J}, + journal={DOI: https://doi. org/10.5281/zenodo}, + volume={4027874}, + year={2020} +} +``` + +
+
+ + +
+FORSEMG + +
+ +**Dataset Description:** +Twelve gestures elicited in three forearm orientations (neutral, pronation, and supination). + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 19 | +| **Num Reps:** | 5 Train, 10 Test (2 Forearm Orientations x 5 Reps) | +| **Classes:** |
  • 0 - Thump Up
  • 1 - Index
  • 2 - Right Angle
  • 3 - Peace
  • 4 - Index Little
  • 5 - Thumb Little
  • 6 - Hand Close
  • 7 - Hand Open
  • 8 - Wrist Flexion
  • 9 - Wrist Extension
  • 10 - Ulnar Deviation
  • 11 - Radial Deviation
| +| **Device:** | Experimental Device | +| **Sampling Rates:** | 985 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['FORSEMG']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://www.kaggle.com/datasets/ummerummanchaity/fors-emg-a-novel-semg-dataset + +**References:** +``` +@article{fors_emg, + title={FORS-EMG: A Novel sEMG Dataset for Hand Gesture Recognition Across Multiple Forearm Orientations}, + author={Rumman, Umme and Ferdousi, Arifa and Hossain, Md Sazzad and Islam, Md Johirul and Ahmad, Shamim and Reaz, Mamun Bin Ibne and Islam, Md Rezaul}, + journal={arXiv preprint arXiv:2409.07484}, + year={2024} +} +``` + +
+
+ + + + +
+FougnerLP + +
+ +**Dataset Description:** +A limb position dataset (with 5 static limb positions). + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 12 | +| **Num Reps:** | 10 Reps (Train), 10 Reps x 4 Positions | +| **Classes:** |
  • 0 - Wrist Flexion
  • 1 - Wrist Extension
  • 2 - Pronation
  • 3 - Supination
  • 4 - Hand Open
  • 5 - Power Grip
  • 6 - Pinch Grip
  • 7 - Rest
| +| **Device:** | BE328 by Liberating Technologies, Inc. | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | + + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['FougnerLP']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/LimbPosition + +**References:** +``` +@article{fougner_lp, + title={Resolving the limb position effect in myoelectric pattern recognition}, + author={Fougner, Anders and Scheme, Erik and Chan, Adrian DC and Englehart, Kevin and Stavdahl, {\O}yvind}, + journal={IEEE Transactions on Neural Systems and Rehabilitation Engineering}, + volume={19}, + number={6}, + pages={644--651}, + year={2011}, + publisher={IEEE} +} +``` + +
+
+ + +
+GRABMyo + +
+ +**Dataset Description:** +A large cross-session dataset including 17 gestures elicited across 3 separate sessions. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 43 | +| **Num Reps:** | 7 Train, 14 Test (2 Separate Days x 7 Reps) --> Cross Day Split | +| **Classes:** |
  • 0 - Lateral Prehension
  • 1 - Thumb Adduction
  • 2 - Thumb and Little Finger Opposition
  • 3 - Thumb and Index Finger Opposition
  • 4 - Thumb and Index Finger Extension
  • 5 - Thumb and Little Finger Extension
  • 6 - Index and Middle Finger Extension
  • 7 - Little Finger Extension
  • 8 - Index Finger Extension
  • 9 - Thumb Finger Extension
  • 10 - Wrist Extension
  • 11 - Wrist Flexion
  • 12 - Forearm Supination
  • 13 - Forearm Pronation
  • 14 - Hand Open
  • 15 - Hand Close
  • 16 - Rest
| +| **Device:** | EMGUSB2+ device (OT Bioelletronica, Italy) | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['GRABMyoBaseline']() # Baseline +dataset = get_dataset_list()['GRABMyoCrossDay']() # CrossDay +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://physionet.org/content/grabmyo/1.0.2/ + +**References:** +``` +@article{grabmyo, + title={Multi-day dataset of forearm and wrist electromyogram for hand gesture recognition and biometrics}, + author={Pradhan, Ashirbad and He, Jiayuan and Jiang, Ning}, + journal={Scientific data}, + volume={9}, + number={1}, + pages={733}, + year={2022}, + publisher={Nature Publishing Group UK London} +} +``` + +
+
+ + + +
+HyserPR + +
+ +**Dataset Description:** +High Density Hyser pattern recognition (PR) dataset. Includes dynamic and maintenance tasks for 34 hand gestures. + +| Attribute | Description | +| ------------------ | ----------- | +| **Num Subjects:** | 18 | +| **Num Reps:** | 1 Train, 1 Test (Consisting of dynamic and maintanance tasks) | +| **Classes:** |
  • 1 - Thumb Extension
  • 2 - Index Finger Extension
  • 3 - Middle Finger Extension
  • 4 - Ring Finger Extension
  • 5 - Little Finger Extension
  • 6 - Wrist Flexion
  • 7 - Wrist Extension
  • 8 - Wrist Radial
  • 9 - Wrist Ulnar
  • 10 - Wrist Pronation
  • 11 - Wrist Supination
  • 12 - Extension of Thumb and Index Fingers
  • 13 - Extension of Index and Middle Fingers
  • 14 - Wrist Flexion Combined with Hand Close
  • 15 - Wrist Extension Combined with Hand Close
  • 16 - Wrist Radial Combined with Hand Close
  • 17 - Wrist Ulnar Combined with Hand Close
  • 18 - Wrist Pronation Combined with Hand Close
  • 19 - Wrist Supination Combined with Hand Close
  • 20 - Wrist Flexion Combined with Hand Open
  • 21 - Wrist Extension Combined with Hand Open
  • 22 - Wrist Radial Combined with Hand Open
  • 23 - Wrist Ulnar Combined with Hand Open
  • 24 - Wrist Pronation Combined with Hand Open
  • 25 - Wrist Supination Combined with Hand Open
  • 26 - Extension of Thumb, Index and Middle Fingers
  • 27 - Extension of Index, Middle and Ring Fingers
  • 28 - Extension of Middle, Ring and Little Fingers
  • 29 - Extension of Index, Middle, Ring and Little Fingers
  • 30 - Hand Close
  • 31 - Hand Open
  • 32 - Thumb and Index Fingers Pinch
  • 33 - Thumb, Index and Middle Fingers Pinch
  • 34 - Thumb and Middle Fingers Pinch
| +| **Device:** | OT Bioelettronica Quattrocento | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['HyserPR']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://www.physionet.org/content/hd-semg/2.0.0/ + +**References:** +``` +@ARTICLE{hyser, + author={Jiang, Xinyu and Liu, Xiangyu and Fan, Jiahao and Ye, Xinming and Dai, Chenyun and Clancy, Edward A. and Akay, Metin and Chen, Wei}, + journal={IEEE Transactions on Neural Systems and Rehabilitation Engineering}, + title={Open Access Dataset, Toolbox and Benchmark Processing Results of High-Density Surface Electromyogram Recordings}, + year={2021}, + volume={29}, + number={}, + pages={1035-1046}, + doi={10.1109/TNSRE.2021.3082551}} +``` +
+
+ + +
+KaufmannMD + +
+ +**Dataset Description:** +A single subject, multi-day (120 days) collection. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 1 | +| **Num Reps:** | 1 rep per day, 120 days total. 60/60 train-test split | +| **Classes:** |
  • 0 - No Motion
  • 1 - Wrist Extension
  • 2 - Wrist Flexion
  • 3 - Wrist Adduction
  • 4 - Wrist Abduction
  • 5 - Wrist Supination
  • 6 - Wrist Pronation
  • 7 - Hand Open
  • 8 - Hand Closed
  • 9 - Key Grip
  • 10 - Index Point
| +| **Device:** | MindMedia | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['KaufmannMD']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/MultiDay + +**References:** +``` +@INPROCEEDINGS{kaufmann, + author={Kaufmann, Paul and Englehart, Kevin and Platzner, Marco}, + booktitle={2010 Annual International Conference of the IEEE Engineering in Medicine and Biology}, + title={Fluctuating emg signals: Investigating long-term effects of pattern matching algorithms}, + year={2010}, + volume={}, + number={}, + pages={6357-6360}, + doi={10.1109/IEMBS.2010.5627288}} +``` + +
+
+ + + +
+NinaProDB2 + +
+ +**Dataset Description:** +The Ninapro DB2 is a dataset that can be used to test how algorithms perform for large gesture sets. The dataset contains 6 repetitions of 50 motion classes (plus optional rest) that were recorded using 12 Delsys Trigno electrodes around the forearm. | Attribute | Description | | ------------------ | ----------- | | **Num Subjects:** | 40 | | **Num Reps:** | 6 | -| **Time Per Rep:** | 5s | | **Classes:** | 50 [Nina Pro DB2](http://ninapro.hevs.ch/node/123) | | **Device:** | Delsys | -| **Sampling Rates:** | EMG (2000 Hz) | +| **Sampling Rates:** | 2000 Hz | +| **Auto Download:** | False | **Using the Dataset:** ```Python -from libemg.datasets import NinaproDB2 -dataset = NinaproDB2("data/NinaDB2") #The loacation of Nina DB2 is downloaded +from libemg.datasets import * +dataset = get_dataset_list()['NinaProDB2']() odh = dataset.prepare_data() ``` +**Dataset Location** +Note, this dataset will not be automatically downloaded. To download this dataset, please see [Nina DB2](http://ninapro.hevs.ch/node/17). Simply download the ZIPs and place them in a folder and LibEMG will handle the rest. All credit for this dataset should be given to the original authors. + **References:** ``` -Atzori, M., Gijsberts, A., Castellini, C. et al. -Electromyography data for non-invasive naturally-controlled robotic hand prostheses. -Sci Data 1, 140053 (2014). -https://doi.org/10.1038/sdata.2014.53 +@article{db2, + title={Electromyography data for non-invasive naturally-controlled robotic hand prostheses}, + author={Atzori, Manfredo and Gijsberts, Arjan and Castellini, Claudio and Caputo, Barbara and Hager, Anne-Gabrielle Mittaz and Elsig, Simone and Giatsidis, Giorgio and Bassetto, Franco and M{\"u}ller, Henning}, + journal={Scientific data}, + volume={1}, + number={1}, + pages={1--13}, + year={2014}, + publisher={Nature Publishing Group} +} ``` -------------

+
-Nina Pro DB8 +RadmandLP + +
-
+**Dataset Description:** +A large limb position dataset (with 16 static limb positions). -Note, this dataset will not be automatically downloaded. To download this dataset, please see [Nina DB8](http://ninapro.hevs.ch/DB8). Simply download the ZIPs and place them in a folder and LibEMG will handle the rest. All credit for this dataset should be given to the original authors. +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 10 | +| **Num Reps:** | 4 Reps (Train), 4 Reps x 15 Positions | +| **Classes:** |
  • Mapping is Uncertain
| +| **Device:** | DelsysTrigno | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['RadmandLP']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/LimbPosition + +**References:** +``` +@INPROCEEDINGS{radmand_lp, + author={Radmand, A. and Scheme, E. and Englehart, K.}, + booktitle={2014 36th Annual International Conference of the IEEE Engineering in Medicine and Biology Society}, + title={A characterization of the effect of limb position on EMG features to guide the development of effective prosthetic control schemes}, + year={2014}, + volume={}, + number={}, + pages={662-667}, + keywords={}, + doi={10.1109/EMBC.2014.6943678}} +``` + +
+
+ + +
+TMRShirleyRyanAbilityLab + +
+ +**Dataset Description:** +6 subjects, 8 reps, 24 motions, pre/post intervention. + +| Attribute | Description | +|-------------------|------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 6 | +| **Num Reps:** | 8 reps per motion (pre/post intervention) | +| **Classes:** |
  • 0 - Hand Open
  • 1 - Key Grip
  • 2 - Power Grip
  • 3 - Fine Pinch Opened
  • 4 - Fine Pinch Closed
  • 5 - Tripod Opened
  • 6 - Tripod Closed
  • 7 - Tool
  • 8 - Hook
  • 9 - Index Point
  • 10 - Thumb Flexion
  • 11 - Thumb Extension
  • 12 - Thumb Abduction
  • 13 - Thumb Adduction
  • 14 - Index Flexion
  • 15 - Ring Flexion
  • 16 - Pinky Flexion
  • 17 - Wrist Supination
  • 18 - Wrist Pronation
  • 19 - Wrist Flexion
  • 20 - Wrist Extension
  • 21 - Radial Deviation
  • 22 - Ulnar Deviation
  • 23 - No Motion
| +| **Device:** | Ag/AgCl | +| **Sampling Rates:** | 1000 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list()['TMRShirleyRyanAbilityLab']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/TMR_ShirleyRyanAbilityLab + +**References:** +``` +@article{tmr, + title={Myoelectric prosthesis hand grasp control following targeted muscle reinnervation in individuals with transradial amputation}, + author={Simon, Ann M and Turner, Kristi L and Miller, Laura A and Dumanian, Gregory A and Potter, Benjamin K and Beachler, Mark D and Hargrove, Levi J and Kuiken, Todd A}, + journal={PloS one}, + volume={18}, + number={1}, + pages={e0280210}, + year={2023}, + publisher={Public Library of Science San Francisco, CA USA} +} +``` + +
+
+ + +## Regression + + + +
+OneSubjectEmaGEr + +
+ +**Dataset Description:** +Simple one subject regression dataset. + +| Attribute | Description | +| ------------------ | ----------- | +| **Num Subjects:** | 1 | +| **Num Reps:** | 5 Reps | +| **Classes:** |
  • 0: Hand Close (-) / Hand Open (+)
  • Pronation (-) / Supination (+)
| +| **Device:** | EmaGEr | +| **Sampling Rates:** | 1010 Hz | +| **Auto Download:** | True | + + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['OneSubjectMyo']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/OneSubjectMyoDataset + +**References:** +``` +@ARTICLE{libemg, + author={Eddy, Ethan and Campbell, Evan and Phinyomark, Angkoon and Bateman, Scott and Scheme, Erik}, + journal={IEEE Access}, + title={LibEMG: An Open Source Library to Facilitate the Exploration of Myoelectric Control}, + year={2023}, + volume={11}, + number={}, + pages={87380-87397}, + doi={10.1109/ACCESS.2023.3304544}} +``` +
-
+
+ + + +
+EMG2POSE + +
+ +**Dataset Description:** +A large dataset from ctrl-labs (Meta) for joint angle estimation. Note that not all subjects have all stages. + +| Attribute | Description | +|-------------------|--------------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 193 | +| **Num Reps:** | N/A | +| **Classes:** |
  • FingerPinches1 - AllFingerPinchesThumbSwipeThumbRotate
  • Object1 - CoffeePanicPete
  • Counting1 - CountingUpDownFaceSideAway
  • Counting2 - CountingUpDownFingerWigglingSpreading
  • DoorknobFingerGraspFistGrab - DoorknobFingerGraspFistGrab
  • Throwing - FastPongFronthandBackhandThrowing
  • Abduction - FingerAbductionSeries
  • FingerFreeform - FingerFreeform
  • FingerPinches2 - FingerPinchesSingleFingerPinchesMultiple
  • HandHandInteractions - FingerTouchPalmClapmrburns
  • Wiggling1 - FingerWigglingSpreading
  • Punch - GraspPunchCloseFar
  • Gesture1 - HandClawGraspFlicks
  • StaticHands - HandDeskSeparateClaspedChest
  • FingerPinches3 - HandOverHandAllFingerPinchesThumbSwipeThumbRotate
  • Wiggling2 - HandOverHandCountingUpDownFingerWigglingSpreading
  • Unconstrained - unconstrained
  • Gesture2 - HookEmHornsOKScissors
  • FingerPinches4 - IndexPinchesMiddlePinchesThumbswipes
  • Pointing - IndividualFingerPointingSnap
  • Freestyle1 - OneHandedFreeStyle
  • Object2 - PlayBlocksChess
  • Draw - PokeDrawPinchRotateclosefar
  • Poke - PokePinchCloseFar
  • Gesture3 - ShakaVulcanPeace
  • ThumbsSwipes - ThumbsSwipesWholeHand
  • ThumbRotations - ThumbsUpDownThumbRotationsCWCCWP
  • Freestyle2 - TwoHandedFreeStyle
  • WristFlex - WristFlexionAbduction
| +| **Device:** | Ctrl Labs Armband | +| **Sampling Rates:** | 2000 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['EMG2POSE']() # Within USer +dataset = get_dataset_list('REGRESSION', cross_user=True)['EMG2POSE']() # Cross User +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://fb-ctrl-oss.s3.amazonaws.com/emg2pose/emg2pose_dataset.tar + +**References:** +``` +@inproceedings{salteremg2pose, + title={emg2pose: A Large and Diverse Benchmark for Surface Electromyographic Hand Pose Estimation}, + author={Salter, Sasha and Warren, Richard and Schlager, Collin and Spurr, Adrian and Han, Shangchen and Bhasin, Rohin and Cai, Yujun and Walkington, Peter and Bolarinwa, Anuoluwapo and Wang, Robert and others}, + booktitle={The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track} +} +``` + +
+
+ + + +
+NinaProDB8 | Attribute | Description | | ------------------ | ----------- | | **Num Subjects:** | 12 | | **Num Reps:** | 20 Training, 2 Testing | -| **Time Per Rep:** | 6-9s | -| **Classes:** | 9 [Nina Pro DB8](http://ninapro.hevs.ch/DB8) | +| **Classes:** | 9 [NinaProDB8](http://ninapro.hevs.ch/DB8) | | **Device:** | Delsys | -| **Sampling Rates:** | EMG (1111 Hz) | +| **Sampling Rates:** | 1111 Hz | +| **Auto Download:** | False | **Using the Dataset:** ```Python -from libemg.datasets import NinaproDB8 -dataset = NinaproDB8("data/NinaDB8") #The loacation of Nina DB8 is downloaded +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['NinaProDB8']() odh = dataset.prepare_data() ``` +**Dataset Location** +Note, this dataset will not be automatically downloaded. To download this dataset, please see [Nina DB8](http://ninapro.hevs.ch/DB8). Simply download the ZIPs and place them in a folder and LibEMG will handle the rest. All credit for this dataset should be given to the original authors. + **References:** ``` -AUTHOR=Krasoulis Agamemnon, Vijayakumar Sethu, Nazarpour Kianoush -TITLE=Effect of User Practice on Prosthetic Finger Control With an Intuitive Myoelectric Decoder -JOURNAL=Frontiers in Neuroscience -VOLUME=13 -YEAR=2019 -URL=https://www.frontiersin.org/articles/10.3389/fnins.2019.00891 -DOI=10.3389/fnins.2019.00891 -ISSN=1662-453X +@article{db8, + title={Effect of user practice on prosthetic finger control with an intuitive myoelectric decoder}, + author={Krasoulis, Agamemnon and Vijayakumar, Sethu and Nazarpour, Kianoush}, + journal={Frontiers in neuroscience}, + volume={13}, + pages={891}, + year={2019}, + publisher={Frontiers Media SA} +} +``` +
+
+ + +
+Hyser1DOF + +
+ +**Dataset Description:** +Hyser 1 DOF dataset. Includes within-DOF finger movements. Ground truth finger forces are recorded for use in finger force regression. +
+ +| Attribute | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 20 | +| **Num Reps:** | 3 | +| **Classes:** |
  • 1 - Thumb
  • 2 - Index
  • 3 - Middle
  • 4 - Ring
  • 5 - Little
| +| **Device:** | OT Bioelettronica Quattrocento | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['Hyser1DOF']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://www.physionet.org/content/hd-semg/2.0.0/ + +**References:** +``` +@ARTICLE{hyser, + author={Jiang, Xinyu and Liu, Xiangyu and Fan, Jiahao and Ye, Xinming and Dai, Chenyun and Clancy, Edward A. and Akay, Metin and Chen, Wei}, + journal={IEEE Transactions on Neural Systems and Rehabilitation Engineering}, + title={Open Access Dataset, Toolbox and Benchmark Processing Results of High-Density Surface Electromyogram Recordings}, + year={2021}, + volume={29}, + number={}, + pages={1035-1046}, + doi={10.1109/TNSRE.2021.3082551}} +``` + +
+
+ + +
+HyserNDOF + +
+ +**Dataset Description:** +Hyser N DOF dataset. Includes combined finger movements. Ground truth finger forces are recorded for use in finger force regression. +
+ +| Attribute | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 20 | +| **Num Reps:** | 2 | +| **Classes:** |
  • 1 - Thumb + Index
  • 2 - Thumb + Middle
  • 3 - Thumb + Ring
  • 4 - Thumb + Little
  • 5 - Index + Middle
  • 6 - Thumb + Index + Middle
  • 7 - Index + Middle + Ring
  • 8 - Middle + Ring + Little
  • 9 - Index + Middle + Ring + Little
  • 10 - All Fingers
  • 11 - Thumb + Index (Opposing)
  • 12 - Thumb + Middle (Opposing)
  • 13 - Thumb + Ring (Opposing)
  • 14 - Thumb + Little (Opposing)
  • 15 - Index + Middle (Opposing)
| +| **Device:** | OT Bioelettronica Quattrocento | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['HyserNDOF']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://www.physionet.org/content/hd-semg/2.0.0/ + +**References:** +``` +@ARTICLE{hyser, + author={Jiang, Xinyu and Liu, Xiangyu and Fan, Jiahao and Ye, Xinming and Dai, Chenyun and Clancy, Edward A. and Akay, Metin and Chen, Wei}, + journal={IEEE Transactions on Neural Systems and Rehabilitation Engineering}, + title={Open Access Dataset, Toolbox and Benchmark Processing Results of High-Density Surface Electromyogram Recordings}, + year={2021}, + volume={29}, + number={}, + pages={1035-1046}, + doi={10.1109/TNSRE.2021.3082551}} ``` --------------

+ +
+HyserRandom + +
+ +**Dataset Description:** +Hyser random dataset. Includes random motions performed by users. Ground truth finger forces are recorded for use in finger force regression. +
+ +| Attribute | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 19 | +| **Num Reps:** | 5 | +| **Classes:** | Random | +| **Device:** | OT Bioelettronica Quattrocento | +| **Sampling Rates:** | 2048 Hz | +| **Auto Download:** | False | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['HyserRandom']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://www.physionet.org/content/hd-semg/2.0.0/ + +**References:** +``` +@ARTICLE{hyser, + author={Jiang, Xinyu and Liu, Xiangyu and Fan, Jiahao and Ye, Xinming and Dai, Chenyun and Clancy, Edward A. and Akay, Metin and Chen, Wei}, + journal={IEEE Transactions on Neural Systems and Rehabilitation Engineering}, + title={Open Access Dataset, Toolbox and Benchmark Processing Results of High-Density Surface Electromyogram Recordings}, + year={2021}, + volume={29}, + number={}, + pages={1035-1046}, + doi={10.1109/TNSRE.2021.3082551}} +``` + +
+
+ + + +
+UserCompliance + +
+ +**Dataset Description:** +Regression dataset used for investigation into user compliance during mimic training. +
+ +| Attribute | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------| +| **Num Subjects:** | 6 | +| **Num Reps:** | 5 | +| **Classes:** |
  • 0 - Hand Close (-) / Hand Open (+)
  • 1 - Pronation (-) / Supination (+)
| +| **Device:** | EMaGer | +| **Sampling Rates:** | 1010 Hz | +| **Auto Download:** | True | + +**Using the Dataset:** +```Python +from libemg.datasets import * +dataset = get_dataset_list('REGRESSION')['UserCompliance']() +odh = dataset.prepare_data() +``` + +**Dataset Location** +https://github.com/LibEMG/UserComplianceDataset + +**References:** +``` +@inproceedings{morrell2024exploring, + title={Exploring user compliance in the training of regression-based myoelectric control}, + author={Morrell, Christian and Campbell, Evan and Scheme, Erik}, + booktitle={Myoelectric Controls Symposium}, + year={2024} +} +``` + +
+
+ + # Offline Data Handler One overhead for most EMG projects is interfacing with a particular dataset since they often have different folder and file structures. LibEMG provides a means to quickly interface datasets so you can focus on using them with minimal setup time. Assuming the files in the dataset are well formatted (i.e., they include all metadata such as rep, class, and subject) and are either .csv or .txt files, the OfflineDataHandler does all accumulation and processing. To do this, LibEMG relies on regular expressions to define a dataset's file and folder structure. These expressions can be used to create a dictionary that is passed to the OfflineDataHandler. Once the data handler has collected all the files that satisfy the regexes, the dataset can be sliced using the metadata tags (e.g., by rep, subjects, classes, etc.). After extracting the data it is ready to be passed through the rest of the pipeline. The following code snippet exemplifies how to process a dataset with testing/training, rep, and class metadata. In this case the file format is: `dataset/train/R_1_C_1_EMG.csv` where R is the rep and C is the class. diff --git a/docs/source/documentation/introduction/core_modules.png b/docs/source/documentation/introduction/core_modules.png index 814f78dc..ac9a4b3a 100644 Binary files a/docs/source/documentation/introduction/core_modules.png and b/docs/source/documentation/introduction/core_modules.png differ diff --git a/docs/source/documentation/introduction/intro_doc.md b/docs/source/documentation/introduction/intro_doc.md index f9384ff0..47623434 100644 --- a/docs/source/documentation/introduction/intro_doc.md +++ b/docs/source/documentation/introduction/intro_doc.md @@ -27,9 +27,9 @@ As EMG signals are stochastic, they do not provide adequate descriptive informat Feature selection is an important design consideration when developing EMG-based control systems, as features can drastically influence performance. Often, however, it is difficult to know what features to select for a particular problem. **This module provides a means to extract the most relevant features for a specific problem.** This module is optional and is primarily a tool to explore the robustness of certain features and groups using a variety of metrics. These are the techniques used by previous work to suggest predefined feature groups. -

Classification Module

+

Prediction Module

-Classification uses machine learning models to predict user intent from EMG data (i.e., features) generated during contractions. **This module enables online (real-time) and offline (after-the-fact) classification.** Currently, it is limited to continuous control schemes where a model continuously predicts user intent based on segments (i.e., windows) of data. +The prediction module uses machine learning models (classification or regression) to predict user intent from EMG data (i.e., features) generated during contractions. **This module enables online (real-time) and offline (after-the-fact) predictions.** Currently, it is limited to continuous control schemes where a model continuously predicts user intent based on segments (i.e., windows) of data.

Evaluation Module

diff --git a/docs/source/documentation/prediction/classification_doc.md b/docs/source/documentation/prediction/classification_doc.md new file mode 100644 index 00000000..da48865c --- /dev/null +++ b/docs/source/documentation/prediction/classification_doc.md @@ -0,0 +1,75 @@ +# Classifiers + +Below is a list of the classifiers that can be instatiated by passing in a string to the `EMGClassifier`. For other classifiers, pass in a custom model that has the `fit`, `predict`, and `predict_proba` methods. + +## Linear Discriminant Analysis (LDA) + +A linear classifier that uses common covariances for all classes and assumes a normal distribution. +```Python +classifier = EMGClassifier('LDA') +classifier.fit(data_set) +``` +Check out the LDA docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html) + +## K-Nearest Neighbour (KNN) + +Discriminates between inputs using the K closest samples in feature space. The implemented version in the library defaults to k = 5. A commonly used classifier for EMG-based recognition. + +```Python +params = {'n_neighbors': 5} # Optional +classifier = EMGClassifier('KNN') +classifier.fit(data_set, parameters=params) +``` +Check out the KNN docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html) + +## Support Vector Machines (SVM) + +A hyperplane that maximizes the distance between classes is used as the boundary for recognition. A commonly used classifier for EMG-based recognition. +```Python +classifier = EMGClassifier('SVM') +classifier.fit(data_set) +``` +Check out the SVM docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html) + +## Artificial Neural Networks (MLP) + +A deep learning technique that uses human-like "neurons" to model data to help discriminate between inputs. Especially for this model, we **highly** recommend you create your own. +```Python +classifier = EMGClassifier('MLP') +classifier.fit(data_set) +``` +Check out the MLP docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html) + +## Random Forest (RF) + +Uses a combination of decision trees to discriminate between inputs. +```Python +classifier = EMGClassifier('RF') +classifier.fit(data_set) +``` +Check out the RF docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html) + +## Quadratic Discriminant Analysis (QDA) + +A quadratic classifier that uses class-specific covariances and assumes normally distributed classes. +```Python +classifier = EMGClassifier('QDA') +classifier.fit(data_set) +``` +Check out the QDA docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html) + +## Gaussian Naive Bayes (NB) + +Assumes independence of all input features and normally distributed classes. +```Python +classifier = EMGClassifier('NB') +classifier.fit(data_set) +``` +Check out the NB docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html) + + diff --git a/docs/source/documentation/classification/decision_stream.png b/docs/source/documentation/prediction/decision_stream.png similarity index 100% rename from docs/source/documentation/classification/decision_stream.png rename to docs/source/documentation/prediction/decision_stream.png diff --git a/docs/source/documentation/classification/post_processing_doc.md b/docs/source/documentation/prediction/post_processing_doc.md similarity index 71% rename from docs/source/documentation/classification/post_processing_doc.md rename to docs/source/documentation/prediction/post_processing_doc.md index 97217ddf..30a9637b 100644 --- a/docs/source/documentation/classification/post_processing_doc.md +++ b/docs/source/documentation/prediction/post_processing_doc.md @@ -1,6 +1,8 @@ # Post-Processing -## Rejection +## Classification + +### Rejection Classifier outputs are overridden to a default or inactive state when the output decision is uncertain. This concept stems from the notion that it is often better (less costly) to incorrectly do nothing than it is to erroneously activate an output. - **Confidence [1]:** Rejects based on a predefined **confidence threshold** (between 0-1). If predicted probability is less than the confidence threshold, the decision is rejected. Figure 1 exemplifies rejection using an SVM classifier with a threshold of 0.8. @@ -9,7 +11,7 @@ Classifier outputs are overridden to a default or inactive state when the output classifier.add_rejection(threshold=0.9) ``` -## Majority Voting [2,3] +### Majority Voting [2,3] Overrides the current output with the label corresponding to the class that occurred most frequently over the past $N$ decisions. As a form of simple low-pass filter, this introduces a delay into the system but reduces the likelihood of spurious false activations. Figure 1 exemplifies applying a majority vote of 5 samples to a decision stream. ```Python @@ -17,7 +19,7 @@ Overrides the current output with the label corresponding to the class that occu classifier.add_majority_vote(num_samples=10) ``` -## Velocity Control [4] +### Velocity Control [4] Outputs an associated *velocity* with each prediction that estimates the level of muscular contractions (normalized by the particular class). This means that within the same contraction, users can contract harder or lighter to control the velocity of a device. Note that ramp contractions should be accumulated during the training phase. ```Python @@ -26,11 +28,27 @@ classifier.add_velocity(train_windows, train_labels) ``` Figure 1 shows the decision stream (i.e., the predictions over time) of a classifier with no post-processing, rejection, and majority voting. In this example, the shaded regions show the ground truth label, whereas the colour of each point represents the predicted label. All black points indicate predictions that have been rejected. - ![alt text](decision_stream.png)

Figure 1: Decision Stream of No Post-Processing, Rejection, and Majority Voting. This can be created using the .visualize() method call.

+## Regression + +### Deadband [5] + +Modifies a regressor's output based on whether the prediction's magnitude is above a certain threshold. Any value whose magnitude is less than the defined threshold is output as 0. This preprocessing technique is typically used to combat drift at lower amplitudes. + +```Python +# Add deadband to regressor (i.e., values with magnitude < 0.25 will be output as 0) +regressor.add_deadband(0.25) +``` + +Figure 2 shows the decision stream of a regressor with no post-processing and deadband thresholding. In this visualization, the shaded blue regions are the ground truth and each black dot corresponds to a single prediction. Predictions for each degree of freedom (DOF) are plotted on separate subplots for visual clarity. + +![alt text](regression_post_processing.png) +

Figure 2: Decision Stream of Regressor with No Post-Processing and Deadband Thresholding. This can be created using the regressor's .visualize() method call.

+ ## References + [1] E. J. Scheme, B. S. Hudgins and K. B. Englehart, "Confidence-Based Rejection for Improved Pattern Recognition Myoelectric Control," in IEEE Transactions on Biomedical Engineering, vol. 60, no. 6, pp. 1563-1570, June 2013, doi: 10.1109/TBME.2013.2238939. @@ -43,5 +61,8 @@ Wahid MF, Tafreshi R, Langari R. A Multi-Window Majority Voting Strategy to Impr [4] E. Scheme, B. Lock, L. Hargrove, W. Hill, U. Kuruganti and K. Englehart, "Motion Normalized Proportional Control for Improved Pattern Recognition-Based Myoelectric Control," in IEEE Transactions on Neural Systems and Rehabilitation Engineering, vol. 22, no. 1, pp. 149-157, Jan. 2014, doi: 10.1109/TNSRE.2013.2247421. +[5] +A. Ameri, E. N. Kamavuako, E. J. Scheme, K. B. Englehart, and P. A. Parker, “Support vector regression for improved real-time, simultaneous myoelectric control,” IEEE Transactions on Neural Systems and Rehabilitation Engineering, vol. 22, no. 6, pp. 1198–1209, Nov. 2014, doi: 10.1109/TNSRE.2014.2323576. + [Sklearn] -Fabian Pedregosa, Gaël Varoquaux, Alexandre Gramfort, Vincent Michel, Bertrand Thirion, Olivier Grisel, Mathieu Blondel, Peter Prettenhofer, Ron Weiss, Vincent Dubourg, Jake Vanderplas, Alexandre Passos, David Cournapeau, Matthieu Brucher, Matthieu Perrot, and Édouard Duchesnay. 2011. Scikit-learn: Machine Learning in Python. J. Mach. Learn. Res. 12, null (2/1/2011), 2825–2830. \ No newline at end of file +Fabian Pedregosa, Gaël Varoquaux, Alexandre Gramfort, Vincent Michel, Bertrand Thirion, Olivier Grisel, Mathieu Blondel, Peter Prettenhofer, Ron Weiss, Vincent Dubourg, Jake Vanderplas, Alexandre Passos, David Cournapeau, Matthieu Brucher, Matthieu Perrot, and Édouard Duchesnay. 2011. Scikit-learn: Machine Learning in Python. J. Mach. Learn. Res. 12, null (2/1/2011), 2825–2830. diff --git a/docs/source/documentation/classification/classification.rst b/docs/source/documentation/prediction/prediction.rst similarity index 54% rename from docs/source/documentation/classification/classification.rst rename to docs/source/documentation/prediction/prediction.rst index 751e673f..f15dfc84 100644 --- a/docs/source/documentation/classification/classification.rst +++ b/docs/source/documentation/prediction/prediction.rst @@ -1,7 +1,13 @@ -Classification +EMG Prediction ------------------------------ +.. include:: predictors.md + :parser: myst_parser.sphinx_ + .. include:: classification_doc.md :parser: myst_parser.sphinx_ +.. include:: regression_doc.md + :parser: myst_parser.sphinx_ + .. include:: post_processing_doc.md :parser: myst_parser.sphinx_ \ No newline at end of file diff --git a/docs/source/documentation/prediction/predictors.md b/docs/source/documentation/prediction/predictors.md new file mode 100644 index 00000000..e9e1246e --- /dev/null +++ b/docs/source/documentation/prediction/predictors.md @@ -0,0 +1,40 @@ +# Models + +After recording, processing, and extracting features from a window of EMG data, it is passed to a machine learning algorithm for prediction. These control systems have evolved in the prosthetics community for continuously predicting muscular contractions for enabling prosthesis control. Therefore, they are primarily limited to recognizing static contractions (e.g., hand open/close and wrist flexion/extension) as they have no temporal awareness. Currently, this is the form of recognition supported by LibEMG and is an initial step to explore EMG as an interaction opportunity for general-purpose use. This section highlights the machine-learning strategies that are part of `LibEMG`'s pipeline. + +There are two types of models supported in `LibEMG`: classifiers and regressors. Classifiers output a discrete motion class for each window, whereas regressors output a continuous prediction along a degree of freedom. For both classifiers and regressors, `LibEMG` supports statistical models as well as deep learning models. Additionally, a number of post-processing methods (i.e., techniques to improve performance after prediction) are supported for all models. + +## Statistical Models + +The statistical models (i.e., traditional machine learning methods) implemented leverage the sklearn package. For most cases, the "base" models use the default options, meaning that the pre-defined models are not necessarily optimal. However, the `parameters` attribute can be used when initializing the models to pass in additional sklearn parameters in a dictionary. For example, looking at the `RandomForestClassifier` docs on sklearn: + +![Random Forest](random_forest.png) + +A classifier with any of those parameters using the `parameters` attribute. For example: + +```Python +parameters = { + 'n_estimators': 99, + 'max_depth': 20, + 'random_state': 5, + 'max_leaf_nodes': 10 +} +classifier.fit(data_set, parameters=parameters) +``` + +The same process can be done using the `RandomForestRegressor` from sklearn and an `EMGRegressor`. Please reference the [sklearn docs](https://scikit-learn.org/stable/) for parameter options for each model. + +Additionally, custom models can be created. Any custom classifier should be modeled after the `sklearn` classifiers and must have the `fit`, `predict`, and `predict_proba` functions to work correctly. Any custom regressor should be modeled after the `sklearn` regressors and must have the `fit` and `predict` methods. + +```Python +from sklearn.ensemble import RandomForestClassifier +from libemg.predictor import EMGClassifier + +rf_custom_classifier = RandomForestClassifier(max_depth=5, random_state=0) +classifier = EMGClassifier(rf_custom_classifier) +classifier.fit(data_set) +``` + +## Deep Learning (Pytorch) + +Another available option is to use [pytorch](https://pytorch.org/) models (i.e., a library for deep learning) to train the model, although this involves making some custom code for preparing the dataset and the deep learning model. For a guide on how to use deep learning models, consult the deep learning example. The same methods are expected to be implemented for both deep and statistical classifiers/regressors. diff --git a/docs/source/documentation/classification/random_forest.png b/docs/source/documentation/prediction/random_forest.png similarity index 100% rename from docs/source/documentation/classification/random_forest.png rename to docs/source/documentation/prediction/random_forest.png diff --git a/docs/source/documentation/prediction/regression_doc.md b/docs/source/documentation/prediction/regression_doc.md new file mode 100644 index 00000000..3ffe4da5 --- /dev/null +++ b/docs/source/documentation/prediction/regression_doc.md @@ -0,0 +1,58 @@ +# Regressors + +Below is a list of the regressors that can be instatiated by passing in a string to the `EMGRegressor`. For other regressors, pass in a custom model that has the `fit` and `predict` methods. + +## Linear Regression (LR) + +A linear regressor that aims to minimize the residual sum of squares between the predicted values and the true targets. + +```Python +regressor = EMGRegressor('LR') +regressor.fit(data_set) +``` + +Check out the LR docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html) + +## Support Vector Machines (SVM) + +A regressor that uses a kernel trick to find the hyperplane that best fits the data. + +```Python +regressor = EMGRegressor('SVM') +regressor.fit(data_set) +``` + +Check out the SVM docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html) + +## Artificial Neural Networks (MLP) + +A deep learning technique that uses human-like "neurons" to model data to help discriminate between inputs. Especially for this model, we **highly** recommend you create your own. + +```Python +regressor = EMGRegressor('MLP') +regressor.fit(data_set) +``` + +Check out the MLP docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPRegressor.html) + +## Random Forest (RF) + +Uses a combination of decision trees to discriminate between inputs. + +```Python +regressor = EMGRegressor('RF') +regressor.fit(data_set) +``` + +Check out the RF docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestRegressor.html) + +## Gradient Boosting (GB) + +Additive model that fits a regression tree on the negative gradient of the loss function. + +```Python +regressor = EMGRegressor('GB') +regressor.fit(data_set) +``` + +Check out the GB docs [here.](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) diff --git a/docs/source/documentation/prediction/regression_post_processing.png b/docs/source/documentation/prediction/regression_post_processing.png new file mode 100644 index 00000000..8286a2df Binary files /dev/null and b/docs/source/documentation/prediction/regression_post_processing.png differ diff --git a/docs/source/documentation/supported_hardware/supported_hardware_doc.md b/docs/source/documentation/supported_hardware/supported_hardware_doc.md index 77c338d8..84fdee71 100644 --- a/docs/source/documentation/supported_hardware/supported_hardware_doc.md +++ b/docs/source/documentation/supported_hardware/supported_hardware_doc.md @@ -23,7 +23,7 @@ By default, LibEMG supports several hardware devices (shown in Table 1). - The [**Delsys**](https://delsys.com/) is a commercially available system primarily used for medical applications due to its relatively high cost. - The [**SIFI Cuff**](https://sifilabs.com/) is a pre-released device that will soon be commercially available. Compared to the Myo armband, this device has a much higher sampling rate (~2000 Hz). - The [**Oymotion Cuff**](http://www.oymotion.com/en/product32/149) is a commercial device that samples EMG at 1000 Hz (8 bits) or 500 Hz (12 bits). -- The [**OTBioelettronica**](https://otbioelettronica.it/hardware/) devices are a set of commercially available HDEMG systems. + If selecting EMG hardware for real-time use, wireless armbands that sample above 500 Hz are preferred. Additionally, future iterations of LibEMG will include Inertial Measurement Unit (IMU) support. As such, devices should have IMUs to enable more interaction opportunities. @@ -33,9 +33,9 @@ If selecting EMG hardware for real-time use, wireless armbands that sample above | Delsys | `delsys_streamer()` or `delsys_API_streamer()` |
![](devices/delsys_trigno.png)
| | SIFI Cuff | `sifi_streamer()` |
![](devices/sifi_cuff.png)
| | Oymotion | `oymotion_streamer()`|
![](devices/oymotion.png)
| -| Muovi | `otb_muovi_streamer()`|
![](devices/muovi.png)
| +

Table 1: The list of all implemented streamers.

diff --git a/docs/source/documentation/visualization/heatmap.gif b/docs/source/documentation/visualization/heatmap.gif new file mode 100644 index 00000000..27079e0e Binary files /dev/null and b/docs/source/documentation/visualization/heatmap.gif differ diff --git a/docs/source/documentation/visualization/regressor.png b/docs/source/documentation/visualization/regressor.png new file mode 100644 index 00000000..4880becd Binary files /dev/null and b/docs/source/documentation/visualization/regressor.png differ diff --git a/docs/source/documentation/visualization/visualization_doc.md b/docs/source/documentation/visualization/visualization_doc.md index 0f7d2061..4c1782e5 100644 --- a/docs/source/documentation/visualization/visualization_doc.md +++ b/docs/source/documentation/visualization/visualization_doc.md @@ -23,16 +23,30 @@ The EMG classifier contains a visualization tool for viewing the decisions strea ![alt text](decision_stream.png)

Figure 2: The decision stream of a classifier.

+# EMG Regressor + +The EMG regressor also contains a visualization tool for viewing the model's decision stream. Similar to the classifier, you can view the decision stream using the `visualize` method. + +![alt text](regressor.png) +

Figure 3: The decision stream of a regressor.

+ # Feature Extractor -The Feature Extrator and Online Data Handler contain a visualization tool for viewing the PCA feature space. This can be done using the `visualize_feature_space` function. If this function is run on an online data handler, a live PCA feature space will be shown (see Figure 3). +The Feature Extrator and Online Data Handler contain a visualization tool for viewing the PCA feature space. This can be done using the `visualize_feature_space` function. If this function is run on an online data handler, a live PCA feature space will be shown (see Figure 4). |
Offline
|
Online (Live)
| | ------------- | ------------- | | ![alt text](feature_space.png) | ![alt text](feature_space.gif) | -

Figure 3: The PCA feature space of a set of data.

+

Figure 4: The PCA feature space of a set of data.

# Filtering -The filtering module has a `visualize_effect` function that demonstrates the effect of a filter on a set of data in the time and frequency domain. +The filtering module has a `visualize_effect` function that demonstrates the effect of a filter on a set of data in the time and frequency domain. ![](filtering_1.png) -

Figure 4: Data before and after filtering in the time and frequency domain.

\ No newline at end of file +

Figure 5: Data before and after filtering in the time and frequency domain.

+ +# Heatmap + +Viewing EMG as a time series may not be appropriate for high-density EMG systems. `LibEMG` offers a live heatmap visualization using the `visualize_heatmap` method. Heatmaps of multiple features can be visualized in real-time to show spatial information (only features that produce a single value per window are supported). + +![alt text](heatmap.gif) +

Figure 6: Real-time heatmap visualization.

diff --git a/docs/source/emg_toolbox.rst b/docs/source/emg_toolbox.rst index 7d3c6fe3..93a56993 100644 --- a/docs/source/emg_toolbox.rst +++ b/docs/source/emg_toolbox.rst @@ -3,6 +3,12 @@ Data Handler .. automodule:: libemg.data_handler :members: +Datasets +------------------------------ +.. automodule:: libemg.datasets + :members: + + Filtering ------------------------------ .. automodule:: libemg.filtering @@ -18,11 +24,19 @@ Feature Selection .. automodule:: libemg.feature_selector :members: -Classification +EMG Prediction ------------------------------ .. automodule:: libemg.emg_predictor :members: +Adaptation +------------------------------ +.. automodule:: libemg.adaptation.managers + :members: + +.. automodule:: libemg.adaptation.memory + :members: + Offline Evaluation Metrics ------------------------------ .. automodule:: libemg.offline_metrics diff --git a/docs/source/examples/fitts_example/fitts.md b/docs/source/examples/fitts_example/fitts.md index 6da5927c..d76dae3d 100644 --- a/docs/source/examples/fitts_example/fitts.md +++ b/docs/source/examples/fitts_example/fitts.md @@ -9,74 +9,139 @@ } -For EMG-based control systems, it has been shown that the offline performance of a system (i.e., classification accuracy) does not necessarily correlate to online usability. In this example, we introduce an Iso Fitts test for assessing the online performance of continuous EMG-based control systems. While this test evaluates systems with 2DOFs that leverage continuous constant control, it could be extended to more complicated systems such as proportional control with discrete inputs. +For EMG-based control systems, it has been shown that the offline performance of a system (i.e., classification accuracy, mean absolute error) does not necessarily correlate to online usability. In this example, we introduce an Iso Fitts test for assessing the online performance of continuous EMG-based control systems. +Different types of models, such as regressors and classifiers, cannot be easily compared offline since different metrics are calculated. Online tests allow us to compare these distinct model types and assess a model's ability to perform a task with a user in the loop. # Methods -This example acts as a mini experiment that you can try out on yourself or a friend where the offline and online performance of four popular classifiers (**LDA, SVM, RF,** and **KNN (k=5**)) are compared. +This example acts as a mini experiment that you can try out on yourself or a friend where the offline and online performance of four popular classifiers (**LDA, SVM, RF,** and **KNN (k=5**)) and two regressors (**LR and SVM**) are compared. The steps of this 'mini experiment' are as follows: -1. **Accumulate 5 repetitions of five contractions (no movement, flexion, extension, hand open, and hand closed).** These classes correspond to movement in the isofitts task (do nothing, and move left, right, up, and down). -
- - - -
-2. **Train and evaluate four classifiers in an offline setting (LDA, SVM, KNN (k=5), and RF).** For this step, the first three reps are used for training and the last two for testing. +1. **Accumulate 3 repetitions of five contractions (no movement, flexion, extension, hand open, and hand closed).** These classes correspond to movement in the isofitts task (do nothing, and move left, right, up, and down). + + + + + + + + + + +
Image 1Image 2
Image 3Image 4
+ +2. **Train and evaluate four classifiers in an offline setting (LDA, SVM, KNN (k=5), and RF).** For this step, the first 2 reps are used for training and the last for testing. 3. **Perform an Iso Fitts test to evaluate the online usability of each classifier trained in step 2.** These fitts law tests are useful for computing throughput, overshoots, and efficiency. Ultimately, these metrics provide an indication of the online usability of a model. The Iso Fitts test is useful for myoelectric control systems as it requires changes in degrees of freedom to complete sucessfully. - +4. **Repeat steps 1-3 using regressors instead of classifiers.** Select 'regression' from the radio buttons and redo data collection. You will now be shown a video of a point moving through a cartesian plane, which indicates the position along each DOF. Follow the point in real-time to provide the regressor with continuously-labelled training data (as opposed to classes in classification). This video will be repeated 3 times (i.e., 3 repetitions). Note that you can now perform simultaneous contractions (i.e., move the cursor along the diagonal) when using a regressor. -**Note:** We have made this example to work with the `Myo Armband`. However, it can easily be used for any hardware by simply switching the `streamer`, `WINDOW_SIZE`, and `INCREMENT`. +**Note:** We have made this example to work with the `Myo Armband`. However, it can easily be used for any hardware by simply switching the `streamer`, `WINDOW_SIZE`, and `WINDOW_INCREMENT`. # Menu ```Python from libemg.streamers import myo_streamer from libemg.gui import GUI -from libemg.data_handler import OnlineDataHandler, OfflineDataHandler, RegexFilter -from libemg.utils import make_regex +from libemg.data_handler import OnlineDataHandler, OfflineDataHandler, RegexFilter, FilePackager from libemg.feature_extractor import FeatureExtractor -from libemg.emg_predictor import OnlineEMGClassifier, EMGClassifier +from libemg.emg_predictor import OnlineEMGClassifier, EMGClassifier, EMGRegressor, OnlineEMGRegressor +from libemg.environments.isofitts import IsoFitts +from libemg.environments.controllers import ClassifierController, RegressorController +from libemg.animator import ScatterPlotAnimator ``` -Similarly to previous examples, we decided to create a simple menu to (1) leverage the training module and (2) enable the use of different classifiers. To do this, we have included two buttons in `menu.py`. When the "accumulate training data button" is clicked, we leverage the training UI module. For this example, we want five reps (3 training - 2 testing), and we point it to the "classes" folder as it contains images for each class. +Similarly to previous examples, we decided to create a simple menu to (1) leverage the training module and (2) enable the use of different models. To do this, we have included two buttons in `menu.py`. When the "accumulate training data button" is clicked, we leverage the training UI module. For this example, we want 3 reps (2 training - 1 testing), and we point it to the "images" folder as it contains images for each class. To instead evaluate a regressor, simply select the 'Regression' radio button and collect the required data. Note that launching training for regression will create a `collection.mp4` file using the `Animator` class. ```Python def launch_training(self): self.window.destroy() - training_ui = GUI(self.odh, width=700, height=700, gesture_height=300, gesture_width=300) + if self.regression_selected(): + args = {'media_folder': 'animation/', 'data_folder': Path('data', 'regression').absolute().as_posix(), 'rep_time': 50} + else: + args = {'media_folder': 'images/', 'data_folder': Path('data', 'classification').absolute().as_posix()} + training_ui = GUI(self.odh, args=args, width=700, height=700, gesture_height=300, gesture_width=300) training_ui.download_gestures([1,2,3,4,5], "images/") + self.create_animation() training_ui.start_gui() + self.initialize_ui() + +def create_animation(self): + output_filepath = Path('animation', 'collection.mp4').absolute() + if not self.regression_selected() or output_filepath.exists(): + return + + print('Creating regression training animation...') + period = 2 # period of sinusoid (seconds) + cycles = 10 + rest_time = 5 # (seconds) + fps = 24 + + coordinates = [] + total_duration = int(cycles * period + rest_time) + t = np.linspace(0, total_duration - rest_time, fps * (total_duration - rest_time)) + coordinates.append(np.sin(2 * np.pi * (1 / period) * t)) # add sinusoids + coordinates.append(np.zeros(fps * rest_time)) # add rest time + + # Convert into 2D (N x M) array with isolated sinusoids per DOF + coordinates = np.expand_dims(np.concatenate(coordinates, axis=0), axis=1) + dof1 = np.hstack((coordinates, np.zeros_like(coordinates))) + dof2 = np.hstack((np.zeros_like(coordinates), coordinates)) + coordinates = np.vstack((dof1, dof2)) + + axis_images = { + 'N': PILImage.open(Path('images', 'Hand_Open.png')), + 'S': PILImage.open(Path('images', 'Hand_Close.png')), + 'E': PILImage.open(Path('images', 'Wrist_Extension.png')), + 'W': PILImage.open(Path('images', 'Wrist_Flexion.png')) + } + animator = ScatterPlotAnimator(output_filepath=output_filepath.as_posix(), show_direction=True, show_countdown=True, axis_images=axis_images) + animator.save_plot_video(coordinates, title='Regression Training', save_coordinates=True, verbose=True) ``` -The next button option involves starting the Iso Fitts task. This occurs after the training data has been recorded. Note that in this step we create the online classifier and start the Fitts law test. We opted for 8 circles, but this can be varied easily with the constructor. +The next button option involves starting the Iso Fitts task. This occurs after the training data has been recorded. We opted for 8 circles in this task, but this can be varied easily with the constructor. Note that in this step we create the online model and start the Fitts law test, so please ensure that you have changed the text box to select the desired model type. If you are performing classification, recommended options are 'LDA', 'SVM', 'KNN', and 'RF'. If you are performing regression, recommended options are 'LR' and 'RF'. To see a full list of available options for classifiers and regressors in `LibEMG`, check out the `EMGClassifier` and `EMGRegressor` in the [source code](https://github.com/LibEMG/libemg). ```Python def start_test(self): self.window.destroy() - self.set_up_classifier() - FittsLawTest(num_trials=8, num_circles=8, savefile=self.model_str.get() + ".pkl").run() - # Its important to stop the classifier after the game has ended + self.set_up_model() + if self.regression_selected(): + controller = RegressorController() + save_file = Path('results', self.model_str.get() + '_reg' + ".pkl").absolute().as_posix() + else: + controller = ClassifierController(output_format=self.model.output_format, num_classes=5) + save_file = Path('results', self.model_str.get() + '_clf' + ".pkl").absolute().as_posix() + IsoFitts(controller, num_trials=8, num_circles=8, save_file=save_file).run() + # Its important to stop the model after the game has ended # Otherwise it will continuously run in a seperate process - self.classifier.stop_running() + self.model.stop_running() self.initialize_ui() ``` -Now, let's break this piece of code up. First, let's explore the `self.set_up_classifier()` function call. This step involves parsing the offline training data using the `OfflineDataHandler`. The file format for this example is R_<#>_C_<#>.csv. So to extract the reps the left bound is `R_` and the right bound is `_C_`. Similarly, to extract the classes, the left bound is `C_` and the right bound is `.csv`. Additionally, there are three training reps and five classes. Once we extract all this information, we create the `OfflineDataHandler` and extract the `train_windows` and `train_metadata` variables. +Now, let's break this piece of code up. First, let's explore the `self.set_up_model()` function call. This step involves parsing the offline training data using the `OfflineDataHandler`. The file format for this example is R_<#>_C_<#>.csv. So to extract the reps the left bound is `R_` and the right bound is `_C_`. Similarly, to extract the classes, the left bound is `C_` and the right bound is `.csv`. Additionally, there are three training reps and five classes. For regression, the labels are extracted from a separate text file instead of from the filename. The `collection.txt` file is used to create the labels for each data file via a `FilePackager`. Once we extract all this information, we create the `OfflineDataHandler` and extract the `train_windows` and `train_metadata` variables. ```Python # Step 1: Parse offline training data -dataset_folder = 'data/' -classes_values = ["0","1","2","3","4"] -reps_values = ["0", "1", "2"] -regex_filters = [ - RegexFilter(left_bound = "_C_", right_bound=".csv", values = classes_values, description='classes'), - RegexFilter(left_bound = "R_", right_bound="_C_", values = reps_values, description='reps') -] +if self.regression_selected(): + regex_filters = [ + RegexFilter(left_bound='regression/C_0_R_', right_bound='_emg.csv', values=['0', '1', '2'], description='reps') + ] + metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='animation/', right_bound='.txt', values=['collection'], description='labels'), package_function=lambda x, y: True) + ] + labels_key = 'labels' + metadata_operations = {'labels': 'last_sample'} +else: + regex_filters = [ + RegexFilter(left_bound = "classification/C_", right_bound="_R", values = ["0","1","2","3","4"], description='classes'), + RegexFilter(left_bound = "R_", right_bound="_emg.csv", values = ["0", "1", "2"], description='reps'), + ] + metadata_fetchers = None + labels_key = 'classes' + metadata_operations = None + odh = OfflineDataHandler() -odh.get_data(folder_location=dataset_folder, regex_filters=regex_filters, delimiter=",") -train_windows, train_metadata = odh.parse_windows(WINDOW_SIZE, WINDOW_INCREMENT) +odh.get_data('./', regex_filters, metadata_fetchers=metadata_fetchers, delimiter=",") +train_windows, train_metadata = odh.parse_windows(WINDOW_SIZE, WINDOW_INCREMENT, metadata_operations=metadata_operations) ``` -The next step involves extracting features from the training data. To do this we leverage the `FeatureExtractor` module. In this example, we use the `Low Sampling 4 (LS4)` feature set as it is a robust group for low sampling rate devices such as the Myo. +The next step involves extracting features from the training data. To do this we leverage the `FeatureExtractor` module. In this example, we use the `Low Sampling 4 (LS4)` feature set as it is a robust group for low sampling rate devices such as the Myo. ```Python # Step 2: Extract features from offline data @@ -91,96 +156,99 @@ We then split the training features and labels into a dataset dictionary for the # Step 3: Dataset creation data_set = {} data_set['training_features'] = training_features -data_set['training_labels'] = train_metadata['classes'] +data_set['training_labels'] = train_metadata[labels_key] ``` -Finally, we create the `EMGClassifier` and the `OnlineEMGClassifier` using the default options. Notice that when creating the classifier, we pass in the text from the menu text field. This enables the user to pass in `LDA`, `SVM`, etc. with ease. Once the classifier is created, the `.run()` function is called and predictions begin. +Finally, we create the offline and online models using the default options. Notice that when creating the model, we pass in the text from the menu text field. This enables the user to pass in `LDA`, `SVM`, etc. with ease. Once the model is created, the `.run()` function is called and predictions begin. ```Python -# Step 4: Create the EMG Classifier -o_classifier = EMGClassifier(self.model_str.get()) -o_classifier.fit(feature_dictionary=data_set) - -# Step 5: Create online EMG classifier and start classifying. -self.classifier = OnlineEMGClassifier(o_classifier, WINDOW_SIZE, WINDOW_INCREMENT, self.odh, feature_list) -self.classifier.run(block=False) # block set to false so it will run in a seperate process. +# Step 4: Create the EMG model +model = self.model_str.get() +if self.regression_selected(): + # Regression + emg_model = EMGRegressor(model=model) + emg_model.fit(feature_dictionary=data_set) + self.model = OnlineEMGRegressor(emg_model, WINDOW_SIZE, WINDOW_INCREMENT, self.odh, feature_list) +else: + # Classification + emg_model = EMGClassifier(model=model) + emg_model.fit(feature_dictionary=data_set) + emg_model.add_velocity(train_windows, train_metadata[labels_key]) + self.model = OnlineEMGClassifier(emg_model, WINDOW_SIZE, WINDOW_INCREMENT, self.odh, feature_list) + +# Step 5: Create online EMG model and start predicting. +self.model.run(block=False) # block set to false so it will run in a seperate process. ``` # Fitts Test -To create the Isofitts test, we leveraged `pygame`. The code for this module can be found in `isofitts.py`. The cursor moves based on the `OnlineEMGClassifier's` predictions: - -```Python -self.current_direction = [0,0] -data, _ = self.sock.recvfrom(1024) -data = str(data.decode("utf-8")) -if data: - input_class = float(data.split(' ')[0]) - # 0 = Hand Closed = down - if input_class == 0: - self.current_direction[1] += self.VEL - # 1 = Hand Open - elif input_class == 1: - self.current_direction[1] -= self.VEL - # 3 = Extension - elif input_class == 3: - self.current_direction[0] += self.VEL - # 4 = Flexion - elif input_class == 4: - self.current_direction[0] -= self.VEL -``` +To create the Isofitts test, we leveraged `pygame`. The code for this module can be found in `libemg.environments.isofitts.py`. -To increase the speed of the cursor we could do one of two things: (1) increase the velocity of the cursor (i.e., how many pixels it moves for each prediction), or (2) decrease the increment so that more predictions are made in the same amount of time. +To increase the speed of the cursor we could do one of two things: (1) increase the velocity of the cursor (i.e., how many pixels it moves for each prediction), or (2) decrease the increment so that more predictions are made in the same amount of time. Parameters like this can be modified by passing arguments to the `IsoFitts` constructor. # Data Analysis -After accumulating data from the experiment, we need a way to analyze the data. In `analyze_data.py`, we added the capability to evaluate each model's offline and online performance. +After accumulating data from the experiment, we need a way to analyze the data. In `analyze_data.py`, we added the capability to evaluate each model's offline and online performance. -To evaluate each model's offline performance, we took a similar approach to set up the online classifier. However, in this case, we have to split up the data into training and testing. To do this, we first extract each of the five reps of data. We will split this into training and testing in a little bit. +To evaluate each model's offline performance, we took a similar approach to set up the online model. However, in this case, we have to split up the data into training and testing. To do this, we first extract each of the 3 reps of data. We will split this into training and testing in a little bit. ```Python -dataset_folder = 'data' -classes_values = ["0","1","2","3","4"] -reps_values = ["0","1","2","3","4"] regex_filters = [ - RegexFilter(left_bound = "_C_", right_bound=".csv", values = classes_values, description='classes'), - RegexFilter(left_bound = "R_", right_bound="_C_", values = reps_values, description='reps') + RegexFilter(left_bound = "classification/C_", right_bound="_R", values = ["0","1","2","3","4"], description='classes'), + RegexFilter(left_bound = "R_", right_bound="_emg.csv", values = ["0", "1", "2"], description='reps'), ] -odh = OfflineDataHandler() -odh.get_data(folder_location=dataset_folder, filename_dic = dic, delimiter=",") + +clf_odh = OfflineDataHandler() +clf_odh.get_data('data/', regex_filters, delimiter=",") + +regex_filters = [ + RegexFilter(left_bound='data/regression/C_0_R_', right_bound='_emg.csv', values=['0', '1', '2'], description='reps') +] +metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='animation/', right_bound='.txt', values=['collection'], description='labels'), package_function=lambda x, y: True) +] +reg_odh = OfflineDataHandler() +reg_odh.get_data('./', regex_filters, metadata_fetchers=metadata_fetchers, delimiter=',') ``` -Using the `isolate_data` function, we can split the data into training and testing. In this specific case, we are splitting on the "reps" keyword and we want values with index 0-2 for training and 3-4 for testing. After isolating the data, we extract windows and associated metadata for both training and testing sets. + +Using the `isolate_data` function, we can split the data into training and testing. In this specific case, we are splitting on the "reps" keyword and we want values with index 0-1 for training and 2 for testing. After isolating the data, we extract windows and associated metadata for both training and testing sets. ```Python -train_odh = odh.isolate_data(key="reps", values=[0,1,2]) -train_windows, train_metadata = train_odh.parse_windows(WINDOW_SIZE,WINDOW_INCREMENT) -test_odh = odh.isolate_data(key="reps", values=[3,4]) -test_windows, test_metadata = test_odh.parse_windows(WINDOW_SIZE,WINDOW_INCREMENT) +train_odh = odh.isolate_data(key="reps", values=[0,1]) +train_windows, train_metadata = train_odh.parse_windows(WINDOW_SIZE,WINDOW_INCREMENT, metadata_operations=metadata_operations) +test_odh = odh.isolate_data(key="reps", values=[2]) +test_windows, test_metadata = test_odh.parse_windows(WINDOW_SIZE,WINDOW_INCREMENT, metadata_operations=metadata_operations) ``` -Next, we create a dataset dictionary consisting of testing and training features and labels. This dictionary is passed into an `OfflineDataHandler.` +Next, we create a dataset dictionary consisting of testing and training features and labels. This dictionary is passed into an `OfflineDataHandler`. ```Python data_set = {} data_set['testing_features'] = fe.extract_feature_group('HTD', test_windows) data_set['training_features'] = fe.extract_feature_group('HTD', train_windows) -data_set['testing_labels'] = test_metadata['classes'] -data_set['training_labels'] = train_metadata['classes'] +data_set['testing_labels'] = test_metadata[labels_key] +data_set['training_labels'] = train_metadata[labels_key] ``` -Finally, to extract the offline performance of each model, we leverage the `OfflineMetrics` module. We do this in a loop to easily evaluate a number of classifiers. We append the metrics to a dictionary for future use. +Finally, to extract the offline performance of each model, we leverage the `OfflineMetrics` module. We do this in a loop to easily evaluate a number of models. We append the metrics to a dictionary for future use. ```Python om = OfflineMetrics() -metrics = ['CA', 'AER', 'INS', 'CONF_MAT'] -# Normal Case - Test all different classifiers -for model in ['LDA', 'SVM', 'KNN', 'RF']: - classifier = EMGClassifier() - classifier.fit(model, data_set.copy()) - preds, probs = classifier.run(data_set['testing_features'], data_set['testing_labels']) +# Normal Case - Test all different models +for model in models: + if is_regression: + model = EMGRegressor(model) + model.fit(data_set.copy()) + preds = model.run(data_set['testing_features']) + else: + model = EMGClassifier(model) + model.fit(data_set.copy()) + preds, _ = model.run(data_set['testing_features']) out_metrics = om.extract_offline_metrics(metrics, data_set['testing_labels'], preds, 2) - offline_metrics['classifier'].append(model) + offline_metrics['model'].append(model) offline_metrics['metrics'].append(out_metrics) -return offline_metric +return offline_metrics ``` # Results -There are clear discrepancies between offline and online metrics. For example, RF outperforms LDA in the offline analysis, but it is clear in the online test that it is much worse. This highlights the need to evaluate EMG-based control systems in online settings with user-in-the-loop feedback. +There are clear discrepancies between offline and online metrics. For example, RF outperforms LDA in the offline classification analysis, but it is clear in the online test that it is much worse. Similarly, RF outperforms LR in the offline regression analysis, but the usability metrics again suggest that LR outperforms RF during an online task. This highlights the need to evaluate EMG-based control systems in online settings with user-in-the-loop feedback. + +These results also show that regressors had worse usability metrics than classifiers despite enabling simultaneous motions. The high number of overshoots indicate that this is likely due to the fact that the model stuggled to stay at rest without drifting, increasing the time each trial took. This example could be expanded by adding things like a threshold to the regressors (see `EMGRegressor.add_deadband`), which may improve regressor performance. **Visual Output:** diff --git a/docs/source/examples/offline_regression_example/offline_regression.md b/docs/source/examples/offline_regression_example/offline_regression.md new file mode 100644 index 00000000..7ecb798b --- /dev/null +++ b/docs/source/examples/offline_regression_example/offline_regression.md @@ -0,0 +1,98 @@ +[View Source Code](https://github.com/LibEMG/LibEMG_OfflineRegression_Showcase) + + + +This simple offline example showcases some of the offline capabilities for regression analysis. In this example, we will load in the OneSubjectEMaGerDataset and assess the performance of multiple regressors. All code can be found in `main.py`. + +## Step 1: Importing LibEMG + +The very first step involves importing the modules needed. In general, each of LibEMG's modules has its own import. Make sure that you have successfully installed libemg through pip. + +```Python +import numpy as np +import matplotlib.pyplot as plt +from libemg.offline_metrics import OfflineMetrics +from libemg.datasets import OneSubjectEMaGerDataset +from libemg.feature_extractor import FeatureExtractor +from libemg.emg_predictor import EMGRegressor +``` + +## Step 2: Setting up Constants + +Preprocessing parameters, such as window size, window increment, and the feature set must be decided before EMG data can be prepared for estimation. LibEMG defines window and increment sizes as the number of samples. In this case, the dataset was recorded from the EMaGer cuff, which samples at 1 kHz, so a window of 150 samples corresponds to 150ms. + +The window increment, window size, and feature set default to 40, 150, and 'HTD', respecively. These variables can be customized in this script using the provided CLI. Use `python main.py -h` for an explanation of the CLI. Example usage is also provided below: + +```Bash +python main.py --window_size 200 --window_increment 50 --feature_set MSWT +``` + +# Step 3: Loading in Dataset + +This example uses the `OneSubjectEMaGerDataset`. Instantiating the `Dataset` will automatically download the data into the specified directory, and calling the `prepare_data()` method will load EMG data and metadata (e.g., reps, movements, labels) into an `OfflineDataHandler`. This dataset consists of 5 repetitions, so we use 4 for training data and 1 for testing data. After splitting our data into training and test splits, we perform windowing on the raw EMG data. By default, the metadata assigned to each window will be based on the mode of that window. Since we are analyzing regression data, we pass in a function that tells the `OfflineDataHandler` to grab the label from the last sample in the window instead of taking the mode of the window. We can specify how we want to handle windowing of each type of metadata by passing in a `metadata_operations` dictionary. + +```Python +# Load data +odh = OneSubjectEMaGerDataset().prepare_data() + +# Split into train/test reps +train_odh = odh.isolate_data('reps', [0, 1, 2, 3]) +test_odh = odh.isolate_data('reps', [4]) + +# Extract windows +metadata_operations = {'labels': lambda x: x[-1]} # grab label of last sample in window +train_windows, train_metadata = train_odh.parse_windows(args.window_size, args.window_increment, metadata_operations=metadata_operations) +test_windows, test_metadata = test_odh.parse_windows(args.window_size, args.window_increment, metadata_operations=metadata_operations) +``` + +# Step 4: Feature Extraction + +We then extract features using the `FeatureExtractor` for our training and test data. The `fit()` method expects a dictionary with the keys `training_features` and `training_labels`, so we create one and pass in our extracted features and training labels. + +```Python +training_features = fe.extract_feature_group(args.feature_set, train_windows, array=True), +training_labels = train_metadata['labels'] +test_features = fe.extract_feature_group(args.feature_set, test_windows, array=True) +test_labels = test_metadata['labels'] + +training_set = { + 'training_features': training_features, + 'training_labels': training_labels +} +``` + +# Step 5: Regression + +`LibEMG` allows you to pass in custom models, but you can also pass in a string that will create a model for you. In this example, we compare a linear regressor to a gradient boosting regressor. We iterate through a list of the models we want to observe, fit the model to the training data, and calculate metrics based on predictions on the test data. We then store these metrics for plotting later. + +```Python +results = {metric: [] for metric in ['R2', 'NRMSE', 'MAE']} +for model in models: + reg = EMGRegressor(model) + + # Fit and run model + print(f"Fitting {model}...") + reg.fit(training_set.copy()) + predictions = reg.run(test_features) + + metrics = om.extract_offline_metrics(results.keys(), test_labels, predictions) + for metric in metrics: + results[metric].append(metrics[metric].mean()) +``` + +# Step 6: Visualization + +Finally, we visualize our results. We first plot the decision stream for each model. After each model is fitted, we plot the offline metrics for each type of model. + +```Python +# Note: this will block the main thread once the plot is shown. Close the plot to continue execution. +reg.visualize(test_labels, predictions) + +fig, axs = plt.subplots(nrows=len(results), layout='constrained', figsize=(8, 8), sharex=True) +for metric, ax in zip(results.keys(), axs): + ax.bar(models, np.array(results[metric]) * 100) + ax.set_ylabel(f"{metric} (%)") + +fig.suptitle('Metrics Summary') +plt.show() +``` diff --git a/docs/source/examples/offline_regression_example/offline_regression_example.rst b/docs/source/examples/offline_regression_example/offline_regression_example.rst new file mode 100644 index 00000000..7e8707f7 --- /dev/null +++ b/docs/source/examples/offline_regression_example/offline_regression_example.rst @@ -0,0 +1,4 @@ +Offline Regression Analysis +========================================== +.. include:: offline_regression.md + :parser: myst_parser.sphinx_ \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 90ff5a3d..95d6b893 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,8 +17,10 @@ LibEMG documentation/filtering/filtering documentation/features/features documentation/feature_selection/feature_selection - documentation/classification/classification + documentation/prediction/prediction + documentation/adaptation/adaptation documentation/evaluation/evaluation + documentation/animation/animation .. toctree:: :maxdepth: 1 @@ -43,6 +45,7 @@ LibEMG examples/features_and_group_example/features_and_group_example examples/feature_optimization_example/feature_optimization_example examples/deep_learning_example/deep_learning_example + examples/offline_regression_example/offline_regression_example .. toctree:: :maxdepth: 1 diff --git a/libemg/__init__.py b/libemg/__init__.py index 1c06465f..00b8fe9e 100644 --- a/libemg/__init__.py +++ b/libemg/__init__.py @@ -10,3 +10,7 @@ from libemg import animator from libemg import gui from libemg import shared_memory_manager +from libemg import environments +from libemg import output_writer +from libemg import environments +from libemg import adaptation \ No newline at end of file diff --git a/libemg/_datasets/_3DC.py b/libemg/_datasets/_3DC.py new file mode 100644 index 00000000..41f2f4bf --- /dev/null +++ b/libemg/_datasets/_3DC.py @@ -0,0 +1,46 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class _3DCDataset(Dataset): + def __init__(self, dataset_folder="_3DCDataset/"): + Dataset.__init__(self, + 1000, + 10, + '3DC Armband (Prototype)', + 22, + {0: "Neutral", 1: "Radial Deviation", 2: "Wrist Flexion", 3: "Ulnar Deviation", 4: "Wrist Extension", 5: "Supination", 6: "Pronation", 7: "Power Grip", 8: "Open Hand", 9: "Chuck Grip", 10: "Pinch Grip"}, + '8 (4 Train, 4 Test)', + "The 3DC dataset including 11 classes.", + "https://doi.org/10.3389/fbioe.2020.00158") + self.url = "https://github.com/libemg/3DCDataset" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + subject_list = np.array(list(range(1,23))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + + + sets_values = ["train", "test"] + reps_values = ["0","1","2","3"] + classes_values = [str(i) for i in range(11)] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound = "/", right_bound="/EMG", values = sets_values, description='sets'), + RegexFilter(left_bound = "_", right_bound=".txt", values = classes_values, description='classes'), + RegexFilter(left_bound = "EMG_gesture_", right_bound="_", values = reps_values, description='reps'), + RegexFilter(left_bound="Participant", right_bound="/",values=subjects_values, description='subjects') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("sets", [0], fast=True), 'Test': odh.isolate_data("sets", [1], fast=True)} + + return data \ No newline at end of file diff --git a/libemg/_datasets/__init__.py b/libemg/_datasets/__init__.py new file mode 100644 index 00000000..975fd02b --- /dev/null +++ b/libemg/_datasets/__init__.py @@ -0,0 +1,17 @@ +from libemg._datasets import _3DC +from libemg._datasets import ciil +from libemg._datasets import continous_transitions +from libemg._datasets import dataset +from libemg._datasets import emg_epn612 +from libemg._datasets import fors_emg +from libemg._datasets import fougner_lp +from libemg._datasets import grab_myo +from libemg._datasets import hyser +from libemg._datasets import intensity +from libemg._datasets import kaufmann_md +from libemg._datasets import nina_pro +from libemg._datasets import one_subject_emager +from libemg._datasets import one_subject_myo +from libemg._datasets import radmand_lp +from libemg._datasets import tmr_shirleyryanabilitylab +from libemg._datasets import emg2pose \ No newline at end of file diff --git a/libemg/_datasets/ciil.py b/libemg/_datasets/ciil.py new file mode 100644 index 00000000..4614cbdd --- /dev/null +++ b/libemg/_datasets/ciil.py @@ -0,0 +1,159 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter, FilePackager +from pathlib import Path +import numpy as np + + +class CIIL_MinimalData(Dataset): + def __init__(self, dataset_folder='CIILData/'): + Dataset.__init__(self, + 200, + 8, + 'Myo Armband', + 11, + {0: 'Close', 1: 'Open', 2: 'Rest', 3: 'Flexion', 4: 'Extension'}, + '1 Train (1s), 15 Test', + "The goal of this Myo dataset is to explore how well models perform when they have a limited amount of training data (1s per class).", + 'https://ieeexplore.ieee.org/abstract/document/10394393') + self.url = "https://github.com/LibEMG/CIILData" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects=None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + subfolder = 'MinimalTrainingData' + subject_list = np.array(list(range(0, 11))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + + classes_values = [str(i) for i in range(0,5)] + reps_values = ["0","1","2"] + sets = ["train", "test"] + regex_filters = [ + RegexFilter(left_bound = "/", right_bound="/", values = sets, description='sets'), + RegexFilter(left_bound = "/subject", right_bound="/", values = subjects_values, description='subjects'), + RegexFilter(left_bound = "R_", right_bound="_", values = reps_values, description='reps'), + RegexFilter(left_bound = "C_", right_bound=".csv", values = classes_values, description='classes') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder + '/' + subfolder, regex_filters=regex_filters, delimiter=",") + + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("sets", [0], fast=True), 'Test': odh.isolate_data("sets", [1], fast=True)} + + return data + +class CIIL_ElectrodeShift(Dataset): + def __init__(self, dataset_folder='CIILData/'): + Dataset.__init__(self, + 200, + 8, + 'Myo Armband', + 21, + {0: 'Close', 1: 'Open', 2: 'Rest', 3: 'Flexion', 4: 'Extension'}, + '5 Train (Before Shift), 8 Test (After Shift)', + "An electrode shift confounding factors dataset.", + 'https://link.springer.com/article/10.1186/s12984-024-01355-4') + self.url = "https://github.com/LibEMG/CIILData" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects=None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + subfolder = 'ElectrodeShift' + subject_list = np.array(list(range(0, 21))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + + classes_values = [str(i) for i in range(0,5)] + reps_values = ["0","1","2","3","4"] + sets = ["training", "trial_1", "trial_2", "trial_3", "trial_4"] + regex_filters = [ + RegexFilter(left_bound = "/", right_bound="/", values = sets, description='sets'), + RegexFilter(left_bound = "/subject", right_bound="/", values = subjects_values, description='subjects'), + RegexFilter(left_bound = "R_", right_bound="_", values = reps_values, description='reps'), + RegexFilter(left_bound = "C_", right_bound=".csv", values = classes_values, description='classes') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder + '/' + subfolder, regex_filters=regex_filters, delimiter=",") + + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("sets", [0], fast=True), 'Test': odh.isolate_data("sets", [1,2,3,4], fast=True)} + + return data + + +class CIIL_WeaklySupervised(Dataset): + def __init__(self, dataset_folder='CIIL_WeaklySupervised/'): + Dataset.__init__(self, + 1000, + 8, + 'OyMotion gForcePro+ EMG Armband', + 16, + {0: 'Close', 1: 'Open', 2: 'Rest', 3: 'Flexion', 4: 'Extension'}, + '30 min weakly supervised, 1 rep calibration, 14 reps test', + "A weakly supervised environment with sparse supervised calibration.", + 'In Submission') + self.url = "https://unbcloud-my.sharepoint.com/:u:/g/personal/ecampbe2_unb_ca/EaABHYybhfJNslTVcvwPPwgB9WwqlTLCStui30maqY53kw?e=MbboMd" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, + subjects = None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download_via_onedrive(self.url, self.dataset_folder) + + # supervised odh loading + subject_list = np.array(list(range(0, 16))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + classes_values = [str(i) for i in range(0,5)] + reps_values = [str(i) for i in range(0,15)] + setting_values = [".csv", ""] # this is arbitrary to get a field that separates WS from S + regex_filters = [ + RegexFilter(left_bound = "", right_bound="", values = setting_values, description='settings'), + RegexFilter(left_bound = "/S", right_bound="/", values = subjects_values, description='subjects'), + RegexFilter(left_bound = "R", right_bound=".csv", values = reps_values, description='reps'), + RegexFilter(left_bound = "C", right_bound="_R", values = classes_values, description='classes') + ] + odh_s = OfflineDataHandler() + odh_s.get_data(folder_location=self.dataset_folder+"CIIL_WeaklySupervised/", + regex_filters=regex_filters, + delimiter=",") + + # weakly supervised odh loading + reps_values = [str(i) for i in range(3)] + setting_values = ["", ".csv"] # this is arbitrary to get a field that separates WS from S + regex_filters = [ + RegexFilter(left_bound = "", right_bound="", values = setting_values, description='settings'), + RegexFilter(left_bound = "/S", right_bound="/", values = subjects_values, description='subjects'), + RegexFilter(left_bound = "WS", right_bound=".csv", values = reps_values, description='reps'), + ] + metadata_fetchers = [ + FilePackager(regex_filter=RegexFilter(left_bound="", right_bound="targets.csv", values=["_"], description="classes"), + package_function=lambda x, y: (x.split("WS")[1][0] == y.split("WS")[1][0]) and (Path(x).parent == Path(y).parent) + ) + ] + odh_ws = OfflineDataHandler() + odh_ws.get_data(folder_location=self.dataset_folder+"CIIL_WeaklySupervised/", + regex_filters=regex_filters, + metadata_fetchers=metadata_fetchers, + delimiter=",") + + data = odh_s + odh_ws + if split: + data = {'All': data, + 'Pretrain': odh_ws, + 'Train': odh_s.isolate_data("reps", [0], fast=True), + 'Test': odh_s.isolate_data("reps", list(range(1,15)), fast=True)} + + return data diff --git a/libemg/_datasets/continous_transitions.py b/libemg/_datasets/continous_transitions.py new file mode 100644 index 00000000..ae9aa6e6 --- /dev/null +++ b/libemg/_datasets/continous_transitions.py @@ -0,0 +1,67 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler +import h5py +import numpy as np + +class ContinuousTransitions(Dataset): + def __init__(self, dataset_folder="ContinuousTransitions/"): + Dataset.__init__(self, + 2000, + 6, + 'Delsys', + 43, + {0: 'No Motion', 1: 'Wrist Flexion', 2: 'Wrist Extension', 3: 'Wrist Pronation', 4: 'Wrist Supination', 5: 'Hand Close', 6: 'Hand Open'}, + '6 Training (Ramp), 42 Transitions (All combinations of Transitions) x 6 Reps', + "The testing set in this dataset has continuous transitions between classes which is a more realistic offline evaluation standard for myoelectric control.", + "https://ieeexplore.ieee.org/document/10254242") + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects=None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + print("Please download the dataset from: https://unbcloud-my.sharepoint.com/:f:/g/personal/ecampbe2_unb_ca/EjgjhM9ZHJxOglKoAf062ngBf4wFj2Mn2bORKY1-aMYGRw?e=WkZNwI") + return + + # Training ODH + odh_tr = OfflineDataHandler() + odh_tr.subjects = [] + odh_tr.classes = [] + odh_tr.extra_attributes = ['subjects', 'classes'] + + # Testing ODH + odh_te = OfflineDataHandler() + odh_te.subjects = [] + odh_te.classes = [] + odh_te.extra_attributes = ['subjects', 'classes'] + + subject_list = np.array([2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,20,21,22,23,25,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47]) + if subjects: + subject_list = subject_list[subjects] + + for s_i, s in enumerate(subject_list): + data = h5py.File(self.dataset_folder + '/P' + f"{s:02}" + '.hdf5', "r") + cont_labels = data['continuous']['emg']['prompt'][()] + cont_labels = np.hstack([np.ones((1000)) * cont_labels[0], cont_labels[0:len(cont_labels)-1000]]) # Rolling about 0.5s as per Shri's suggestion + cont_emg = data['continuous']['emg']['signal'][()] + cont_chg_idxs = np.insert(np.where(cont_labels[:-1] != cont_labels[1:])[0], 0, -1) + cont_chg_idxs = np.insert(cont_chg_idxs, len(cont_chg_idxs), len(cont_emg)) + for i in range(0, len(cont_chg_idxs)-1): + odh_te.data.append(cont_emg[cont_chg_idxs[i]+1:cont_chg_idxs[i+1]]) + odh_te.classes.append(np.expand_dims(cont_labels[cont_chg_idxs[i]+1:cont_chg_idxs[i+1]]-1, axis=1)) + odh_te.subjects.append(np.ones((len(odh_te.data[-1]), 1)) * s_i) + + ramp_emg = data['ramp']['emg']['signal'][()] + ramp_labels = data['ramp']['emg']['prompt'][()] + r_chg_idxs = np.insert(np.where(ramp_labels[:-1] != ramp_labels[1:])[0], 0, -1) + r_chg_idxs = np.insert(r_chg_idxs, len(r_chg_idxs), len(ramp_emg)) + for i in range(0, len(r_chg_idxs)-1): + odh_tr.data.append(ramp_emg[r_chg_idxs[i]+1:r_chg_idxs[i+1]]) + odh_tr.classes.append(np.expand_dims(ramp_labels[r_chg_idxs[i]+1:r_chg_idxs[i+1]]-1, axis=1)) + odh_tr.subjects.append(np.ones((len(odh_tr.data[-1]), 1)) * s_i) + + odh_all = odh_tr + odh_te + data = odh_all + if split: + data = {'All': odh_all, 'Train': odh_tr, 'Test': odh_te} + + return data diff --git a/libemg/_datasets/dataset.py b/libemg/_datasets/dataset.py new file mode 100644 index 00000000..df25be1c --- /dev/null +++ b/libemg/_datasets/dataset.py @@ -0,0 +1,55 @@ +import os +from libemg.data_handler import OfflineDataHandler +from onedrivedownloader import download as onedrive_download +# this assumes you have git downloaded (not pygit, but the command line program git) + +class Dataset: + def __init__(self, sampling, num_channels, recording_device, num_subjects, gestures, num_reps, description, citation): + # Every class should have this + self.sampling=sampling + self.num_channels=num_channels + self.recording_device=recording_device + self.num_subjects=num_subjects + self.gestures=gestures + self.num_reps=num_reps + self.description=description + self.citation=citation + + def download(self, url, dataset_name): + clone_command = "git clone " + url + " " + dataset_name + os.system(clone_command) + + def download_via_onedrive(self, url, dataset_name, unzip=True, clean=True): + onedrive_download(url=url, + filename = dataset_name, + unzip=unzip, + clean=clean) + + def remove_dataset(self, dataset_folder): + remove_command = "rm -rf " + dataset_folder + os.system(remove_command) + + def check_exists(self, dataset_folder): + return os.path.exists(dataset_folder) + + def prepare_data(self, split = True): + pass + + def get_info(self): + print(str(self.description) + '\n' + 'Sampling Rate: ' + str(self.sampling) + '\nNumber of Channels: ' + str(self.num_channels) + + '\nDevice: ' + self.recording_device + '\nGestures: ' + str(self.gestures) + '\nNumber of Reps: ' + str(self.num_reps) + '\nNumber of Subjects: ' + str(self.num_subjects) + + '\nCitation: ' + str(self.citation)) + +# given a directory, return a list of files in that directory matching a format +# can be nested +# this is just a handly utility +def find_all_files_of_type_recursively(dir, terminator): + files = os.listdir(dir) + file_list = [] + for file in files: + if file.endswith(terminator): + file_list.append(dir+file) + else: + if os.path.isdir(dir+file): + file_list += find_all_files_of_type_recursively(dir+file+'/',terminator) + return file_list \ No newline at end of file diff --git a/libemg/_datasets/emg2pose.py b/libemg/_datasets/emg2pose.py new file mode 100644 index 00000000..f696cd49 --- /dev/null +++ b/libemg/_datasets/emg2pose.py @@ -0,0 +1,234 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler +from libemg.feature_extractor import FeatureExtractor +from libemg.utils import * +import numpy as np +import pandas as pd +import h5py + +class EMG2POSE(Dataset): + def __init__(self, dataset_folder="Meta/emg2pose_data/"): + self.mapping = {'FingerPinches1': 'AllFingerPinchesThumbSwipeThumbRotate', 'Object1': 'CoffeePanicPete', 'Counting1': 'CountingUpDownFaceSideAway', 'Counting2': 'CountingUpDownFingerWigglingSpreading', 'DoorknobFingerGraspFistGrab': 'DoorknobFingerGraspFistGrab', 'Throwing': 'FastPongFronthandBackhandThrowing', 'Abduction': 'FingerAbductionSeries', 'FingerFreeform': 'FingerFreeform', 'FingerPinches2': 'FingerPinchesSingleFingerPinchesMultiple', 'HandHandInteractions': 'FingerTouchPalmClapmrburns', 'Wiggling1': 'FingerWigglingSpreading', 'Punch': 'GraspPunchCloseFar', 'Gesture1': 'HandClawGraspFlicks', 'StaticHands': 'HandDeskSeparateClaspedChest', 'FingerPinches3': 'HandOverHandAllFingerPinchesThumbSwipeThumbRotate', 'Wiggling2': 'HandOverHandCountingUpDownFingerWigglingSpreading', 'Unconstrained': 'unconstrained', 'Gesture2': 'HookEmHornsOKScissors', 'FingerPinches4': 'IndexPinchesMiddlePinchesThumbswipes', 'Pointing': 'IndividualFingerPointingSnap', 'Freestyle1': 'OneHandedFreeStyle', 'Object2': 'PlayBlocksChess', 'Draw': 'PokeDrawPinchRotateclosefar', 'Poke': 'PokePinchCloseFar', 'Gesture3': 'ShakaVulcanPeace', 'ThumbsSwipes': 'ThumbsSwipesWholeHand', 'ThumbRotations': 'ThumbsUpDownThumbRotationsCWCCWP', 'Freestyle2': 'TwoHandedFreeStyle', 'WristFlex': 'WristFlexionAbduction'} + + Dataset.__init__(self, + 2000, + 32, + 'Ctrl Labs Armband', + 193, + self.mapping, + 'N/A', + "A large dataset from ctrl-labs (Meta) for joint angle estimation. Note that not all subjects have all stages.", + "https://openreview.net/forum?id=b5n3lKRLzk") + self.dataset_folder = dataset_folder + + def check_files(self): + if not self.check_exists(self.dataset_folder): + print("Please download the dataset from: https://fb-ctrl-oss.s3.amazonaws.com/emg2pose/emg2pose_dataset.tar") + return False + + if not self.check_exists(self.dataset_folder + 'metadata.csv'): + print("Could not find metadata file... Please make sure this is downloaded and in the folder.") + return False + + return True + +class EMG2POSECU(EMG2POSE): + """ + The cross user version of emg2pose. We are testing generalization within across users within the same stage. + + Parameters + ---------- + stage: str (default='Wiggling2') + The stage to test. Will grab all subjects with that stage. + split: list (default=[80,20]) + Defaults to 80/20 split for train and test data respectively. + """ + def __init__(self, dataset_folder="Meta/emg2pose_data/", stage = 'Wiggling2', split = [0.8,0.2]): + EMG2POSE.__init__(self, dataset_folder=dataset_folder) + self.stage = stage + self.split = split + + def prepare_data(self, split = True, feature_list = None, window_size = None, window_inc = None, feature_dic = None): + """ + Use the features, window_size, and window_inc parameters to extract features directly so that you save on memory usage. + + Parameters + ---------- + feature_list: list (default=None) + List of featurs. + window_size: int (default=None) + Number of samples. + window_inc: int (default=None) + Number of samples. + feature_dic: dic (default=None) + Feature parameters. + """ + if feature_list or window_size or window_inc: + assert feature_list + assert window_size + assert window_inc + fe = FeatureExtractor() + + odh = OfflineDataHandler() + unique_subjects = [] + odh.subjects = [] + odh.labels = [] + odh.extra_attributes = ['subjects', 'labels'] + + self.check_files() + df = pd.read_csv(self.dataset_folder + 'metadata.csv') + subject_ids = list(np.unique(df['user'])) + + target_gesture = self.mapping[self.stage] + for s_i, s in enumerate(subject_ids): + sub_mask = df['user'] == s + gesture_mask = df['stage'] == target_gesture + + # Get all files for that subject + files = df['filename'][(sub_mask) & (gesture_mask)] + files = [f.replace('left', '') for f in files] + files = [f.replace('right', '') for f in files] + for f in np.unique(files): + unique_subjects.append(s_i) + # Check that files exists otherwise skip + if not (self.check_exists(self.dataset_folder + '/' + f + 'left.hdf5') and self.check_exists(self.dataset_folder + '/' + f + 'right.hdf5')): + continue + left = h5py.File(self.dataset_folder + '/' + f + 'left.hdf5', "r") + right = h5py.File(self.dataset_folder + '/' + f + 'right.hdf5', "r") + + emg_left = left['emg2pose']['timeseries']['emg'] + emg_right = right['emg2pose']['timeseries']['emg'] + min_idx = min([len(emg_left), len(emg_right)]) + + ja_left = left['emg2pose']['timeseries']['joint_angles'] + ja_right = right['emg2pose']['timeseries']['joint_angles'] + + if feature_list: + feats = fe.extract_features(feature_list, get_windows(np.hstack([emg_left[0:min_idx], emg_right[0:min_idx]]), window_size, window_inc), feature_dic=feature_dic, array=True) + odh.data.append(feats) + labels = get_windows(np.hstack([ja_left[0:min_idx], ja_right[0:min_idx]]), window_size, window_inc)[:,:,-1] + odh.labels.append(labels) + odh.subjects.append(np.ones((len(odh.data[-1]), 1)) * s_i) + else: + odh.data.append(np.hstack([emg_left[0:min_idx], emg_right[0:min_idx]])) + odh.labels.append(np.hstack([ja_left[0:min_idx], ja_right[0:min_idx]])) + odh.subjects.append(np.ones((len(odh.data[-1]), 1)) * s_i) + + unique_subjects = np.unique(unique_subjects) + tr_subjects = list(unique_subjects[0:int(len(unique_subjects)*self.split[0])]) + te_subjects = list(unique_subjects[-int(len(unique_subjects)*self.split[1]):]) + + if split: + odh = {'All': odh, 'Train': odh.isolate_data('subjects', tr_subjects), 'Test': odh.isolate_data('subjects', te_subjects)} + return odh + +class EMG2POSEUD(EMG2POSE): + """ + The user dependent version of emg2pose. We are testing generalization within user to unseen stages. + + Parameters + ---------- + train_stages: list (default = None) + If None, the training stages will be all of the ones not included in the test stages. + test_stages: list (default=['Wiggling2', 'Gesture3', 'Gesture2', 'Counting2', 'FingerFreeform', 'Counting1']) + A list of stages to use for training. See self.mapping for options. If a user doesn't have that testing stage then it is ignored. + """ + def __init__(self, dataset_folder="Meta/emg2pose_data/", train_stages = None, test_stages = None): + EMG2POSE.__init__(self, dataset_folder=dataset_folder) + self.num_subjects = 192 # One participant was too low - assuming something was off + self.train_stages = train_stages + self.test_stages = test_stages + + # This split works for stage generalization - takes the average across stages, though + def prepare_data(self, split = True, subjects = None): + if self.test_stages: + for t in self.test_stages: + assert t in self.mapping.keys() + else: + self.test_stages = ['Wiggling2', 'Gesture3', 'Gesture2', 'Counting2', 'FingerFreeform', 'Counting1'] + + if self.train_stages: + for t in self.train_stages: + assert t in self.mapping.keys() + else: + self.train_stages = [] + for k in self.mapping.keys(): + if k not in self.test_stages: + self.train_stages.append(k) + + # (1) Make sure everything is downloaded + self.check_files() + + # (2) Load metadata file + df = pd.read_csv(self.dataset_folder + 'metadata.csv') + subject_ids = np.delete(np.array(list(np.unique(df['user']))), 144) + if subjects: + subject_ids = subject_ids[subjects] + subject_ids = list(subject_ids) + + odh_tr = OfflineDataHandler() + odh_tr.subjects = [] + odh_tr.labels = [] + odh_tr.stages = [] + odh_tr.reps = [] + odh_tr.extra_attributes = ['subjects', 'labels', 'stages', 'reps'] + + odh_te = OfflineDataHandler() + odh_te.subjects = [] + odh_te.labels = [] + odh_te.stages = [] + odh_te.reps = [] + odh_te.extra_attributes = ['subjects', 'labels', 'stages', 'reps'] + + # (3) Iterate through subjects and grab all of the relevant files + for s_i, s in enumerate(subject_ids): + sub_mask = df['user'] == s + gestures = [self.mapping[v] for v in np.hstack([self.train_stages, self.test_stages])] + reps = [0] * len(gestures) + gesture_mask = df['stage'].isin(gestures) + + # Get all files for that subject + files = df['filename'][(sub_mask) & (gesture_mask)] + files = [f.replace('left', '') for f in files] + files = [f.replace('right', '') for f in files] + for f in np.unique(files): + # Check that files exists otherwise skip + if not (self.check_exists(self.dataset_folder + '/' + f + 'left.hdf5') and self.check_exists(self.dataset_folder + '/' + f + 'right.hdf5')): + continue + + left = h5py.File(self.dataset_folder + '/' + f + 'left.hdf5', "r") + right = h5py.File(self.dataset_folder + '/' + f + 'right.hdf5', "r") + gest = df[df['filename'] == f + 'right']['stage'].item() + gesture_name = list(self.mapping.keys())[list(self.mapping.values()).index(gest)] + + emg_left = left['emg2pose']['timeseries']['emg'] + emg_right = right['emg2pose']['timeseries']['emg'] + min_idx = min([len(emg_left), len(emg_right)]) + + ja_left = left['emg2pose']['timeseries']['joint_angles'] + ja_right = right['emg2pose']['timeseries']['joint_angles'] + + if gesture_name in self.train_stages: + odh_tr.data.append(np.hstack([emg_left[0:min_idx], emg_right[0:min_idx]])) + odh_tr.labels.append(np.hstack([ja_left[0:min_idx], ja_right[0:min_idx]])) + odh_tr.stages.append(np.ones((len(odh_tr.data[-1]), 1)) * gestures.index(gest)) + odh_tr.subjects.append(np.ones((len(odh_tr.data[-1]), 1)) * s_i) + odh_tr.reps.append(np.ones((len(odh_tr.data[-1]), 1)) * reps[gestures.index(gest)]) + reps[gestures.index(gest)] += 1 + if gesture_name in self.test_stages: + odh_te.data.append(np.hstack([emg_left[0:min_idx], emg_right[0:min_idx]])) + odh_te.labels.append(np.hstack([ja_left[0:min_idx], ja_right[0:min_idx]])) + odh_te.stages.append(np.ones((len(odh_te.data[-1]), 1)) * gestures.index(gest)) + odh_te.subjects.append(np.ones((len(odh_te.data[-1]), 1)) * s_i) + odh_te.reps.append(np.ones((len(odh_te.data[-1]), 1)) * reps[gestures.index(gest)]) + reps[gestures.index(gest)] += 1 + + if len(odh_tr.data) == 0 or len(odh_te.data) == 0: + print('Invalid Subject Information: Please confirm that the subject has the desired stages') + return None + + odh_all = odh_tr + odh_te + data = odh_all + if split: + data = {'All': odh_all, 'Train': odh_tr, 'Test': odh_te} + return data \ No newline at end of file diff --git a/libemg/_datasets/emg_epn612.py b/libemg/_datasets/emg_epn612.py new file mode 100644 index 00000000..5e7a35e8 --- /dev/null +++ b/libemg/_datasets/emg_epn612.py @@ -0,0 +1,119 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler +import pickle +import numpy as np +from libemg.feature_extractor import FeatureExtractor +from libemg.utils import * + +class EMGEPN612(Dataset): + def __init__(self, dataset_file='EMGEPN612.pkl', cross_user=True): + split = '50 Reps x 306 Users (Train), 25 Reps x 306 Users (Test) --> Cross User Split' + if not cross_user: + split = '20 Reps (Train), 5 Reps (Test) from the 612 Test Users --> User Dependent Split' + + Dataset.__init__(self, + 200, + 8, + 'Myo Armband', + 612, + {0: 'No Movement', 1: 'Hand Close', 2: 'Flexion', 3: 'Extension', 4: 'Hand Open', 5: 'Pinch'}, + split, + "A large 612 user dataset for developing cross user models.", + 'https://doi.org/10.5281/zenodo.4421500') + self.url = "https://unbcloud-my.sharepoint.com/:u:/g/personal/ecampbe2_unb_ca/EWf3sEvRxg9HuAmGoBG2vYkBLyFv6UrPYGwAISPDW9dBXw?e=vjCA14" + self.dataset_name = dataset_file + + def get_odh(self, subjects=None, feature_list = None, window_size = None, window_inc = None, feature_dic = None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_name)): + self.download_via_onedrive(self.url, self.dataset_name, unzip=False, clean=False) + + if feature_list or window_size or window_inc: + assert feature_list + assert window_size + assert window_inc + fe = FeatureExtractor() + + subject_list = np.array(list(range(0,612))) + if subjects: + subject_list = np.array(subjects) + + file = open(self.dataset_name, 'rb') + data = pickle.load(file) + + emg = data[0] + labels = data[2] + + odh_tr = OfflineDataHandler() + odh_tr.subjects = [] + odh_tr.classes = [] + odh_tr.reps = [] + tr_reps = [0,0,0,0,0,0] + odh_tr.extra_attributes = ['subjects', 'classes', 'reps'] + for i, e in enumerate(emg['training']): + if i // 300 not in subject_list: + continue + if feature_list: + odh_tr.data.append(fe.extract_features(feature_list, get_windows(e, window_size, window_inc), feature_dic=feature_dic, array=True)) + odh_tr.classes.append(np.ones((len(odh_tr.data[-1]), 1)) * labels['training'][i]) + odh_tr.subjects.append(np.ones((len(odh_tr.data[-1]), 1)) * i//300) + odh_tr.reps.append(np.ones((len(odh_tr.data[-1]), 1)) * tr_reps[labels['training'][i]]) + else: + odh_tr.data.append(e) + odh_tr.classes.append(np.ones((len(e), 1)) * labels['training'][i]) + odh_tr.subjects.append(np.ones((len(e), 1)) * i//300) + odh_tr.reps.append(np.ones((len(e), 1)) * tr_reps[labels['training'][i]]) + tr_reps[labels['training'][i]] += 1 + if i % 300 == 0: + tr_reps = [0,0,0,0,0,0] + odh_te = OfflineDataHandler() + odh_te.subjects = [] + odh_te.classes = [] + odh_te.reps = [] + te_reps = [0,0,0,0,0,0] + odh_te.extra_attributes = ['subjects', 'classes', 'reps'] + for i, e in enumerate(emg['testing']): + if (i // 150 + 306) not in subject_list: + continue + if feature_list: + odh_te.data.append(fe.extract_features(feature_list, get_windows(e, window_size, window_inc), feature_dic=feature_dic, array=True)) + odh_te.classes.append(np.ones((len(odh_te.data[-1]), 1)) * labels['testing'][i]) + odh_te.subjects.append(np.ones((len(odh_te.data[-1]), 1)) * (i//150 + 306)) + odh_te.reps.append(np.ones((len(odh_te.data[-1]), 1)) * te_reps[labels['testing'][i]]) + else: + odh_te.data.append(e) + odh_te.classes.append(np.ones((len(e), 1)) * labels['testing'][i]) + odh_te.subjects.append(np.ones((len(e), 1)) * (i//150 + 306)) + odh_te.reps.append(np.ones((len(e), 1)) * te_reps[labels['testing'][i]]) + te_reps[labels['testing'][i]] += 1 + if i % 150 == 0: + te_reps = [0,0,0,0,0,0] + + return odh_tr + odh_te + +class EMGEPN_UserDependent(EMGEPN612): + def __init__(self, dataset_file='EMGEPN612.pkl'): + EMGEPN612.__init__(self, dataset_file=dataset_file, cross_user=False) + + def prepare_data(self, split = True, subjects = None): + odh = self.get_odh(subjects) + odh_tr = odh.isolate_data('reps', list(range(0,20))) + odh_te = odh.isolate_data('reps', list(range(20,25))) + + if split: + data = {'All': odh, 'Train': odh_tr, 'Test': odh_te} + return data + +class EMGEPN_UserIndependent(EMGEPN612): + def __init__(self, dataset_file='EMGEPN612.pkl'): + EMGEPN612.__init__(self, dataset_file=dataset_file, cross_user=True) + + def prepare_data(self, split = True, subjects=None, feature_list = None, window_size = None, window_inc = None, feature_dic = None): + odh = self.get_odh(subjects, feature_list, window_size, window_inc, feature_dic) + odh_tr = odh.isolate_data('subjects', values=list(range(0,306))) + odh_te = odh.isolate_data('subjects', values=list(range(306,612))) + if split: + data = {'All': odh_tr + odh_te, 'Train': odh_tr, 'Test': odh_te} + return data + + \ No newline at end of file diff --git a/libemg/_datasets/fors_emg.py b/libemg/_datasets/fors_emg.py new file mode 100644 index 00000000..d8390c4b --- /dev/null +++ b/libemg/_datasets/fors_emg.py @@ -0,0 +1,56 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import scipy.io +import numpy as np + +class FORSEMG(Dataset): + def __init__(self, dataset_folder='FORS-EMG/'): + Dataset.__init__(self, + 985, + 8, + 'Experimental Device', + 19, + {0: 'Thump Up', 1: 'Index', 2: 'Right Angle', 3: 'Peace', 4: 'Index Little', 5: 'Thumb Little', 6: 'Hand Close', 7: 'Hand Open', 8: 'Wrist Flexion', 9: 'Wrist Extension', 10: 'Ulnar Deviation', 11: 'Radial Deviation'}, + '5 Train, 10 Test (2 Forarm Orientations x 5 Reps)', + "FORS-EMG: Twelve gestures elicited in three forearm orientations (neutral, pronation, and supination).", + 'https://arxiv.org/abs/2409.07484t') + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + print("Please download the dataset from: https://www.kaggle.com/datasets/ummerummanchaity/fors-emg-a-novel-semg-dataset?resource=download") + return + + odh = OfflineDataHandler() + odh.subjects = [] + odh.classes = [] + odh.reps = [] + odh.orientation = [] + odh.extra_attributes = ['subjects', 'classes', 'reps', 'orientation'] + + subject_list = np.array(list(range(1,20))) + if subjects: + subject_list = subject_list[subjects] + + for s in subject_list: + for g_i, g in enumerate(['Thumb_UP', 'Index', 'Right_Angle', 'Peace', 'Index_Little', 'Thumb_Little', 'Hand_Close', 'Hand_Open', 'Wrist_Flexion', 'Wrist_Extension', 'Radial_Deviation']): + for r in [1,2,3,4,5]: + for o_i, o in enumerate(['Rest', 'Pronation', 'Supination']): + try: + mat = scipy.io.loadmat('FORS-EMG/Subject' + str(s) + '/' + o + '/' + g + '-' + str(r) + '.mat') + except: + o = o.lower() + mat = scipy.io.loadmat('FORS-EMG/Subject' + str(s) + '/' + o + '/' + g + '-' + str(r) + '.mat') + + odh.data.append(mat['value'].T) + odh.classes.append(np.ones((len(odh.data[-1]), 1)) * g_i) + odh.subjects.append(np.ones((len(odh.data[-1]), 1)) * s-1) + odh.reps.append(np.ones((len(odh.data[-1]), 1)) * r-1) + odh.orientation.append(np.ones((len(odh.data[-1]), 1)) * o_i) + + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data('orientation', [0], fast=True), 'Test': odh.isolate_data('orientation', [1,2], fast=True)} + + return data diff --git a/libemg/_datasets/fougner_lp.py b/libemg/_datasets/fougner_lp.py new file mode 100644 index 00000000..4a287421 --- /dev/null +++ b/libemg/_datasets/fougner_lp.py @@ -0,0 +1,46 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class FougnerLP(Dataset): + def __init__(self, dataset_folder="LimbPosition/"): + Dataset.__init__(self, + 1000, + 8, + 'BE328 by Liberating Technologies, Inc.', + 12, + {0: 'Wrist Flexion', 1: 'Wrist Extension', 2: 'Pronation', 3: 'Supination', 4: 'Hand Open', 5: 'Power Grip', 6: 'Pinch Grip', 7: 'Rest'}, + '10 Reps (Train), 10 Reps x 4 Positions', + "A limb position dataset (with 5 static limb positions).", + "https://ieeexplore.ieee.org/document/5985538") + self.url = "https://github.com/libemg/LimbPosition" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + subject_list = np.array(list(range(1,13))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + + position_values = ["1", "2", "3", "4", "5"] + classes_values = ["1", "2", "3", "4", "5", "8", "9", "12"] + reps_values = ["1","2","3","4","5","6","7","8","9","10"] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound="S", right_bound="_C",values=subjects_values, description='subjects'), + RegexFilter(left_bound = "_P", right_bound="_R", values = position_values, description='positions'), + RegexFilter(left_bound = "_C", right_bound="_P", values = classes_values, description='classes'), + RegexFilter(left_bound = "_R", right_bound=".txt", values = reps_values, description='reps'), + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder + 'FougnerLimbPosition/', regex_filters=regex_filters, delimiter=",") + odh = odh.isolate_channels(list(range(0,8))) + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("positions", [0], fast=True), 'Test': odh.isolate_data("positions", list(range(1, len(position_values))), fast=True)} + + return data \ No newline at end of file diff --git a/libemg/_datasets/grab_myo.py b/libemg/_datasets/grab_myo.py new file mode 100644 index 00000000..232d26e7 --- /dev/null +++ b/libemg/_datasets/grab_myo.py @@ -0,0 +1,103 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class GRABMyo(Dataset): + """ + By default this just uses the 16 forearm electrodes. + """ + def __init__(self, dataset_folder='GRABMyo/', baseline=False, cross_user=False): + if not cross_user: + split = '7 Train, 14 Test (2 Seperate Days x 7 Reps) --> Cross Day Split' + if baseline: + split = '5 Train, 2 Test --> Baseline Split' + else: + split = '30 Subjects x 3 Sessions (Train) - 14 Subjects x 3 Sessions (Test) --> Cross User Split' + Dataset.__init__(self, + 2048, + 16, + 'EMGUSB2+ device (OT Bioelletronica, Italy)', + 43, + {0: 'Lateral Prehension', 1: 'Thumb Adduction', 2: 'Thumb and Little Finger Opposition', 3: 'Thumb and Index Finger Opposition', 4: 'Thumb and Index Finger Extension', 5: 'Thumb and Little Finger Extension', 6: 'Index and Middle Finger Extension', + 7: 'Little Finger Extension', 8: 'Index Finger Extension', 9: 'Thumb Finger Extension', 10: 'Wrist Extension', 11: 'Wrist Flexion', 12: 'Forearm Supination', 13: 'Forearm Pronation', 14: 'Hand Open', 15: 'Hand Close', 16: 'Rest'}, + split, + "GrabMyo: A large cross session dataset including 17 gestures elicited across 3 seperate sessions.", + 'https://www.nature.com/articles/s41597-022-01836-y') + self.dataset_folder = dataset_folder + + def get_odh(self, subjects = None): + self.check_if_exist() + + sessions = ["1", "2", "3"] + subject_list = np.array(list(range(1,44))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + classes_values = ["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17"] + reps_values = ["1","2","3","4","5","6","7"] + + regex_filters = [ + RegexFilter(left_bound = "session", right_bound="_", values = sessions, description='sessions'), + RegexFilter(left_bound = "_gesture", right_bound="_", values = classes_values, description='classes'), + RegexFilter(left_bound = "trial", right_bound=".hea", values = reps_values, description='reps'), + RegexFilter(left_bound="participant", right_bound="_",values=subjects_values, description='subjects') + ] + + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") + + return odh.isolate_channels(list(range(0,16))) + + def check_if_exist(self): + if (not self.check_exists(self.dataset_folder)): + print("Please download the GRABMyo dataset from: https://physionet.org/content/grabmyo/1.0.2/") + return + print('\nPlease cite: ' + self.citation+'\n') + +class GrabMyoCrossUser(GRABMyo): + def __init__(self, dataset_folder="GRABMyo"): + GRABMyo.__init__(self, dataset_folder=dataset_folder, baseline=False) + + def prepare_data(self, split = True): + forearm_data = self.get_odh() + + train_data = forearm_data.isolate_data('subjects', list(range(0,30)), fast=True) + test_data = forearm_data.isolate_data('subjects', list(range(30,43)), fast=True) + + data = forearm_data + if split: + data = {'All': forearm_data, 'Train': train_data, 'Test': test_data} + + return data + +class GRABMyoCrossDay(GRABMyo): + def __init__(self, dataset_folder="GRABMyo"): + GRABMyo.__init__(self, dataset_folder=dataset_folder, baseline=False) + + def prepare_data(self, split = True, subjects = None): + forearm_data = self.get_odh(subjects) + train_data = forearm_data.isolate_data('sessions', [0], fast=True) + test_data = forearm_data.isolate_data('sessions', [1,2], fast=True) + + data = forearm_data + if split: + data = {'All': forearm_data, 'Train': train_data, 'Test': test_data} + + return data + +class GRABMyoBaseline(GRABMyo): + def __init__(self, dataset_folder="GRABMyo"): + GRABMyo.__init__(self, dataset_folder=dataset_folder, baseline=True) + + def prepare_data(self, split = True, subjects = None): + forearm_data = self.get_odh(subjects) + forearm_data = forearm_data.isolate_data('sessions', [0]) + + train_data = forearm_data.isolate_data('reps', [0,1,2,3,4], fast=True) + test_data = forearm_data.isolate_data('reps', [5,6], fast=True) + + data = forearm_data + if split: + data = {'All': forearm_data, 'Train': train_data, 'Test': test_data} + + return data \ No newline at end of file diff --git a/libemg/_datasets/hyser.py b/libemg/_datasets/hyser.py new file mode 100644 index 00000000..34b94188 --- /dev/null +++ b/libemg/_datasets/hyser.py @@ -0,0 +1,377 @@ +from abc import ABC, abstractmethod +from copy import deepcopy +from pathlib import Path +from typing import Sequence + +import numpy as np + +from libemg.data_handler import RegexFilter, FilePackager, OfflineDataHandler, MetadataFetcher +from libemg._datasets.dataset import Dataset + + +class _Hyser(Dataset, ABC): + def __init__(self, gestures, num_reps, description, dataset_folder, analysis = 'baseline'): + super().__init__( + sampling=2048, + num_channels=256, + recording_device='OT Bioelettronica Quattrocento', + num_subjects=20, + gestures=gestures, + num_reps=num_reps, + description=description, + citation='https://doi.org/10.13026/ym7v-bh53' + ) + subjects = [str(idx + 1).zfill(2) for idx in range(self.num_subjects)] # +1 due to Python indexing + + self.url = 'https://www.physionet.org/content/hd-semg/1.0.0/' + self.dataset_folder = dataset_folder + self.analysis = analysis + self.subjects = subjects + + @property + def common_regex_filters(self): + sessions_values = ['1', '2'] if self.analysis == 'sessions' else ['1'] # only grab first session unless both are desired + filters = [ + RegexFilter(left_bound='subject', right_bound='_session', values=self.subjects, description='subjects'), + RegexFilter(left_bound='_session', right_bound='/', values=sessions_values, description='sessions') + ] + return filters + + def prepare_data(self, split = True, subjects = None): + if (not self.check_exists(self.dataset_folder)): + raise FileNotFoundError(f"Didn't find Hyser data in {self.dataset_folder} directory. Please download the dataset and \ + store it in the appropriate directory before running prepare_data(). See {self.url} for download details.") + return self._prepare_data_helper(split=split, subjects = subjects) + + @abstractmethod + def _prepare_data_helper(self, split = True, subjects = None) -> dict | OfflineDataHandler: + ... + + +class Hyser1DOF(_Hyser): + def __init__(self, dataset_folder: str = 'Hyser1DOF', analysis: str = 'baseline'): + """1 degree of freedom (DOF) Hyser dataset. + + Parameters + ---------- + dataset_folder: str, default='Hyser1DOF' + Directory that contains Hyser 1 DOF dataset. + analysis: str, default='baseline' + Determines which type of data will be extracted and considered train/test splits. If 'baseline', only grabs data from the first session and splits based on + reps. If 'sessions', grabs data from both sessions and return the first session as train and the second session as test. + """ + gestures = {1: 'Thumb', 2: 'Index', 3: 'Middle', 4: 'Ring', 5: 'Little'} + description = 'Hyser 1 DOF dataset. Includes within-DOF finger movements. Ground truth finger forces are recorded for use in finger force regression.' + super().__init__(gestures=gestures, num_reps=3, description=description, dataset_folder=dataset_folder, analysis=analysis) + + def _prepare_data_helper(self, split = True, subjects = None): + subject_list = np.array(list(range(1,21))) + if subjects: + subject_list = subject_list[subjects] + self.subjects = [f'{s:02d}' for s in subject_list] + + filename_filters = deepcopy(self.common_regex_filters) + filename_filters.append(RegexFilter(left_bound='_sample', right_bound='.hea', values=[str(idx + 1) for idx in range(self.num_reps)], description='reps')) + filename_filters.append(RegexFilter(left_bound='_finger', right_bound='_sample', values=['1', '2', '3', '4', '5'], description='finger')) + + regex_filters = deepcopy(filename_filters) + regex_filters.append(RegexFilter(left_bound='1dof_', right_bound='_finger', values=['raw'], description='data_type')) + + metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='/1dof_', right_bound='_finger', values=['force'], description='labels'), + package_function=filename_filters, load='p_signal') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + data = odh + if split: + if self.analysis == 'sessions': + data = {'All': odh, 'Train': odh.isolate_data('sessions', [0], fast=True), 'Test': odh.isolate_data('sessions', [1], fast=True)} + elif self.analysis == 'baseline': + data = {'All': odh, 'Train': odh.isolate_data('reps', [0, 1], fast=True), 'Test': odh.isolate_data('reps', [2], fast=True)} + else: + raise ValueError(f"Unexpected value for analysis. Suported values are sessions, baseline. Got: {self.analysis}.") + return data + + +class HyserNDOF(_Hyser): + def __init__(self, dataset_folder: str = 'HyserNDOF', analysis: str = 'baseline'): + """N degree of freedom (DOF) Hyser dataset. + + Parameters + ---------- + dataset_folder: str, default='HyserNDOF' + Directory that contains Hyser N DOF dataset. + analysis: str, default='baseline' + Determines which type of data will be extracted and considered train/test splits. If 'baseline', only grabs data from the first session and splits based on + reps. If 'sessions', grabs data from both sessions and return the first session as train and the second session as test. + """ + self.finger_combinations = { + 1: 'Thumb + Index', + 2: 'Thumb + Middle', + 3: 'Thumg + Ring', + 4: 'Thumb + Little', + 5: 'Index + Middle', + 6: 'Thumb + Index + Middle', + 7: 'Index + Middle + Ring', + 8: 'Middle + Ring + Little', + 9: 'Index + Middle + Ring + Little', + 10: 'All Fingers', + 11: 'Thumb + Index (Opposing)', + 12: 'Thumb + Middle (Opposing)', + 13: 'Thumg + Ring (Opposing)', + 14: 'Thumb + Little (Opposing)', + 15: 'Index + Middle (Opposing)' + } + description = 'Hyser N DOF dataset. Includes combined finger movements. Ground truth finger forces are recorded for use in finger force regression.' + super().__init__(gestures=self.finger_combinations, num_reps=2, description=description, dataset_folder=dataset_folder, analysis=analysis) + + def _prepare_data_helper(self, split = True, subjects = None) -> dict | OfflineDataHandler: + subject_list = np.array(list(range(1,21))) + if subjects: + subject_list = subject_list[subjects] + self.subjects = [f'{s:02d}' for s in subject_list] + + filename_filters = deepcopy(self.common_regex_filters) + filename_filters.append(RegexFilter(left_bound='_sample', right_bound='.hea', values=[str(idx + 1) for idx in range(self.num_reps)], description='reps')) + filename_filters.append(RegexFilter(left_bound='_combination', right_bound='_sample', values=[str(idx + 1) for idx in range(len(self.finger_combinations))], description='finger_combinations')) + + regex_filters = deepcopy(filename_filters) + regex_filters.append(RegexFilter(left_bound='/ndof_', right_bound='_combination', values=['raw'], description='data_type')) + + metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='/ndof_', right_bound='_combination', values=['force'], description='labels'), + package_function=filename_filters, load='p_signal') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + data = odh + if split: + if self.analysis == 'sessions': + data = {'All': odh, 'Train': odh.isolate_data('sessions', [0], fast=True), 'Test': odh.isolate_data('sessions', [1], fast=True)} + elif self.analysis == 'baseline': + data = {'All': odh, 'Train': odh.isolate_data('reps', [0], fast=True), 'Test': odh.isolate_data('reps', [1], fast=True)} + else: + raise ValueError(f"Unexpected value for analysis. Suported values are sessions, baseline. Got: {self.analysis}.") + + return data + + +class HyserRandom(_Hyser): + def __init__(self, dataset_folder: str = 'HyserRandom', analysis: str = 'baseline'): + """Random task (DOF) Hyser dataset. + + Parameters + ---------- + dataset_folder: str, default='HyserRandom' + Directory that contains Hyser random task dataset. + analysis: str, default='baseline' + Determines which type of data will be extracted and considered train/test splits. If 'baseline', only grabs data from the first session and splits based on + reps. If 'sessions', grabs data from both sessions and return the first session as train and the second session as test. + """ + description = 'Hyser random dataset. Includes random motions performed by users. Ground truth finger forces are recorded for use in finger force regression.' + super().__init__(gestures={}, num_reps=5, description=description, dataset_folder=dataset_folder, analysis=analysis) + self.num_subjects = 19 + + + def _prepare_data_helper(self, split = True, subjects = None) -> dict | OfflineDataHandler: + subject_list = np.delete(np.array(list(range(1,21))), 9) + if subjects: + subject_list = subject_list[subjects] + self.subjects = [f'{s:02d}' for s in subject_list] + + filename_filters = deepcopy(self.common_regex_filters) + filename_filters.append(RegexFilter(left_bound='_sample', right_bound='.hea', values=[str(idx + 1) for idx in range(self.num_reps)], description='reps')) + + regex_filters = deepcopy(filename_filters) + regex_filters.append(RegexFilter(left_bound='/random_', right_bound='_sample', values=['raw'], description='data_type')) + + metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='/random_', right_bound='_sample', values=['force'], description='labels'), + package_function=filename_filters, load='p_signal') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + + data = odh + if split: + if self.analysis == 'sessions': + data = {'All': odh, 'Train': odh.isolate_data('sessions', [0], fast=True), 'Test': odh.isolate_data('sessions', [1], fast=True)} + elif self.analysis == 'baseline': + data = {'All': odh, 'Train': odh.isolate_data('reps', [0, 1, 2], fast=True), 'Test': odh.isolate_data('reps', [3, 4], fast=True)} + else: + raise ValueError(f"Unexpected value for analysis. Suported values are sessions, baseline. Got: {self.analysis}.") + + return data + + +class _PRLabelsFetcher(MetadataFetcher): + def __init__(self): + super().__init__(description='classes') + self.sample_regex = RegexFilter(left_bound='_sample', right_bound='.hea', values=[str(idx + 1) for idx in range(204)], description='samples') + + def _get_labels(self, filename): + label_filename_map = { + 'dynamic': 'label_dynamic.txt', + 'maintenance': 'label_maintenance.txt' + } + matches = [] + for task_type, labels_file in label_filename_map.items(): + if task_type in filename: + matches.append(labels_file) + + assert len(matches) == 1, f"Expected a single label file for this file, but got {len(matches)}. Got filename: {filename}. Filename should contain either 'dynamic' or 'maintenance'." + + labels_file = matches[0] + parent = Path(filename).absolute().parent + labels_file = Path(parent, labels_file).as_posix() + return np.loadtxt(labels_file, delimiter=',', dtype=int) + + def __call__(self, filename, file_data, all_files): + labels = self._get_labels(filename) + sample_idx = self.sample_regex.get_metadata(filename) + return labels[sample_idx] - 1 # -1 to produce 0-indexed labels + + +class _PRRepFetcher(_PRLabelsFetcher): + def __init__(self): + super().__init__() + self.description = 'reps' + + def __call__(self, filename, file_data, all_files): + label = super().__call__(filename, file_data, all_files) + 1 # +1 b/c this returns 0-indexed labels, but the files are 1-indexed + labels = self._get_labels(filename) + same_label_mask = np.where(labels == label)[0] + sample_idx = self.sample_regex.get_metadata(filename) + rep_idx = list(same_label_mask).index(sample_idx) + if 'dynamic' in filename: + # Each trial is 3 dynamic reps, 1 maintenance rep + rep_idx = rep_idx // 3 + + assert rep_idx <= 1, f"Rep values should be 0 or 1 (2 total reps). Got: {rep_idx}." + return np.array(rep_idx) + + +class HyserPR(_Hyser): + def __init__(self, dataset_folder: str = 'HyserPR', analysis: str = 'baseline'): + """Pattern recognition (PR) Hyser dataset. + + Parameters + ---------- + dataset_folder: str, default='HyserPR' + Directory that contains Hyser PR dataset. + analysis: str, default='baseline' + Determines which type of data will be extracted and considered train/test splits. If 'baseline', only grabs data from the first session and splits based on + reps. If 'sessions', grabs data from both sessions and return the first session as train and the second session as test. + """ + gestures = { + 1: 'Thumb Extension', + 2: 'Index Finger Extension', + 3: 'Middle Finger Extension', + 4: 'Ring Finger Extension', + 5: 'Little Finger Extension', + 6: 'Wrist Flexion', + 7: 'Wrist Extension', + 8: 'Wrist Radial', + 9: 'Wrist Ulnar', + 10: 'Wrist Pronation', + 11: 'Wrist Supination', + 12: 'Extension of Thumb and Index Fingers', + 13: 'Extension of Index and Middle Fingers', + 14: 'Wrist Flexion Combined with Hand Close', + 15: 'Wrist Extension Combined with Hand Close', + 16: 'Wrist Radial Combined with Hand Close', + 17: 'Wrist Ulnar Combined with Hand Close', + 18: 'Wrist Pronation Combined with Hand Close', + 19: 'Wrist Supination Combined with Hand Close', + 20: 'Wrist Flexion Combined with Hand Open', + 21: 'Wrist Extension Combined with Hand Open', + 22: 'Wrist Radial Combined with Hand Open', + 23: 'Wrist Ulnar Combined with Hand Open', + 24: 'Wrist Pronation Combined with Hand Open', + 25: 'Wrist Supination Combined with Hand Open', + 26: 'Extension of Thumb, Index and Middle Fingers', + 27: 'Extension of Index, Middle and Ring Fingers', + 28: 'Extension of Middle, Ring and Little Fingers', + 29: 'Extension of Index, Middle, Ring and Little Fingers', + 30: 'Hand Close', + 31: 'Hand Open', + 32: 'Thumb and Index Fingers Pinch', + 33: 'Thumb, Index and Middle Fingers Pinch', + 34: 'Thumb and Middle Fingers Pinch' + } + description = 'Hyser pattern recognition (PR) dataset. Includes dynamic and maintenance tasks for 34 hand gestures.' + super().__init__(gestures=gestures, num_reps=2, description=description, dataset_folder=dataset_folder, analysis=analysis) # num_reps=2 b/c 2 trials + self.num_subjects = 18 # Removed 2 subjects because they're missing classes + + def _prepare_data_helper(self, split = True, subjects = None) -> dict | OfflineDataHandler: + # Need to remove subjects 3 and 11 b/c they're missing classes + subject_list = np.delete(np.array(list(range(1,21))), [2,10]) + if subjects: + subject_list = subject_list[subjects] + + self.subjects = [f'{s:02d}' for s in subject_list] + + filename_filters = deepcopy(self.common_regex_filters) + filename_filters.append(RegexFilter(left_bound='_sample', right_bound='.hea', values=[str(idx + 1) for idx in range(204)], description='samples')) # max # of dynamic tasks + filename_filters.append(RegexFilter(left_bound='/', right_bound='_', values=['dynamic', 'maintenance'], description='tasks')) + + regex_filters = deepcopy(filename_filters) + regex_filters.append(RegexFilter(left_bound='_', right_bound='_sample', values=['raw'], description='data_type')) + + metadata_fetchers = [ + _PRLabelsFetcher(), + _PRRepFetcher() + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + + data = odh + if split: + if self.analysis == 'sessions': + data = {'All': odh, 'Train': odh.isolate_data('sessions', [0], fast=True), 'Test': odh.isolate_data('sessions', [1], fast=True)} + elif self.analysis == 'baseline': + data = {'All': odh, 'Train': odh.isolate_data('reps', [0], fast=True), 'Test': odh.isolate_data('reps', [1], fast=True)} + else: + raise ValueError(f"Unexpected value for analysis. Suported values are sessions, baseline. Got: {self.analysis}.") + + return data + + +class HyserMVC(_Hyser): + def __init__(self, dataset_folder: str = 'HyserMVC'): + """Maximum voluntary contraction (MVC) Hyser dataset. + + Parameters + ---------- + dataset_folder: str, default='HyserMVC' + Directory that contains the Hyser MVC dataset. + """ + gestures = {1: 'Thumb', 2: 'Index', 3: 'Middle', 4: 'Ring', 5: 'Little'} + description = 'Hyser maximum voluntary contraction (MVC) dataset. Includes MVC for flexion and extension of each finger. Typically used for normalization of other Hyser datasets.' + super().__init__(gestures=gestures, num_reps=5, description=description, dataset_folder=dataset_folder, analysis='sessions') + + def _prepare_data_helper(self, split=True, subjects=None): + subject_list = np.array(list(range(1,21))) + if subjects: + subject_list = subject_list[subjects] + self.subjects = [f'{s:02d}' for s in subject_list] + + filename_filters = deepcopy(self.common_regex_filters) + filename_filters.append(RegexFilter(left_bound='_', right_bound='.hea', values=['flexion', 'extension'], description='movement')) + filename_filters.append(RegexFilter(left_bound='_finger', right_bound='_', values=['1', '2', '3', '4', '5'], description='finger')) + + regex_filters = deepcopy(filename_filters) + regex_filters.append(RegexFilter(left_bound='mvc_', right_bound='_finger', values=['raw'], description='data_type')) + + metadata_fetchers = [ + FilePackager(RegexFilter(left_bound='/mvc_', right_bound='_finger', values=['force'], description='labels'), + package_function=filename_filters, load='p_signal') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + data = odh + if split: + # Split on different sessions (no split for within-session) + data = {'All': odh, 'Train': odh.isolate_data('sessions', [0], fast=True), 'Test': odh.isolate_data('sessions', [1], fast=True)} + return data diff --git a/libemg/_datasets/intensity.py b/libemg/_datasets/intensity.py new file mode 100644 index 00000000..863a40de --- /dev/null +++ b/libemg/_datasets/intensity.py @@ -0,0 +1,44 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class ContractionIntensity(Dataset): + def __init__(self, dataset_folder="ContractionIntensity/"): + Dataset.__init__(self, + 1000, + 8, + 'BE328 by Liberating Technologies, Inc', + 10, + {0: "No Motion", 1: "Wrist Flexion", 2: "Wrist Extension", 3: "Wrist Pronation", 4: "Wrist Supination", 5: "Chuck Grip", 6: "Hand Open"}, + '4 Ramp Reps (Train), 4 Reps x 20%, 30%, 40%, 50%, 60%, 70%, 80%, MVC (Test)', + "A contraction intensity dataset.", + "https://pubmed.ncbi.nlm.nih.gov/23894224/") + self.url = "https://github.com/libemg/ContractionIntensity" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + subject_list = np.array(list(range(1,11))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + intensity_values = ["Ramp", "20P", "30P", "40P", "50P", "60P", "70P", "80P", "MVC"] + classes_values = [str(i) for i in range(1,8)] + reps_values = ["1","2","3","4"] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound="/S", right_bound="/",values=subjects_values, description='subjects'), + RegexFilter(left_bound = "_", right_bound="_C", values = intensity_values, description='intensities'), + RegexFilter(left_bound = "_C", right_bound="_R", values = classes_values, description='classes'), + RegexFilter(left_bound = "_R", right_bound=".csv", values = reps_values, description='reps'), + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("intensities", [0], fast=True), 'Test': odh.isolate_data("intensities", list(range(1, len(intensity_values))), fast=True)} + + return data \ No newline at end of file diff --git a/libemg/_datasets/kaufmann_md.py b/libemg/_datasets/kaufmann_md.py new file mode 100644 index 00000000..368ea203 --- /dev/null +++ b/libemg/_datasets/kaufmann_md.py @@ -0,0 +1,40 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter + +class KaufmannMD(Dataset): + def __init__(self, dataset_folder="MultiDay/"): + Dataset.__init__(self, + 2048, + 4, + 'MindMedia', + 1, + {0: "No Motion", 1:"Wrist Extension", 2:"Wrist Flexion", 3:"Wrist Adduction", + 4:"Wrist Abduction", 5:"Wrist Supination", 6:"Wrist Pronation", 7:"Hand Open", + 8:"Hand Closed", 9:"Key Grip", 10:"Index Point"}, + '1 rep per day, 120 days total. 60/60 train-test split', + "A single subject, multi-day (120) collection.", + "https://ieeexplore.ieee.org/document/5627288") + self.url = "https://github.com/LibEMG/MultiDay" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + subjects_values = ["0"] + day_values = [str(i) for i in range(1,122)] + classes_values = [str(i) for i in range(11)] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound="/S", right_bound="_D",values=subjects_values, description='subjects'), + RegexFilter(left_bound = "_D", right_bound="_C", values = day_values, description='days'), + RegexFilter(left_bound = "_C", right_bound=".csv", values = classes_values, description='classes'), + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=" ") + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("days", list(range(60)), fast=True), 'Test': odh.isolate_data("days", list(range(60,121)), fast=True)} + + return data \ No newline at end of file diff --git a/libemg/_datasets/nina_pro.py b/libemg/_datasets/nina_pro.py new file mode 100644 index 00000000..0a9e30a6 --- /dev/null +++ b/libemg/_datasets/nina_pro.py @@ -0,0 +1,260 @@ +from pathlib import Path + +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter, ColumnFetcher +import os +import scipy.io as sio +import zipfile +import numpy as np +from sklearn.preprocessing import MinMaxScaler + +def find_all_files_of_type_recursively(dir, terminator): + files = os.listdir(dir) + file_list = [] + for file in files: + if file.endswith(terminator): + file_list.append(dir+file) + else: + if os.path.isdir(dir+file): + file_list += find_all_files_of_type_recursively(dir+file+'/',terminator) + return file_list + +class Ninapro(Dataset): + def __init__(self, + sampling, num_channels, recording_device, num_subjects, gestures, num_reps, description, citation, + dataset_folder="Ninapro"): + # downloading the Ninapro dataset is not supported (no permission given from the authors)' + # however, you can download it from http://ninapro.hevs.ch/DB8 + # the subject zip files should be placed at: /NinaproDB8/DB8_s#.zip + Dataset.__init__(self, sampling, num_channels, recording_device, num_subjects, gestures, num_reps, description, citation) + self.dataset_folder = dataset_folder + self.exercise_step = [] + + def convert_to_compatible(self): + # get the zip files (original format they're downloaded in) + zip_files = find_all_files_of_type_recursively(self.dataset_folder,".zip") + # unzip the files -- if any are there (successive runs skip this) + for zip_file in zip_files: + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(zip_file[:-4]+'/') + os.remove(zip_file) + # get the mat files (the files we want to convert to csv) + mat_files = find_all_files_of_type_recursively(self.dataset_folder,".mat") + for mat_file in mat_files: + self.convert_to_csv(mat_file) + + def convert_to_csv(self, mat_file): + # read the mat file + mat_file = mat_file.replace("\\", "/") + mat_dir = mat_file.split('/') + mat_dir = os.path.join(*mat_dir[:-1],"") + mat = sio.loadmat(mat_file) + # get the data + exercise = int(mat_file.split('_')[-1][1]) + exercise_offset = self.exercise_step[exercise-1] # 0 reps already included + data = mat['emg'] + restimulus = mat['restimulus'] + rerepetition = mat['rerepetition'] + try: + cyberglove_data = mat['glove'] + cyberglove_directory = 'cyberglove' + except KeyError: + # No cyberglove data + cyberglove_data = None + cyberglove_directory = '' + if data.shape[0] != restimulus.shape[0]: # this happens in some cases + min_shape = min([data.shape[0], restimulus.shape[0]]) + data = data[:min_shape,:] + restimulus = restimulus[:min_shape,] + rerepetition = rerepetition[:min_shape,] + if cyberglove_data is not None: + cyberglove_data = cyberglove_data[:min_shape,] + # remove 0 repetition - collection buffer + remove_mask = (rerepetition != 0).squeeze() + data = data[remove_mask,:] + restimulus = restimulus[remove_mask] + rerepetition = rerepetition[remove_mask] + if cyberglove_data is not None: + cyberglove_data = cyberglove_data[remove_mask, :] + # important little not here: + # the "rest" really is only the rest between motions, not a dedicated rest class. + # there will be many more rest repetitions (as it is between every class) + # so usually we really care about classifying rest as its important (most of the time we do nothing) + # but for this dataset it doesn't make sense to include (and not its just an offline showcase of the library) + # I encourage you to plot the restimulus to see what I mean. -> plt.plot(restimulus) + # so we remove the rest class too + remove_mask = (restimulus != 0).squeeze() + data = data[remove_mask,:] + restimulus = restimulus[remove_mask] + rerepetition = rerepetition[remove_mask] + if cyberglove_data is not None: + cyberglove_data = cyberglove_data[remove_mask, :] + tail = 0 + while tail < data.shape[0]-1: + rep = rerepetition[tail][0] # remove the 1 offset (0 was the collection buffer) + motion = restimulus[tail][0] # remove the 1 offset (0 was between motions "rest") + # find head + head = np.where(rerepetition[tail:] != rep)[0] + if head.shape == (0,): # last segment of data + head = data.shape[0] -1 + else: + head = head[0] + tail + if cyberglove_data is not None: + # Combine cyberglove and EMG data + data_for_file = np.concatenate((data[tail:head, :], cyberglove_data[tail:head, :]), axis=1) + else: + data_for_file = data[tail:head,:] + + # downsample to 1kHz from 2kHz using decimation + data_for_file = data_for_file[::2, :] + # write to csv + csv_file = Path(mat_dir, cyberglove_directory, f"C{motion - 1}R{rep - 1 + exercise_offset}.csv") + csv_file.parent.mkdir(parents=True, exist_ok=True) + np.savetxt(csv_file, data_for_file, delimiter=',') + tail = head + os.remove(mat_file) + + +class NinaproDB2(Ninapro): + def __init__(self, dataset_folder="NinaProDB2/", use_cyberglove: bool = False): + Ninapro.__init__(self, + 2000, + 12, + 'Delsys', + 40, + {0: 'See Exercises B and C from: https://ninapro.hevs.ch/instructions/DB2.html'}, + '4 Train, 2 Test', + "NinaProb DB2.", + 'https://ninapro.hevs.ch/', + dataset_folder = dataset_folder) + self.exercise_step = [0,0,0] + self.num_cyberglove_dofs = 22 + self.use_cyberglove = use_cyberglove # needed b/c some files have EMG but no cyberglove + + def prepare_data(self, split = True, subjects = None): + subject_list = np.array(list(range(1,41))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + reps_values = [str(i) for i in range(6)] + classes_values = [str(i) for i in range(50)] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + raise FileNotFoundError("Please download the NinaProDB2 dataset from: https://ninapro.hevs.ch/instructions/DB2.html") + self.convert_to_compatible() + regex_filters = [ + RegexFilter(left_bound = "/C", right_bound="R", values = classes_values, description='classes'), + RegexFilter(left_bound="R", right_bound=".csv", values=reps_values, description='reps'), + RegexFilter(left_bound="DB2_s", right_bound="/",values=subjects_values, description='subjects') + ] + + if self.use_cyberglove: + # Only want cyberglove files + regex_filters.append(RegexFilter(left_bound="/", right_bound="/C", values=['cyberglove'], description='')) + metadata_fetchers = [ + ColumnFetcher('cyberglove', column_mask=[idx for idx in range(self.num_channels, self.num_channels + self.num_cyberglove_dofs)]) + ] + else: + metadata_fetchers = None + + emg_column_mask = [idx for idx in range(self.num_channels)] # first columns should be EMG + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers, delimiter=",", data_column=emg_column_mask) + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data('reps', [0,1,2,3], fast=True), 'Test': odh.isolate_data('reps', [4,5], fast=True)} + + return data + +class NinaproDB8(Ninapro): + def __init__(self, dataset_folder="NinaProDB8/", map_to_finger_dofs = True, normalize_labels = True): + # NOTE: This expects each subject's data to be in its own zip file, so the data files for one subject end up in a single directory once we unzip them (e.g., DB8_s1) + gestures = { + 0: "rest", + 1: "thumb flexion/extension", + 2: "thumb abduction/adduction", + 3: "index finger flexion/extension", + 4: "middle finger flexion/extension", + 5: "combined ring and little fingers flexion/extension", + 6: "index pointer", + 7: "cylindrical grip", + 8: "lateral grip", + 9: "tripod grip" + } + + super().__init__( + sampling=1111, + num_channels=16, + recording_device='Delsys Trigno', + num_subjects=12, + gestures=gestures, + num_reps=22, + description='Ninapro DB8 - designed for regression of finger kinematics. Ground truth labels are provided via cyberglove data.', + citation='https://ninapro.hevs.ch/', + dataset_folder=dataset_folder + ) + self.exercise_step = [0,10,20] + self.num_cyberglove_dofs = 18 + self.map_to_finger_dofs = map_to_finger_dofs + self.normalize_labels = normalize_labels + + def _remap_labels(self, odh): + # Linear mapping matrix pulled from original paper: https://www.frontiersin.org/journals/neuroscience/articles/10.3389/fnins.2019.00891/full + finger_map_matrix = np.array([ + [0.639, 0, 0, 0, 0], + [0.383, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [-0.639, 0, 0, 0, 0], + [0, 0, 0.4, 0, 0], + [0, 0, 0.6, 0, 0], + [0, 0, 0, 0.4, 0], + [0, 0, 0, 0.6, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0.1667], + [0, 0, 0, 0, 0.3333], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0.1667], + [0, 0, 0, 0, 0.3333], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [-0.19, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ]) + + remapped_labels = [] + for labels in odh.labels: + finger_labels = np.copy(labels) + if self.map_to_finger_dofs: + finger_labels = labels @ finger_map_matrix + if self.normalize_labels: + finger_labels = MinMaxScaler().fit_transform(finger_labels) + remapped_labels.append(finger_labels) + odh.labels = remapped_labels + return odh + + def prepare_data(self, split = True, subjects = None): + subjects_values = np.array([str(i) for i in range(1,self.num_subjects + 1)]) + if subjects: + subjects_values = subjects_values[subjects] + reps_values = [str(i) for i in range(self.num_reps)] + classes_values = [str(i) for i in range(9)] + + self.convert_to_compatible() + + regex_filters = [ + RegexFilter(left_bound = "/C", right_bound="R", values = classes_values, description='classes'), + RegexFilter(left_bound = "R", right_bound=".csv", values = reps_values, description='reps'), + RegexFilter(left_bound="DB8_s", right_bound="/",values=list(subjects_values), description='subjects') + ] + metadata_fetchers = [ + ColumnFetcher('labels', column_mask=[idx for idx in range(self.num_channels, self.num_channels + self.num_cyberglove_dofs)]) + ] + emg_column_mask = [idx for idx in range(self.num_channels)] # first columns should be EMG + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers, delimiter=",", data_column=emg_column_mask) + odh = self._remap_labels(odh) + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data('reps', [0, 1, 2, 3], fast=True), 'Test': odh.isolate_data('reps', [4, 5], fast=True)} + return data diff --git a/libemg/_datasets/one_subject_emager.py b/libemg/_datasets/one_subject_emager.py new file mode 100644 index 00000000..54dceebd --- /dev/null +++ b/libemg/_datasets/one_subject_emager.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import numpy as np +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter, FilePackager + + +class OneSubjectEMaGerDataset(Dataset): + def __init__(self, dataset_folder = 'OneSubjectEMaGerDataset/'): + super().__init__( + sampling=1010, + num_channels=64, + recording_device='EMaGer', + num_subjects=1, + gestures={0: 'Hand Close (-) / Hand Open (+)', 1: 'Pronation (-) / Supination (+)'}, + num_reps=5, + description='A simple EMaGer dataset used for regression examples in LibEMG demos.', + citation='N/A' + ) + self.url = 'https://github.com/LibEMG/OneSubjectEMaGerDataset' + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + regex_filters = [ + RegexFilter(left_bound='/', right_bound='/', values=['open-close', 'pro-sup'], description='movements'), + RegexFilter(left_bound='_R_', right_bound='_emg.csv', values=[str(idx) for idx in range(self.num_reps)], description='reps') + ] + package_function = lambda x, y: Path(x).parent.absolute() == Path(y).parent.absolute() + metadata_fetchers = [FilePackager(RegexFilter(left_bound='/', right_bound='.txt', values=['labels'], description='labels'), package_function)] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + odh.subjects = [] + odh.subjects = [np.zeros((len(d), 1)) for d in odh.data] + odh.extra_attributes.append('subjects') + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data('reps', [0, 1, 2, 3], fast=True), 'Test': odh.isolate_data('reps', [4], fast=True)} + + return data diff --git a/libemg/_datasets/one_subject_myo.py b/libemg/_datasets/one_subject_myo.py new file mode 100644 index 00000000..1c833e06 --- /dev/null +++ b/libemg/_datasets/one_subject_myo.py @@ -0,0 +1,40 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class OneSubjectMyoDataset(Dataset): + def __init__(self, dataset_folder="OneSubjectMyoDataset/"): + Dataset.__init__(self, + 200, + 8, + 'Myo Armband', + 1, + {0: 'Close', 1: 'Open', 2: 'Rest', 3: 'Flexion', 4: 'Extension'}, + '6 (4 Train, 2 Test)', + "A simple Myo dataset that is used for some of the LibEMG offline demos.", + 'N/A') + self.url = "https://github.com/libemg/OneSubjectMyoDataset" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects=None): + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + sets_values = ["1","2","3","4","5","6"] + classes_values = ["0","1","2","3","4"] + reps_values = ["0","1"] + regex_filters = [ + RegexFilter(left_bound = "/trial_", right_bound="/", values = sets_values, description='sets'), + RegexFilter(left_bound = "C_", right_bound=".csv", values = classes_values, description='classes'), + RegexFilter(left_bound = "R_", right_bound="_", values = reps_values, description='reps') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") + odh.subjects = [] + odh.subjects = [np.zeros((len(d), 1)) for d in odh.data] + odh.extra_attributes.append('subjects') + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("sets", [0,1,2,3,4], fast=True), 'Test': odh.isolate_data("sets", [5,6], fast=True)} + + return data diff --git a/libemg/_datasets/radmand_lp.py b/libemg/_datasets/radmand_lp.py new file mode 100644 index 00000000..c40883c2 --- /dev/null +++ b/libemg/_datasets/radmand_lp.py @@ -0,0 +1,44 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class RadmandLP(Dataset): + def __init__(self, dataset_folder="LimbPosition/"): + Dataset.__init__(self, + 1000, + 6, + 'DelsysTrigno', + 10, + {'N/A': 'Uncertain'}, + '4 Reps (Train), 4 Reps x 15 Positions', + "A large limb position dataset (with 16 static limb positions).", + "https://pubmed.ncbi.nlm.nih.gov/25570046/") + self.url = "https://github.com/libemg/LimbPosition" + self.dataset_folder = dataset_folder + + def prepare_data(self, split = True, subjects = None): + subject_list = np.array(list(range(1,11))) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + position_values = ["P1", "P2", "P3", "P4", "P5", "P6", "P7", "P8", "P9", "P10", "P11", "P12", "P13", "P14", "P15", "P16"] + classes_values = [str(i) for i in range(1,9)] + reps_values = ["1","2","3","4"] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound="/S", right_bound="/",values=subjects_values, description='subjects'), + RegexFilter(left_bound = "_", right_bound="_R", values = position_values, description='positions'), + RegexFilter(left_bound = "_C", right_bound="_P", values = classes_values, description='classes'), + RegexFilter(left_bound = "_R", right_bound=".csv", values = reps_values, description='reps'), + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder + 'RadmandLimbPosition/', regex_filters=regex_filters, delimiter=",") + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("positions", [0], fast=True), 'Test': odh.isolate_data("positions", list(range(1, len(position_values))), fast=True)} + + return data \ No newline at end of file diff --git a/libemg/_datasets/tmr_shirleyryanabilitylab.py b/libemg/_datasets/tmr_shirleyryanabilitylab.py new file mode 100644 index 00000000..56c001db --- /dev/null +++ b/libemg/_datasets/tmr_shirleyryanabilitylab.py @@ -0,0 +1,94 @@ +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter +import numpy as np + +class TMRShirleyRyanAbilityLab(Dataset): + def __init__(self, dataset_folder="TMR/", desc=''): + Dataset.__init__(self, + 1000, + 32, + 'Ag/AgCl', + 6, + {0:"HandOpen", + 1:"KeyGrip", + 2:"PowerGrip", + 3:"FinePinchOpened", + 4:"FinePinchClosed", + 5:"TripodOpened", + 6:"TripodClosed", + 7:"Tool", + 8:"Hook", + 9:"IndexPoint", + 10:"ThumbFlexion", + 11:"ThumbExtension", + 12:"ThumbAbduction", + 13:"ThumbAdduction", + 14:"IndexFlexion", + 15:"RingFlexion", + 16:"PinkyFlexion", + 17:"WristSupination", + 18:"WristPronation", + 19:"WristFlexion", + 20:"WristExtension", + 21:"RadialDeviation", + 22:"UlnarDeviation", + 23:"NoMotion"}, + 8, + desc, + "https://pmc.ncbi.nlm.nih.gov/articles/PMC9879512/") + self.url = "https://github.com/LibEMG/TMR_ShirleyRyanAbilityLab" + self.dataset_folder = dataset_folder + + def get_odh(self, subjects = None): + subject_list = np.array([1,2,3,4,7,10]) + if subjects: + subject_list = subject_list[subjects] + subjects_values = [str(s) for s in subject_list] + + reps_values = [str(i) for i in range(8)] + classes_values = [str(i) for i in range(24)] + intervention_values = ["preTMR","postTMR"] + + print('\nPlease cite: ' + self.citation+'\n') + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound="/S", right_bound="/",values=subjects_values, description='subjects'), + RegexFilter(left_bound = "_R", right_bound=".txt", values = reps_values, description='reps'), + RegexFilter(left_bound = "/C", right_bound="_R", values = classes_values, description='classes'), + RegexFilter(left_bound = "/", right_bound="/C", values = intervention_values, description='intervention') + ] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") + return odh + +class TMR_Pre(TMRShirleyRyanAbilityLab): + """ + Data from participants pre TMR surgery. + """ + def __init__(self, dataset_folder="TMR/"): + TMRShirleyRyanAbilityLab.__init__(self, dataset_folder=dataset_folder, desc='TMR Dataset: 6 subjects, 8 reps, 24 motions, pre intervention') + + def prepare_data(self, split=True, subjects=None): + odh = self.get_odh(subjects) + odh = odh.isolate_data('intervention', [0]) + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("reps", list(range(6)), fast=True), 'Test': odh.isolate_data("reps", list(range(6,8)), fast=True)} + return data + +class TMR_Post(TMRShirleyRyanAbilityLab): + """ + Data from participants post TMR surgery. + """ + def __init__(self, dataset_folder="TMR/"): + TMRShirleyRyanAbilityLab.__init__(self, dataset_folder=dataset_folder, desc='TMR Dataset: 6 subjects, 8 reps, 24 motions, post intervention') + + def prepare_data(self, split=True, subjects=None): + odh = self.get_odh(subjects) + odh = odh.isolate_data('intervention', [1]) + data = odh + if split: + data = {'All': odh, 'Train': odh.isolate_data("reps", list(range(6)), fast=True), 'Test': odh.isolate_data("reps", list(range(6,8)), fast=True)} + return data \ No newline at end of file diff --git a/libemg/_datasets/user_compliance.py b/libemg/_datasets/user_compliance.py new file mode 100644 index 00000000..a9258844 --- /dev/null +++ b/libemg/_datasets/user_compliance.py @@ -0,0 +1,56 @@ +import numpy as np +from pathlib import Path +from libemg._datasets.dataset import Dataset +from libemg.data_handler import OfflineDataHandler, RegexFilter, FilePackager + +class UserComplianceDataset(Dataset): + def __init__(self, dataset_folder = 'UserComplianceDataset/', analysis = 'baseline'): + super().__init__( + sampling=1010, + num_channels=64, + recording_device='EMaGer', + num_subjects=6, + gestures={0: 'Hand Close (-) / Hand Open (+)', 1: 'Pronation (-) / Supination (+)'}, + num_reps=5, + description='Regression dataset used for investigation into user compliance during mimic training.', + citation='https://conferences.lib.unb.ca/index.php/mec/article/view/2507' + ) + self.url = 'https://github.com/LibEMG/UserComplianceDataset' + self.dataset_folder = dataset_folder + self.analysis = analysis + self.subject_list = np.array(['subject-001', 'subject-002', 'subject-003', 'subject-006', 'subject-007', 'subject-008']) + + def prepare_data(self, split = True, subjects = None): + subject_list = self.subject_list + if subjects: + subject_list = subject_list[subjects] + + if (not self.check_exists(self.dataset_folder)): + self.download(self.url, self.dataset_folder) + + regex_filters = [ + RegexFilter(left_bound='/', right_bound='/', values=['open-close', 'pro-sup'], description='movements'), + RegexFilter(left_bound='_R_', right_bound='.csv', values=[str(idx) for idx in range(self.num_reps)], description='reps'), + RegexFilter(left_bound='/', right_bound='/', values=['anticipation', 'all-or-nothing', 'baseline'], description='behaviours'), + RegexFilter(left_bound='/', right_bound='/', values=list(subject_list), description='subjects') + ] + package_function = lambda x, y: Path(x).parent.absolute() == Path(y).parent.absolute() + metadata_fetchers = [FilePackager(RegexFilter(left_bound='/', right_bound='.txt', values=['labels'], description='labels'), package_function)] + odh = OfflineDataHandler() + odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) + data = odh + if split: + if self.analysis == 'baseline': + data = { + 'All': odh, + 'Train': odh.isolate_data('behaviours', [0, 1], fast=True), + 'Test': odh.isolate_data('behaviours', [2], fast=True) + } + elif self.analysis == 'all-or-nothing': + data = {'All': odh, 'Train': odh.isolate_data('behaviours', [1], fast=True), 'Test': odh.isolate_data('behaviours', [2], fast=True)} + elif self.analysis == 'anticipation': + data = {'All': odh, 'Train': odh.isolate_data('behaviours', [0], fast=True), 'Test': odh.isolate_data('behaviours', [2], fast=True)} + else: + raise ValueError(f"Unexpected value for analysis. Got: {self.analysis}.") + + return data diff --git a/libemg/_gui/_data_collection_panel.py b/libemg/_gui/_data_collection_panel.py index 26a9f7a0..9c6d672e 100644 --- a/libemg/_gui/_data_collection_panel.py +++ b/libemg/_gui/_data_collection_panel.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path import dearpygui.dearpygui as dpg import numpy as np @@ -15,6 +16,7 @@ class DataCollectionPanel: def __init__(self, + online_data_handler, num_reps=3, rep_time=3, media_folder='media/', @@ -22,10 +24,10 @@ def __init__(self, rest_time=2, auto_advance=True, exclude_files=[], - gui = None, video_player_width = 720, video_player_height = 480): + self.online_data_handler = online_data_handler self.num_reps = num_reps self.rep_time = rep_time self.media_folder = media_folder @@ -33,7 +35,6 @@ def __init__(self, self.rest_time = rest_time self.auto_advance=auto_advance self.exclude_files = exclude_files - self.gui = gui self.video_player_width = video_player_width self.video_player_height = video_player_height @@ -107,15 +108,17 @@ def spawn_configuration_window(self): def start_callback(self): - if self.gui.online_data_handler and sum(list(self.gui.online_data_handler.get_data()[1].values())): - self.get_settings() - dpg.delete_item("__dc_configuration_window") - self.cleanup_window("configuration") - media_list = self.gather_media() + if not (self.online_data_handler and sum(list(self.online_data_handler.get_data()[1].values()))): + raise ConnectionError('Attempted to start data collection, but data are not being received. Please ensure the OnlineDataHandler is receiving data.') - self.spawn_collection_thread = threading.Thread(target=self.spawn_collection_window, args=(media_list,)) - self.spawn_collection_thread.start() - # self.spawn_collection_window(media_list) + self.get_settings() + dpg.delete_item("__dc_configuration_window") + self.cleanup_window("configuration") + media_list = self.gather_media() + + self.spawn_collection_thread = threading.Thread(target=self.spawn_collection_window, args=(media_list,)) + self.spawn_collection_thread.start() + # self.spawn_collection_window(media_list) def get_settings(self): self.num_reps = int(dpg.get_value("__dc_num_reps")) @@ -129,8 +132,8 @@ def gather_media(self): # find everything in the media folder files = os.listdir(self.media_folder) files = sorted(files) - valid_files = [file.endswith((".gif",".png",".mp4","jpg")) for file in files] - files = list(compress(files, valid_files)) + labels_files = [file for file in files if file.endswith(('.txt', '.csv'))] + files = [file for file in files if file.endswith((".gif",".png",".mp4","jpg"))] self.num_motions = len(files) collection_conf = [] # make the collection_details.json file @@ -145,18 +148,35 @@ def gather_media(self): with open(Path(self.output_folder, "collection_details.json").absolute().as_posix(), 'w') as f: json.dump(collection_details, f) + for media_file in files: + matching_labels_files = [labels_file for labels_file in labels_files if Path(labels_file).stem == Path(media_file).stem] + if len(matching_labels_files) == 1: + # Copy labels file to data directory + labels_file = matching_labels_files[0] + class_index = [idx for idx, filename in collection_details['class_map'].items() if filename == Path(labels_file).stem] + assert len(class_index) == 1, f"Expected a single matching filename in collection_details.json, but got {len(class_index)} for {labels_file}." + class_index = class_index[0] + labels_new_filename = Path(labels_file).with_stem(f"C_{class_index}").name + shutil.copy(Path(self.media_folder, labels_file).absolute(), Path(self.data_folder, labels_new_filename).absolute()) + # make the media list for SGT progression for rep_index in range(self.num_reps): for class_index, motion_class in enumerate(files): # entry for collection of rep media = Media() media.from_file(Path(self.media_folder, motion_class).absolute().as_posix()) - collection_conf.append([media,motion_class.split('.')[0],class_index,rep_index,self.rep_time]) + + if media.type in ('mp4', 'gif'): + # Automatically calculate length of video + rep_time = media.n_frames / media.fps + else: + rep_time = self.rep_time + collection_conf.append([media, motion_class.split('.')[0], class_index, rep_index, rep_time]) return collection_conf def spawn_collection_window(self, media_list): # open first frame of gif - self.gui.online_data_handler.prepare_smm() + self.online_data_handler.prepare_smm() texture = media_list[0][0].get_dpg_formatted_texture(width=self.video_player_width,height=self.video_player_height) set_texture("__dc_collection_visual", texture, width=self.video_player_width, height=self.video_player_height) @@ -170,6 +190,9 @@ def spawn_collection_window(self, media_list): with dpg.group(horizontal=True): dpg.add_spacer(height=20,width=self.video_player_width/2+30-(7*len("Collection Menu"))/2) dpg.add_text(default_value="Collection Menu") + with dpg.group(horizontal=True): + dpg.add_spacer(tag="__dc_rep_spacer",height=20,width=self.video_player_width/2+30 - (7*len(media_list[0][1]))/2) + dpg.add_text(f"Rep 1 of {self.num_reps}", tag="__dc_rep") with dpg.group(horizontal=True): dpg.add_spacer(tag="__dc_prompt_spacer",height=20,width=self.video_player_width/2+30 - (7*len(media_list[0][1]))/2) dpg.add_text(media_list[0][1], tag="__dc_prompt") @@ -202,15 +225,15 @@ def spawn_collection_window(self, media_list): def run_sgt(self, media_list): self.i = 0 self.advance = True - self.gui.online_data_handler.reset() + self.online_data_handler.reset() while self.i < len(media_list): - self.rep_buffer = {mod:[] for mod in self.gui.online_data_handler.modalities} - self.rep_count = {mod:0 for mod in self.gui.online_data_handler.modalities} + self.rep_buffer = {mod:[] for mod in self.online_data_handler.modalities} + self.rep_count = {mod:0 for mod in self.online_data_handler.modalities} # do the rest if self.rest_time and self.i < len(media_list): self.play_collection_visual(media_list[self.i], active=False) media_list[self.i][0].reset() - self.gui.online_data_handler.reset() + self.online_data_handler.reset() self.play_collection_visual(media_list[self.i], active=True) @@ -218,11 +241,19 @@ def run_sgt(self, media_list): self.save_data(output_path) last_rep = media_list[self.i][3] self.i = self.i+1 - if self.i == len(media_list): - break - current_rep = media_list[self.i][3] + is_final_media = self.i == len(media_list) + if is_final_media: + # At the end of the list, so we must be finished a rep + rep_is_finished = True + current_rep = self.num_reps - 1 + else: + # Check if we've finished a rep + current_rep = media_list[self.i][3] + rep_is_finished = last_rep != current_rep + # pause / redo goes here! - if last_rep != current_rep or (not self.auto_advance): + if rep_is_finished or (not self.auto_advance): + # Show redo / continue buttons self.advance = False dpg.show_item(item="__dc_redo_button") dpg.show_item(item="__dc_continue_button") @@ -232,6 +263,8 @@ def run_sgt(self, media_list): jobs = dpg.get_callback_queue() dpg.run_callbacks(jobs) dpg.configure_app(manual_callback_management=False) + if not is_final_media: + dpg.set_value('__dc_rep', value=f"Rep {media_list[self.i][3] + 1} of {self.num_reps}") def redo_collection_callback(self): if self.auto_advance: @@ -249,7 +282,7 @@ def continue_collection_callback(self): def play_collection_visual(self, media, active=True): if active: - timer_duration = self.rep_time + timer_duration = media[-1] dpg.set_value("__dc_prompt", value=media[1]) dpg.set_item_width("__dc_prompt_spacer",width=self.video_player_width/2+30 - (7*len(media[1]))/2) else: @@ -272,8 +305,8 @@ def play_collection_visual(self, media, active=True): progress = min(1,(time.perf_counter_ns() - motion_timer)/(1e9*timer_duration)) # grab incoming new data if active: - vals, count = self.gui.online_data_handler.get_data() - for mod in self.gui.online_data_handler.modalities: + vals, count = self.online_data_handler.get_data() + for mod in self.online_data_handler.modalities: new_samples = count[mod][0][0]-self.rep_count[mod] self.rep_buffer[mod] = [vals[mod][:new_samples,:]] + self.rep_buffer[mod] self.rep_count[mod] = self.rep_count[mod] + new_samples @@ -286,6 +319,8 @@ def save_data(self, filename): for mod in self.rep_buffer: filename = file_parts[0] + "_" + mod + "." + file_parts[1] data = np.vstack(self.rep_buffer[mod])[::-1,:] + if data.size == 0: + raise ConnectionError('Attempting to store data, but received 0 samples during repetition, suggesting that the data stream from the device has been interrupted. Please check the device connection and verify that previous files are not missing samples.') with open(filename, "w", newline='', encoding='utf-8') as file: writer = csv.writer(file) for row in data: @@ -296,4 +331,4 @@ def visualize_callback(self): self.visualization_thread.start() def _run_visualization_helper(self): - self.gui.online_data_handler.visualize(block=False) + self.online_data_handler.visualize(block=False) diff --git a/libemg/_streamers/__init__.py b/libemg/_streamers/__init__.py index 79da57c6..3e89541d 100644 --- a/libemg/_streamers/__init__.py +++ b/libemg/_streamers/__init__.py @@ -8,6 +8,5 @@ from libemg._streamers import _OTB_MuoviPlus from libemg._streamers import _OTB_SessantaquattroPlus from libemg._streamers import _OTB_Syncstation -from libemg._streamers import _oymotion_windows_streamer from libemg._streamers import _emager_streamer from libemg._streamers import _leap_streamer diff --git a/libemg/_streamers/_delsys_API_streamer.py b/libemg/_streamers/_delsys_API_streamer.py index 88051d17..ac52cf59 100644 --- a/libemg/_streamers/_delsys_API_streamer.py +++ b/libemg/_streamers/_delsys_API_streamer.py @@ -1,12 +1,8 @@ -""" -This is the class that handles the data that is output from the Delsys Trigno Base. -Create an instance of this and pass it a reference to the Trigno base for initialization. -See CollectDataController.py for a usage example. -""" -import numpy as np from libemg.shared_memory_manager import SharedMemoryManager from multiprocessing import Process, Event, Lock +import numpy as np + class DataKernel(): def __init__(self, trigno_base): self.TrigBase = trigno_base diff --git a/libemg/_streamers/_delsys_streamer.py b/libemg/_streamers/_delsys_streamer.py index ca268161..d2ac8164 100644 --- a/libemg/_streamers/_delsys_streamer.py +++ b/libemg/_streamers/_delsys_streamer.py @@ -135,7 +135,7 @@ def write_imu(imu): data = np.asarray(struct.unpack('<'+'f'*16, packet)) data = data[self.channel_list] if len(data.shape)==1: - data = data[:, None] + data = data[None, :] for e in self.emg_handlers: e(data) if self.imu: @@ -188,4 +188,4 @@ def _validate(response): if 'OK' not in s: print("warning: TrignoDaq command failed: {}".format(s)) - \ No newline at end of file + diff --git a/libemg/_streamers/_oymotion_streamer.py b/libemg/_streamers/_oymotion_streamer.py index 8bdabada..b0be94f9 100644 --- a/libemg/_streamers/_oymotion_streamer.py +++ b/libemg/_streamers/_oymotion_streamer.py @@ -1,823 +1,697 @@ -# OyMotionStreamer begins here ------ -import socket -import pickle -import time +import asyncio import struct -import numpy as np - -socket_ = None -ip_ = None -port_ = None - -def set_cmd_cb(resp): - print('Command result: {}'.format(resp)) - -def ondata(data): - global socket_ - global ip_ - global port_ - if len(data) > 0: - if data[0] == NotifDataType['NTF_EMG_ADC_DATA'] and len(data) == 129: - emg = np.array(list(data[1:])).reshape(128 // 8,8) - for e in emg: - emg_arr = pickle.dumps(list(e)) - socket_.sendto(emg_arr, (ip_, port_)) - -class OyMotionStreamer(): - def __init__(self, ip, port, - sampRate=1000, - channelMask=0xFF, - dataLen=128, - resolution=8): - global ip_ - ip_ = ip - global port_ - port_ = port - global socket_ - socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - - - self.sampRate = sampRate - self.channelMask = channelMask - self.dataLen = dataLen - self.resolution = resolution - - def start_stream(self): - - - GF = GForceProfile() +from asyncio import Queue +from contextlib import suppress +from dataclasses import dataclass +from enum import IntEnum +from typing import Optional, Dict, List - # Scan all gforces,return [[num,dev_name,dev_addr,dev_Rssi,dev_connectable],...] - scan_results = GF.scan(5) - - if scan_results == []: - print('No bracelet was found') - return - else: - # TODO: check for gpro in address - addr = scan_results[0][2] - GF.connect(addr) - time.sleep(1) - - - GF.setEmgRawDataConfig(self.sampRate, self.channelMask, self.dataLen, self.resolution, cb=set_cmd_cb, timeout=1000) - GF.setDataNotifSwitch(DataNotifFlags['DNF_EMG_RAW'], set_cmd_cb, 1000) - time.sleep(1) - GF.startDataNotification(ondata) - -## BEGIN HARDWARE SPECIFIC CONFIG -import platform -if platform.system() == 'Linux': - from bluepy import btle - from bluepy.btle import DefaultDelegate, Scanner, Peripheral -from datetime import datetime, timedelta -import struct -from enum import Enum -import threading -import time -import queue +""" +Thanks to @zubaidah93 for providing the source code. +""" +import numpy as np +from bleak import BleakScanner, BLEDevice, AdvertisementData, BleakClient, BleakGATTCharacteristic -class GF_RET_CODE(Enum): - - # Method returns successfully. - GF_SUCCESS = 0, - - # Method returns with a generic error. - GF_ERROR = 1, - - # Given parameters are not match required. - GF_ERROR_BAD_PARAM = 2, - - # Method call is not allowed by the inner state. - GF_ERROR_BAD_STATE = 3, - - # Method is not supported at this time. - GF_ERROR_NOT_SUPPORT = 4, - - # Hub is busying on device scan and cannot fulfill the call. - GF_ERROR_SCAN_BUSY = 5, +SERVICE_GUID = '0000ffd0-0000-1000-8000-00805f9b34fb' +CMD_NOTIFY_CHAR_UUID = 'f000ffe1-0451-4000-b000-000000000000' +DATA_NOTIFY_CHAR_UUID = 'f000ffe2-0451-4000-b000-000000000000' - # Insufficient resource to perform the call. - GF_ERROR_NO_RESOURCE = 6, - # A preset timer is expired. - GF_ERROR_TIMEOUT = 7, +@dataclass +class Characteristic: + uuid: str + service_uuid: str + descriptor_uuids: List[str] - # Target device is busy and cannot fulfill the call. - GF_ERROR_DEVICE_BUSY = 8, - # The retrieving data is not ready yet - GF_ERROR_NOT_READY = 9 +class Command(IntEnum): + GET_PROTOCOL_VERSION = 0x00, + GET_FEATURE_MAP = 0x01, + GET_DEVICE_NAME = 0x02, + GET_MODEL_NUMBER = 0x03, + GET_SERIAL_NUMBER = 0x04, + GET_HW_REVISION = 0x05, + GET_FW_REVISION = 0x06, + GET_MANUFACTURER_NAME = 0x07, + GET_BOOTLOADER_VERSION = 0x0A, + GET_BATTERY_LEVEL = 0x08, + GET_TEMPERATURE = 0x09, -CommandType = dict( - CMD_GET_PROTOCOL_VERSION=0x00, - CMD_GET_FEATURE_MAP=0x01, - CMD_GET_DEVICE_NAME=0x02, - CMD_GET_MODEL_NUMBER=0x03, - CMD_GET_SERIAL_NUMBER=0x04, - CMD_GET_HW_REVISION=0x05, - CMD_GET_FW_REVISION=0x06, - CMD_GET_MANUFACTURER_NAME=0x07, - CMD_GET_BOOTLOADER_VERSION=0x0A, + POWEROFF = 0x1D, + SWITCH_TO_OAD = 0x1E, + SYSTEM_RESET = 0x1F, + SWITCH_SERVICE = 0x20, - CMD_GET_BATTERY_LEVEL=0x08, - CMD_GET_TEMPERATURE=0x09, + SET_LOG_LEVEL = 0x21, + SET_LOG_MODULE = 0x22, + PRINT_KERNEL_MSG = 0x23, + MOTOR_CONTROL = 0x24, + LED_CONTROL_TEST = 0x25, + PACKAGE_ID_CONTROL = 0x26, + SEND_TRAINING_PACKAGE = 0x27, - CMD_POWEROFF=0x1D, - CMD_SWITCH_TO_OAD=0x1E, - CMD_SYSTEM_RESET=0x1F, - CMD_SWITCH_SERVICE=0x20, + GET_ACCELERATE_CAP = 0x30, + SET_ACCELERATE_CONFIG = 0x31, - CMD_SET_LOG_LEVEL=0x21, - CMD_SET_LOG_MODULE=0x22, - CMD_PRINT_KERNEL_MSG=0x23, - CMD_MOTOR_CONTROL=0x24, - CMD_LED_CONTROL_TEST=0x25, - CMD_PACKAGE_ID_CONTROL=0x26, - CMD_SEND_TRAINING_PACKAGE=0x27, + GET_GYROSCOPE_CAP = 0x32, + SET_GYROSCOPE_CONFIG = 0x33, - CMD_GET_ACCELERATE_CAP=0x30, - CMD_SET_ACCELERATE_CONFIG=0x31, + GET_MAGNETOMETER_CAP = 0x34, + SET_MAGNETOMETER_CONFIG = 0x35, - CMD_GET_GYROSCOPE_CAP=0x32, - CMD_SET_GYROSCOPE_CONFIG=0x33, + GET_EULER_ANGLE_CAP = 0x36, + SET_EULER_ANGLE_CONFIG = 0x37, - CMD_GET_MAGNETOMETER_CAP=0x34, - CMD_SET_MAGNETOMETER_CONFIG=0x35, + QUATERNION_CAP = 0x38, + QUATERNION_CONFIG = 0x39, - CMD_GET_EULER_ANGLE_CAP=0x36, - CMD_SET_EULER_ANGLE_CONFIG=0x37, + GET_ROTATION_MATRIX_CAP = 0x3A, + SET_ROTATION_MATRIX_CONFIG = 0x3B, - CMD_GET_QUATERNION_CAP=0x38, - CMD_SET_QUATERNION_CONFIG=0x39, + GET_GESTURE_CAP = 0x3C, + SET_GESTURE_CONFIG = 0x3D, - CMD_GET_ROTATION_MATRIX_CAP=0x3A, - CMD_SET_ROTATION_MATRIX_CONFIG=0x3B, + GET_EMG_RAWDATA_CAP = 0x3E, + SET_EMG_RAWDATA_CONFIG = 0x3F, - CMD_GET_GESTURE_CAP=0x3C, - CMD_SET_GESTURE_CONFIG=0x3D, + GET_MOUSE_DATA_CAP = 0x40, + SET_MOUSE_DATA_CONFIG = 0x41, - CMD_GET_EMG_RAWDATA_CAP=0x3E, - CMD_SET_EMG_RAWDATA_CONFIG=0x3F, + GET_JOYSTICK_DATA_CAP = 0x42, + SET_JOYSTICK_DATA_CONFIG = 0x43, - CMD_GET_MOUSE_DATA_CAP=0x40, - CMD_SET_MOUSE_DATA_CONFIG=0x41, + GET_DEVICE_STATUS_CAP = 0x44, + SET_DEVICE_STATUS_CONFIG = 0x45, - CMD_GET_JOYSTICK_DATA_CAP=0x42, - CMD_SET_JOYSTICK_DATA_CONFIG=0x43, + GET_EMG_RAWDATA_CONFIG = 0x46, - CMD_GET_DEVICE_STATUS_CAP=0x44, - CMD_SET_DEVICE_STATUS_CONFIG=0x45, + SET_DATA_NOTIF_SWITCH = 0x4F, + # Partial command packet, format: [CMD_PARTIAL_DATA, packet number in reverse order, packet content] + MD_PARTIAL_DATA = 0xFF - CMD_GET_EMG_RAWDATA_CONFIG=0x46, - CMD_SET_DATA_NOTIF_SWITCH=0x4F, - # Partial command packet, format: [CMD_PARTIAL_DATA, packet number in reverse order, packet content] - MD_PARTIAL_DATA=0xFF -) - -# Response from remote device -ResponseResult = dict( - RSP_CODE_SUCCESS=0x00, - RSP_CODE_NOT_SUPPORT=0x01, - RSP_CODE_BAD_PARAM=0x02, - RSP_CODE_FAILED=0x03, - RSP_CODE_TIMEOUT=0x04, - # Partial packet, format: [RSP_CODE_PARTIAL_PACKET, packet number in reverse order, packet content] - RSP_CODE_PARTIAL_PACKET=0xFF -) - -DataNotifFlags = dict( +class DataSubscription(IntEnum): # Data Notify All Off - DNF_OFF=0x00000000, + OFF = 0x00000000, # Accelerate On(C.7) - DNF_ACCELERATE=0x00000001, + ACCELERATE = 0x00000001, # Gyroscope On(C.8) - DNF_GYROSCOPE=0x00000002, + GYROSCOPE = 0x00000002, # Magnetometer On(C.9) - DNF_MAGNETOMETER=0x00000004, + MAGNETOMETER = 0x00000004, # Euler Angle On(C.10) - DNF_EULERANGLE=0x00000008, + EULERANGLE = 0x00000008, # Quaternion On(C.11) - DNF_QUATERNION=0x00000010, + QUATERNION = 0x00000010, # Rotation Matrix On(C.12) - DNF_ROTATIONMATRIX=0x00000020, + ROTATIONMATRIX = 0x00000020, # EMG Gesture On(C.13) - DNF_EMG_GESTURE=0x00000040, + EMG_GESTURE = 0x00000040, # EMG Raw Data On(C.14) - DNF_EMG_RAW=0x00000080, + EMG_RAW = 0x00000080, # HID Mouse On(C.15) - DNF_HID_MOUSE=0x00000100, + HID_MOUSE = 0x00000100, # HID Joystick On(C.16) - DNF_HID_JOYSTICK=0x00000200, + HID_JOYSTICK = 0x00000200, # Device Status On(C.17) - DNF_DEVICE_STATUS=0x00000400, + DEVICE_STATUS = 0x00000400, # Device Log On - DNF_LOG=0x00000800, + LOG = 0x00000800, # Data Notify All On - DNF_ALL=0xFFFFFFFF -) - - -class ProfileCharType(Enum): - PROF_SIMPLE_DATA = 0 # simple profile: data char - PROF_DATA_CMD = 1, # data profile: cmd char - PROF_DATA_NTF = 2, # data profile：nty char - PROF_OAD_IDENTIFY = 3, # OAD profile：identify char - PROF_OAD_BLOCK = 4, # OAD profile：block char - PROF_OAD_FAST = 5 # OAD profile：fast char - - -NotifDataType = dict( - NTF_ACC_DATA=0x01, - NTF_GYO_DATA=0x02, - NTF_MAG_DATA=0x03, - NTF_EULER_DATA=0x04, - NTF_QUAT_FLOAT_DATA=0x05, - NTF_ROTA_DATA=0x06, - NTF_EMG_GEST_DATA=0x07, - NTF_EMG_ADC_DATA=0x08, - NTF_HID_MOUSE=0x09, - NTF_HID_JOYSTICK=0x0A, - NTF_DEV_STATUS=0x0B, - NTF_LOG_DATA=0x0C, # Log data - - # Partial packet, format: [NTF_PARTIAL_DATA, packet number in reverse order, packet content] - NTF_PARTIAL_DATA=0xFF -) - -LogLevel = dict( - LOG_LEVEL_DEBUG=0x00, - LOG_LEVEL_INFO=0x01, - LOG_LEVEL_WARN=0x02, - LOG_LEVEL_ERROR=0x03, - LOG_LEVEL_FATAL=0x04, - LOG_LEVEL_NONE=0x05 -) - - -class BluetoothDeviceState(Enum): - disconnected = 0, - connected = 1 - - -SERVICE_GUID = '0000ffd0-0000-1000-8000-00805f9b34fb' -CMD_NOTIFY_CHAR_UUID = 'f000ffe1-0451-4000-b000-000000000000' -DATA_NOTIFY_CHAR_UUID = 'f000ffe2-0451-4000-b000-000000000000' - - -class CommandCallbackTableEntry(): - def __init__(self, _cmd, _timeoutTime, _cb): - self._cmd = _cmd - self._timeoutTime = _timeoutTime - self._cb = _cb - -if platform.system() == 'Linux': - class MyDelegate(btle.DefaultDelegate): - def __init__(self, gforce): - super().__init__() - self.gforce = gforce - self.bluepy_thread = threading.Thread(target=self.bluepy_handler) - self.bluepy_thread.setDaemon(True) - self.bluepy_thread.start() - - def bluepy_handler(self): - while True: - if not self.gforce.send_queue.empty(): - cmd = self.gforce.send_queue.get_nowait() - self.gforce.cmdCharacteristic.write(cmd) - self.gforce.device.waitForNotifications(1) - - def handleNotification(self, cHandle, data): - # check cHandle - # self.gforce.lock.acquire() - if cHandle == self.gforce.cmdCharacteristic.getHandle(): - self.gforce._onResponse(data) - - # check cHandle - if cHandle == self.gforce.notifyCharacteristic.getHandle(): - self.gforce.handleDataNotification(data, self.gforce.onData) - # self.gforce.lock.release() - - -class GForceProfile(): - def __init__(self): - self.device = Peripheral() - self.state = BluetoothDeviceState.disconnected - self.cmdCharacteristic = None - self.notifyCharacteristic = None - self.timer = None - self.cmdMap = {} - self.mtu = None - self.cmdForTimeout = -1 - self.incompleteCmdRespPacket = [] - self.lastIncompleteCmdRespPacketId = 0 - self.incompleteNotifPacket = [] - self.lastIncompleteNotifPacketId = 0 - self.onData = None - self.lock = threading.Lock() - self.send_queue = queue.Queue(maxsize=20) - - def getCharacteristic(self, device, uuid): - ches = device.getCharacteristics() - for ch in ches: - if uuid == str(ch.uuid): - return ch - else: - continue - - # Establishes a connection to the Bluetooth Device. - def connect(self, addr): - self.device.connect(addr) - print('connection succeeded') - - # set mtu - MTU = self.device.setMTU(200) - self.mtu = MTU['mtu'][0] - # self.device.setMTU(self.mtu) - # print('mtu:{}'.format(self.mtu)) - - self.state = BluetoothDeviceState.connected - - self.cmdCharacteristic = self.getCharacteristic( - self.device, CMD_NOTIFY_CHAR_UUID) - self.notifyCharacteristic = self.getCharacteristic( - self.device, DATA_NOTIFY_CHAR_UUID) - - # Listen cmd - self.setNotify(self.cmdCharacteristic, True) - - # Open the listening thread - self.device.setDelegate(MyDelegate(self)) - - # Connect the bracelet with the strongest signal - - def connectByRssi(self): - scanner = Scanner() - devices = scanner.scan(10.0) - rssi_devices = {} - - for dev in devices: - print("Device %s (%s), RSSI=%d dB" % - (dev.addr, dev.addrType, dev.rssi)) - for (_, desc, value) in dev.getScanData(): - print(" %s = %s" % (desc, value)) - if (value == SERVICE_GUID): - rssi_devices[dev.rssi] = dev.addr - - rssi = rssi_devices.keys() - dev_addr = rssi_devices[max(rssi)] - - # connect the bracelet - self.device.connect(dev_addr) - print('connection succeeded') - - # set mtu - MTU = self.device.setMTU(2000) - self.mtu = MTU['mtu'][0] - # self.device.setMTU(self.mtu) - # print('mtu:{}'.format(self.mtu)) - - self.state = BluetoothDeviceState.connected - - self.cmdCharacteristic = self.getCharacteristic( - self.device, CMD_NOTIFY_CHAR_UUID) - self.notifyCharacteristic = self.getCharacteristic( - self.device, DATA_NOTIFY_CHAR_UUID) - - # Listen cmd - self.setNotify(self.cmdCharacteristic, True) - - # Open the listening thread - self.device.setDelegate(MyDelegate(self)) - - # Enable a characteristic's notification - def setNotify(self, Chara, swich): - if swich: - setup_data = b"\x01\x00" - else: - setup_data = b"\x00\x00" - - setup_handle = Chara.getHandle() + 1 - self.device.writeCharacteristic( - setup_handle, setup_data, withResponse=False) - - def scan(self, timeout): - scanner = Scanner() - devices = scanner.scan(timeout) - - gforce_scan = [] - i = 1 - for dev in devices: - for (_, _, value) in dev.getScanData(): - if (value == SERVICE_GUID): - gforce_scan.append([i, dev.getValueText( - 9), dev.addr, dev.rssi, str(dev.connectable)]) - i += 1 - return gforce_scan - - # Disconnect from device - def disconnect(self): - - if self.timer != None: - self.timer.cancel() - self.timer = None - # Close the listenThread - - if self.state == BluetoothDeviceState.disconnected: - return True - else: - self.device.disconnect() - self.state == BluetoothDeviceState.disconnected - - # Set data notification flag - def setDataNotifSwitch(self, flags, cb, timeout): - - # Pack data - data = [] - data.append(CommandType['CMD_SET_DATA_NOTIF_SWITCH']) - data.append(0xFF & (flags)) - data.append(0xFF & (flags >> 8)) - data.append(0xFF & (flags >> 16)) - data.append(0xFF & (flags >> 24)) - data = bytes(data) - - def temp(resp, respData): - if cb != None: - cb(resp) - - # Send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - # def switchToOAD(self,cb,timeout): - # # Pack data - # data = [] - # data.append(CommandType['CMD_SWITCH_TO_OAD']) - # data = bytes(data) - # def temp(resp,respData): - # if cb != None: - # cb(resp,None) - - # # Send data - # return self.sendCommand(ProfileCharType.PROF_DATA_CMD,data,True,temp,timeout) - - def powerOff(self, timeout): - # Pack data - data = [] - data.append(CommandType['CMD_POWEROFF']) - data = bytes(data) - - def temp(resp, respData): - pass - - # Send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - def systemReset(self, timeout): - # Pack data - data = [] - data.append(CommandType['CMD_SYSTEM_RESET']) - data = bytes(data) - - def temp(resp, respData): - pass - - # Send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - def setMotor(self, switchStatus, cb, timeout): - data = [] - data.append(CommandType['CMD_MOTOR_CONTROL']) - - tem = 0x01 if switchStatus else 0x00 - data.append(tem) - data = bytes(data) - - def temp(resp, respData): - if cb != None: - cb(resp) - - # send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - def setLED(self, switchStatus, cb, timeout): - data = [] - data.append(CommandType['CMD_LED_CONTROL_TEST']) - - tem = 0x01 if switchStatus else 0x00 - data.append(tem) - data = bytes(data) - - def temp(resp, respData): - if cb != None: - cb(resp) - - # send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - # Get controller firmware version - def setLogLevel(self, logLevel, cb, timeout): - # Pack data - data = [] - data.append(CommandType['CMD_SET_LOG_LEVEL']) - data.append(0xFF & logLevel) - data = bytes(data) - - def temp(resp, respData): - if cb != None: - cb(resp) - - # Send data - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - # Set Emg Raw Data Config - def setEmgRawDataConfig(self, sampRate, channelMask, dataLen, resolution, cb, timeout): - # Pack data - data = b'' - data += struct.pack(' 4: - firmwareVersion = respData.decode('ascii') - else: - firmwareVersion = '' - for i in respData: - firmwareVersion += str(i) + '.' - firmwareVersion = firmwareVersion[0:len(firmwareVersion)] - cb(resp, firmwareVersion) - return self.sendCommand(ProfileCharType.PROF_DATA_CMD, data, True, temp, timeout) - - def sendCommand(self, profileCharType, data, hasResponse, cb, timeout): - if hasResponse and cb != None: - cmd = data[0] - - self.lock.acquire() - - if cmd in self.cmdMap.keys(): - self.lock.release() - return GF_RET_CODE.GF_ERROR_DEVICE_BUSY - self.cmdMap[cmd] = CommandCallbackTableEntry( - cmd, datetime.now()+timedelta(milliseconds=timeout), cb) - self._refreshTimer() - self.lock.release() - - if profileCharType == ProfileCharType.PROF_DATA_CMD: - if self.cmdCharacteristic == None: - return GF_RET_CODE.GF_ERROR_BAD_STATE - else: - if len(data) > self.mtu: - contentLen = self.mtu - 2 - packetCount = (len(data)+contentLen-1)//contentLen - startIndex = 0 - buf = [] - - for i in range(packetCount-1, 0, -1): - buf.append(CommandType['CMD_PARTIAL_DATA']) - buf.append(i) - buf += data[startIndex:startIndex+contentLen] - startIndex += contentLen - self.send_queue.put_nowait(buf) - buf.clear() - # Packet end - buf.append(CommandType['CMD_PARTIAL_DATA']) - buf.append(0) - buf += data[startIndex:] - self.send_queue.put_nowait(buf) - else: - self.send_queue.put_nowait(data) - - return GF_RET_CODE.GF_SUCCESS + ALL = 0xFFFFFFFF + + +class DataType(IntEnum): + ACC = 0x01, + GYO = 0x02, + MAG = 0x03, + EULER = 0x04, + QUAT = 0x05, + ROTA = 0x06, + EMG_GEST = 0x07, + EMG_ADC = 0x08, + HID_MOUSE = 0x09, + HID_JOYSTICK = 0x0A, + DEV_STATUS = 0x0B, + LOG = 0x0C, + + PARTIAL = 0xFF + + +class SampleResolution(IntEnum): + BITS_8 = 8, + BITS_12 = 12 + + +class SamplingRate(IntEnum): + HZ_500 = 500, + HZ_650 = 650, + HZ_1000 = 1000 + + +@dataclass +class EmgRawDataConfig: + fs: SamplingRate = SamplingRate.HZ_1000 + channel_mask: int = 0xFF + batch_len: int = 32 + resolution: SampleResolution = SampleResolution.BITS_8 + + def to_bytes(self): + body = b'' + body += struct.pack(' OyMotionStreamer (No GForce device found). N={count}.") + count += 1 + + + if device == None: + raise Exception("LibEMG -> OyMotionStreamer (No GForce device found). Tries Exceeded.") + + def handle_disconnect(_: BleakClient): + for task in asyncio.all_tasks(): + task.cancel() + + client = BleakClient(device, disconnected_callback=handle_disconnect) + await client.connect() + + await client.start_notify( + CMD_NOTIFY_CHAR_UUID, self._on_cmd_response, + ) + + self.client = client + print("LibEMG -> OyMotionStreamer (connected).") + + def _on_data_response(self, q: Queue, bs: bytearray): + bs = bytes(bs) + full_packet = [] + + is_partial_data = bs[0] == ResponseCode.PARTIAL_PACKET + if is_partial_data: + packet_id = bs[1] + if self.packet_id != 0 and self.packet_id != packet_id + 1: + raise Exception("Unexpected packet id: expected {} got {}".format( + self.packet_id + 1, + packet_id, + )) + elif self.packet_id == 0 or self.packet_id > packet_id: + self.packet_id = packet_id + self.data_packet += bs[2:] + + if self.packet_id == 0: + full_packet = self.data_packet + self.data_packet = [] else: - return GF_RET_CODE.GF_ERROR_BAD_PARAM - - # Refresh time,need external self.lock - def _refreshTimer(self): - def cmp_time(cb): - return cb._timeoutTime - - if self.timer != None: - self.timer.cancel() - - self.timer = None - cmdlist = self.cmdMap.values() - - if len(cmdlist) > 0: - cmdlist = sorted(cmdlist, key=cmp_time) - - # Process timeout entries - timeoutTime = None - listlen = len(cmdlist) - - for i in range(listlen): - timeoutTime = cmdlist[0]._timeoutTime - print('_' * 40) - print('system time : ', datetime.now()) - print('timeout time: ', timeoutTime) - print('\ncmd: {0}, timeout: {1}'.format( - hex(cmdlist[0]._cmd), timeoutTime < datetime.now())) - print('_' * 40) - - if timeoutTime > datetime.now(): - self.cmdForTimeout = cmdlist[0]._cmd - ms = int((timeoutTime.timestamp() - - datetime.now().timestamp())*1000) - - if ms <= 0: - ms = 1 - self.timer = threading.Timer(ms/1000, self._onTimeOut) - self.timer.start() - - break - - cmd = cmdlist.pop(0) - - if cmd._cb != None: - cmd._cb(ResponseResult['RSP_CODE_TIMEOUT'], None) + full_packet = bs - def startDataNotification(self, onData): - - self.onData = onData - - try: - self.setNotify(self.notifyCharacteristic, True) - success = True - except: - success = False + if len(full_packet) == 0: + return - if success: - return GF_RET_CODE.GF_SUCCESS + data = None + data_type = DataType(full_packet[0]) + packet = full_packet[1:] + if data_type == DataType.EMG_ADC: + data = self._convert_emg_to_uv(packet) + elif data_type == DataType.ACC: + data = self._convert_acceleration_to_g(packet) + elif data_type == DataType.GYO: + data = self._convert_gyro_to_dps(packet) + elif data_type == DataType.MAG: + data = self._convert_magnetometer_to_ut(packet) + elif data_type == DataType.EULER: + data = self._convert_euler(packet) + elif data_type == DataType.QUAT: + data = self._convert_quaternion(packet) + elif data_type == DataType.ROTA: + data = self._convert_rotation_matrix(packet) else: - return GF_RET_CODE.GF_ERROR_BAD_STATE - - def stopDataNotification(self): - try: - self.setNotify(self.notifyCharacteristic, False) - success = True - except: - success = False - - if success: - return GF_RET_CODE.GF_SUCCESS + raise Exception(f"Unknown data type {data_type}, full packet: {full_packet}") + + q.put_nowait(data) + + def _convert_emg_to_uv(self, data: bytes): + min_voltage = -1.25 + max_voltage = 1.25 + + if self.resolution == SampleResolution.BITS_8: + dtype = np.uint8 + div = 127.0 + sub = 128 + elif self.resolution == SampleResolution.BITS_12: + dtype = np.uint16 + div = 2047.0 + sub = 2048 else: - return GF_RET_CODE.GF_ERROR_BAD_STATE + raise Exception(f"Unsupported resolution {self.resolution}") - def handleDataNotification(self, data, onData): - fullPacket = [] + gain = 1200.0 + conversion_factor = (max_voltage - min_voltage) / gain / div - if len(data) >= 2: - if data[0] == NotifDataType['NTF_PARTIAL_DATA']: - if self.lastIncompleteNotifPacketId != 0 and self.lastIncompleteNotifPacketId != data[1]+1: - print('Error:lastIncompleteNotifPacketId:{0},current packet id:{1}'.format( - self.lastIncompleteNotifPacketId, data[1])) - # How to do with packet loss? - # Must validate packet len in onData callback! + emg_data = (np.frombuffer(data, dtype=dtype).astype(np.float32) - sub) * conversion_factor + num_channels = 8 - if self.lastIncompleteNotifPacketId == 0 or self.lastIncompleteNotifPacketId > data[1]: - # Only accept packet with smaller packet num - self.lastIncompleteNotifPacketId = data[1] - self.incompleteNotifPacket += data[2:] + return emg_data.reshape(-1, num_channels) - if self.lastIncompleteNotifPacketId == 0: - fullPacket = self.incompleteNotifPacket - self.incompleteNotifPacket = [] + @staticmethod + def _convert_acceleration_to_g(data: bytes): + normalizing_factor = 65536.0 - else: - fullPacket = data + acceleration_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor + num_channels = 3 - if len(fullPacket) > 0: - onData(fullPacket) + return acceleration_data.reshape(-1, num_channels) - # Command notification callback - def _onResponse(self, data): - print('_onResponse: data=', data) + @staticmethod + def _convert_gyro_to_dps(data: bytes): + normalizing_factor = 65536.0 - fullPacket = [] + gyro_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor + num_channels = 3 - if len(data) >= 2: - if data[0] == ResponseResult['RSP_CODE_PARTIAL_PACKET']: - if self.lastIncompleteCmdRespPacketId != 0 and self.lastIncompleteCmdRespPacketId != data[1] + 1: - print('Error: _lastIncompletePacketId:{0}, current packet id:{1}' - .format(self.lastIncompleteCmdRespPacketId, data[1])) + return gyro_data.reshape(-1, num_channels) - if (self.lastIncompleteCmdRespPacketId == 0 or self.lastIncompleteCmdRespPacketId > data[1]): - self.lastIncompleteCmdRespPacketId = data[1] - self.incompleteCmdRespPacket += data[2:] - print('_incompleteCmdRespPacket 等于 ', - self.incompleteCmdRespPacket) + @staticmethod + def _convert_magnetometer_to_ut(data: bytes): + normalizing_factor = 65536.0 - if self.lastIncompleteCmdRespPacketId == 0: - fullPacket = self.incompleteCmdRespPacket - self.incompleteCmdRespPacket = [] - else: - fullPacket = data + magnetometer_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor + num_channels = 3 - if fullPacket != None and len(fullPacket) >= 2: - resp = fullPacket[0] - cmd = fullPacket[1] + return magnetometer_data.reshape(-1, num_channels) - # Delete command callback table entry & refresh timer's timeout + @staticmethod + def _convert_euler(data: bytes): - self.lock.acquire() + euler_data = np.frombuffer(data, dtype=np.float32).astype(np.float32) + num_channels = 3 - if cmd > 0 and self.cmdMap.__contains__(cmd): - cb = self.cmdMap[cmd]._cb + return euler_data.reshape(-1, num_channels) - del self.cmdMap[cmd] + @staticmethod + def _convert_quaternion(data: bytes): - self._refreshTimer() + quaternion_data = np.frombuffer(data, dtype=np.float32).astype(np.float32) + num_channels = 4 - if cb != None: - cb(resp, fullPacket[2:]) + return quaternion_data.reshape(-1, num_channels) - self.lock.release() + @staticmethod + def _convert_rotation_matrix(data: bytes): - # Timeout callback function - def _onTimeOut(self): - print('_onTimeOut: _cmdForTimeout={0}, time={1}'.format( - self.cmdForTimeout, datetime.now())) + rotation_matrix_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) + num_channels = 9 - # Delete command callback table entry & refresh timer's timeout + return rotation_matrix_data.reshape(-1, num_channels) - cb = None - self.lock.acquire() + @staticmethod + def _convert_emg_gesture(data: bytes): - if self.cmdForTimeout > 0 and self.cmdMap.__contains__(self.cmdForTimeout): - cb = self.cmdMap[self.cmdForTimeout]._cb - del self.cmdMap[self.cmdForTimeout] + emg_gesture_data = np.frombuffer(data, dtype=np.int16).astype(np.float16) + num_channels = 6 - self._refreshTimer() + return emg_gesture_data.reshape(-1, num_channels) - self.lock.release() + def _on_cmd_response(self, _: BleakGATTCharacteristic, bs: bytearray): + try: + response = self._parse_response(bytes(bs)) + if response.cmd in self.responses: + self.responses[response.cmd].put_nowait( + response.data, + ) + except Exception as e: + raise Exception("Failed to parse response: %s" % e) + + @staticmethod + def _parse_response(res: bytes): + code = int.from_bytes(res[:1], byteorder='big') + code = ResponseCode(code) + + cmd = int.from_bytes(res[1:2], byteorder='big') + cmd = Command(cmd) + + data = res[2:] + + return Response( + code=code, + cmd=cmd, + data=data, + ) + + async def get_protocol_version(self): + buf = await self._send_request(Request( + cmd=Command.GET_PROTOCOL_VERSION, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_feature_map(self): + buf = await self._send_request(Request( + cmd=Command.GET_FEATURE_MAP, + has_res=True, + )) + return int.from_bytes(buf, byteorder='big') # TODO: check if this is correct + + async def get_device_name(self): + buf = await self._send_request(Request( + cmd=Command.GET_DEVICE_NAME, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_firmware_revision(self): + buf = await self._send_request(Request( + cmd=Command.GET_FW_REVISION, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_hardware_revision(self): + buf = await self._send_request(Request( + cmd=Command.GET_HW_REVISION, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_model_number(self): + buf = await self._send_request(Request( + cmd=Command.GET_MODEL_NUMBER, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_serial_number(self): + buf = await self._send_request(Request( + cmd=Command.GET_SERIAL_NUMBER, + has_res=True, + )) + return buf.decode('utf-8') + + async def get_manufacturer_name(self): + buf = await self._send_request(Request( + cmd=Command.GET_MANUFACTURER_NAME, + has_res=True, + )) + + return buf.decode('utf-8') + + async def get_bootloader_version(self): + buf = await self._send_request(Request( + cmd=Command.GET_BOOTLOADER_VERSION, + has_res=True, + )) + + return buf.decode('utf-8') + + async def get_battery_level(self): + buf = await self._send_request(Request( + cmd=Command.GET_BATTERY_LEVEL, + has_res=True, + )) + return int.from_bytes(buf, byteorder='big') + + async def get_temperature(self): + buf = await self._send_request(Request( + cmd=Command.GET_TEMPERATURE, + has_res=True, + )) + return int.from_bytes(buf, byteorder='big') + + async def power_off(self): + await self._send_request(Request( + cmd=Command.POWEROFF, + has_res=False, + )) + + async def switch_to_oad(self): + await self._send_request(Request( + cmd=Command.SWITCH_TO_OAD, + has_res=False, + )) + + async def system_reset(self): + await self._send_request(Request( + cmd=Command.SYSTEM_RESET, + has_res=False, + )) + + async def switch_service(self): + await self._send_request(Request( + cmd=Command.SWITCH_SERVICE, + has_res=False, + )) + + async def set_motor(self): # TODO: check if this works and what it does + await self._send_request(Request( + cmd=Command.MOTOR_CONTROL, + has_res=True, + )) + + async def set_led(self): # TODO: check if this works and what it does + await self._send_request(Request( + cmd=Command.LED_CONTROL_TEST, + has_res=True, + )) + + async def set_log_level(self): + await self._send_request(Request( + cmd=Command.SET_LOG_LEVEL, + has_res=False, + )) + + async def set_log_module(self): + await self._send_request(Request( + cmd=Command.SET_LOG_MODULE, + has_res=False, + )) + + async def print_kernel_msg(self): + await self._send_request(Request( + cmd=Command.PRINT_KERNEL_MSG, + has_res=True, + )) + + async def set_package_id(self): + await self._send_request(Request( + cmd=Command.PACKAGE_ID_CONTROL, + has_res=False, + )) + + async def send_training_package(self): + await self._send_request(Request( + cmd=Command.SEND_TRAINING_PACKAGE, + has_res=False, + )) + + async def set_emg_raw_data_config(self, cfg=EmgRawDataConfig()): + body = cfg.to_bytes() + await self._send_request(Request( + cmd=Command.SET_EMG_RAWDATA_CONFIG, + body=body, + has_res=True, + )) + self.resolution = cfg.resolution + + async def get_emg_raw_data_config(self): + buf = await self._send_request(Request( + cmd=Command.GET_EMG_RAWDATA_CONFIG, + has_res=True, + )) + return EmgRawDataConfig.from_bytes(buf) + + async def set_subscription(self, subscription: DataSubscription): + body = [0xFF & subscription, 0xFF & (subscription >> 8), 0xFF & (subscription >> 16), + 0xFF & (subscription >> 24)] + body = bytes(body) + await self._send_request(Request( + cmd=Command.SET_DATA_NOTIF_SWITCH, + body=body, + has_res=True, + )) + + async def start_streaming(self): + q = Queue() + await self.client.start_notify( + DATA_NOTIFY_CHAR_UUID, + lambda _, data: self._on_data_response(q, data), + ) + print("LibEMG -> OyMotionStreamer (streaming started).") + return q + + - if cb != None: - cb(ResponseResult['RSP_CODE_TIMEOUT'], None) + async def stop_streaming(self): + exceptions = [] + try: + await self.set_subscription(DataSubscription.OFF) + except Exception as e: + exceptions.append(e) + try: + await self.client.stop_notify(DATA_NOTIFY_CHAR_UUID) + except Exception as e: + exceptions.append(e) + try: + await self.client.stop_notify(CMD_NOTIFY_CHAR_UUID) + except Exception as e: + exceptions.append(e) + + if len(exceptions) > 0: + raise Exception("Failed to stop streaming: %s" % exceptions) + print("LibEMG -> OyMotionStreamer (streaming stopped).") + + async def disconnect(self): + with suppress(asyncio.CancelledError): + await self.client.disconnect() + print("LibEMG -> OyMotionStreamer (disconnected).") + + def _get_response_channel(self, cmd: Command): + q = Queue() + self.responses[cmd] = q + return q + + async def _send_request(self, req: Request): + q = None + if req.has_res: + q = self._get_response_channel(req.cmd) + + bs = bytes([req.cmd]) + if req.body is not None: + bs += req.body + await self.client.write_gatt_char(CMD_NOTIFY_CHAR_UUID, bs) + + if not req.has_res: + return None + + return await asyncio.wait_for(q.get(), 5) + + def run(self): + asyncio.run(self.start_stream()) + + async def start_stream(self): + for item in self.shared_memory_items: + self.smm.create_variable(*item) + await self.connect() + await self.set_emg_raw_data_config(self.emg_conf) + await self.set_subscription( + DataSubscription.EMG_RAW + ) + + q = await self.start_streaming() + try: + while True: + if self.signal.is_set(): + break + + for e in await q.get(): + emg = np.expand_dims(np.array(e),0) + self.smm.modify_variable("emg", lambda x: np.vstack((emg, x))[:x.shape[0],:]) + self.smm.modify_variable("emg_count", lambda x: x + emg.shape[0]) + + except Exception as e: + print(f"Errored within LibEMG-> OyMotionStreamer: {e}") + + finally: + await self._cleanup() + quit() + + async def _cleanup(self): + await self.stop_streaming() + await self.disconnect() + self.smm.cleanup() + print("LibEMG -> OyMotionStreamer (smm cleaned up).") + print("LibEMG -> OyMotionStreamer (process ended).") + + def stop(self): + self.signal.set() + self.join() \ No newline at end of file diff --git a/libemg/_streamers/_oymotion_windows_streamer.py b/libemg/_streamers/_oymotion_windows_streamer.py deleted file mode 100644 index b149e8db..00000000 --- a/libemg/_streamers/_oymotion_windows_streamer.py +++ /dev/null @@ -1,670 +0,0 @@ -import asyncio -import struct -from asyncio import Queue -from contextlib import suppress -from dataclasses import dataclass -from enum import IntEnum -from typing import Optional, Dict, List - -""" -Thanks to @zubaidah93 for providing the source code. -""" - -import numpy as np -from bleak import BleakScanner, BLEDevice, AdvertisementData, BleakClient, BleakGATTCharacteristic - -SERVICE_GUID = '0000ffd0-0000-1000-8000-00805f9b34fb' -CMD_NOTIFY_CHAR_UUID = 'f000ffe1-0451-4000-b000-000000000000' -DATA_NOTIFY_CHAR_UUID = 'f000ffe2-0451-4000-b000-000000000000' - - -@dataclass -class Characteristic: - uuid: str - service_uuid: str - descriptor_uuids: List[str] - - -class Command(IntEnum): - GET_PROTOCOL_VERSION = 0x00, - GET_FEATURE_MAP = 0x01, - GET_DEVICE_NAME = 0x02, - GET_MODEL_NUMBER = 0x03, - GET_SERIAL_NUMBER = 0x04, - GET_HW_REVISION = 0x05, - GET_FW_REVISION = 0x06, - GET_MANUFACTURER_NAME = 0x07, - GET_BOOTLOADER_VERSION = 0x0A, - - GET_BATTERY_LEVEL = 0x08, - GET_TEMPERATURE = 0x09, - - POWEROFF = 0x1D, - SWITCH_TO_OAD = 0x1E, - SYSTEM_RESET = 0x1F, - SWITCH_SERVICE = 0x20, - - SET_LOG_LEVEL = 0x21, - SET_LOG_MODULE = 0x22, - PRINT_KERNEL_MSG = 0x23, - MOTOR_CONTROL = 0x24, - LED_CONTROL_TEST = 0x25, - PACKAGE_ID_CONTROL = 0x26, - SEND_TRAINING_PACKAGE = 0x27, - - GET_ACCELERATE_CAP = 0x30, - SET_ACCELERATE_CONFIG = 0x31, - - GET_GYROSCOPE_CAP = 0x32, - SET_GYROSCOPE_CONFIG = 0x33, - - GET_MAGNETOMETER_CAP = 0x34, - SET_MAGNETOMETER_CONFIG = 0x35, - - GET_EULER_ANGLE_CAP = 0x36, - SET_EULER_ANGLE_CONFIG = 0x37, - - QUATERNION_CAP = 0x38, - QUATERNION_CONFIG = 0x39, - - GET_ROTATION_MATRIX_CAP = 0x3A, - SET_ROTATION_MATRIX_CONFIG = 0x3B, - - GET_GESTURE_CAP = 0x3C, - SET_GESTURE_CONFIG = 0x3D, - - GET_EMG_RAWDATA_CAP = 0x3E, - SET_EMG_RAWDATA_CONFIG = 0x3F, - - GET_MOUSE_DATA_CAP = 0x40, - SET_MOUSE_DATA_CONFIG = 0x41, - - GET_JOYSTICK_DATA_CAP = 0x42, - SET_JOYSTICK_DATA_CONFIG = 0x43, - - GET_DEVICE_STATUS_CAP = 0x44, - SET_DEVICE_STATUS_CONFIG = 0x45, - - GET_EMG_RAWDATA_CONFIG = 0x46, - - SET_DATA_NOTIF_SWITCH = 0x4F, - # Partial command packet, format: [CMD_PARTIAL_DATA, packet number in reverse order, packet content] - MD_PARTIAL_DATA = 0xFF - - -class DataSubscription(IntEnum): - # Data Notify All Off - OFF = 0x00000000, - - # Accelerate On(C.7) - ACCELERATE = 0x00000001, - - # Gyroscope On(C.8) - GYROSCOPE = 0x00000002, - - # Magnetometer On(C.9) - MAGNETOMETER = 0x00000004, - - # Euler Angle On(C.10) - EULERANGLE = 0x00000008, - - # Quaternion On(C.11) - QUATERNION = 0x00000010, - - # Rotation Matrix On(C.12) - ROTATIONMATRIX = 0x00000020, - - # EMG Gesture On(C.13) - EMG_GESTURE = 0x00000040, - - # EMG Raw Data On(C.14) - EMG_RAW = 0x00000080, - - # HID Mouse On(C.15) - HID_MOUSE = 0x00000100, - - # HID Joystick On(C.16) - HID_JOYSTICK = 0x00000200, - - # Device Status On(C.17) - DEVICE_STATUS = 0x00000400, - - # Device Log On - LOG = 0x00000800, - - # Data Notify All On - ALL = 0xFFFFFFFF - - -class DataType(IntEnum): - ACC = 0x01, - GYO = 0x02, - MAG = 0x03, - EULER = 0x04, - QUAT = 0x05, - ROTA = 0x06, - EMG_GEST = 0x07, - EMG_ADC = 0x08, - HID_MOUSE = 0x09, - HID_JOYSTICK = 0x0A, - DEV_STATUS = 0x0B, - LOG = 0x0C, - - PARTIAL = 0xFF - - -class SampleResolution(IntEnum): - BITS_8 = 8, - BITS_12 = 12 - - -class SamplingRate(IntEnum): - HZ_500 = 500, - HZ_650 = 650, - HZ_1000 = 1000 - - -@dataclass -class EmgRawDataConfig: - fs: SamplingRate = SamplingRate.HZ_1000 - channel_mask: int = 0xFF - batch_len: int = 32 - resolution: SampleResolution = SampleResolution.BITS_8 - - def to_bytes(self): - body = b'' - body += struct.pack(' packet_id: - self.packet_id = packet_id - self.data_packet += bs[2:] - - if self.packet_id == 0: - full_packet = self.data_packet - self.data_packet = [] - else: - full_packet = bs - - if len(full_packet) == 0: - return - - data = None - data_type = DataType(full_packet[0]) - packet = full_packet[1:] - if data_type == DataType.EMG_ADC: - data = self._convert_emg_to_uv(packet) - elif data_type == DataType.ACC: - data = self._convert_acceleration_to_g(packet) - elif data_type == DataType.GYO: - data = self._convert_gyro_to_dps(packet) - elif data_type == DataType.MAG: - data = self._convert_magnetometer_to_ut(packet) - elif data_type == DataType.EULER: - data = self._convert_euler(packet) - elif data_type == DataType.QUAT: - data = self._convert_quaternion(packet) - elif data_type == DataType.ROTA: - data = self._convert_rotation_matrix(packet) - else: - raise Exception(f"Unknown data type {data_type}, full packet: {full_packet}") - - q.put_nowait(data) - - def _convert_emg_to_uv(self, data: bytes): - min_voltage = -1.25 - max_voltage = 1.25 - - if self.resolution == SampleResolution.BITS_8: - dtype = np.uint8 - div = 127.0 - sub = 128 - elif self.resolution == SampleResolution.BITS_12: - dtype = np.uint16 - div = 2047.0 - sub = 2048 - else: - raise Exception(f"Unsupported resolution {self.resolution}") - - gain = 1200.0 - conversion_factor = (max_voltage - min_voltage) / gain / div - - emg_data = (np.frombuffer(data, dtype=dtype).astype(np.float32) - sub) * conversion_factor - num_channels = 8 - - return emg_data.reshape(-1, num_channels) - - @staticmethod - def _convert_acceleration_to_g(data: bytes): - normalizing_factor = 65536.0 - - acceleration_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor - num_channels = 3 - - return acceleration_data.reshape(-1, num_channels) - - @staticmethod - def _convert_gyro_to_dps(data: bytes): - normalizing_factor = 65536.0 - - gyro_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor - num_channels = 3 - - return gyro_data.reshape(-1, num_channels) - - @staticmethod - def _convert_magnetometer_to_ut(data: bytes): - normalizing_factor = 65536.0 - - magnetometer_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) / normalizing_factor - num_channels = 3 - - return magnetometer_data.reshape(-1, num_channels) - - @staticmethod - def _convert_euler(data: bytes): - - euler_data = np.frombuffer(data, dtype=np.float32).astype(np.float32) - num_channels = 3 - - return euler_data.reshape(-1, num_channels) - - @staticmethod - def _convert_quaternion(data: bytes): - - quaternion_data = np.frombuffer(data, dtype=np.float32).astype(np.float32) - num_channels = 4 - - return quaternion_data.reshape(-1, num_channels) - - @staticmethod - def _convert_rotation_matrix(data: bytes): - - rotation_matrix_data = np.frombuffer(data, dtype=np.int32).astype(np.float32) - num_channels = 9 - - return rotation_matrix_data.reshape(-1, num_channels) - - @staticmethod - def _convert_emg_gesture(data: bytes): - - emg_gesture_data = np.frombuffer(data, dtype=np.int16).astype(np.float16) - num_channels = 6 - - return emg_gesture_data.reshape(-1, num_channels) - - def _on_cmd_response(self, _: BleakGATTCharacteristic, bs: bytearray): - try: - response = self._parse_response(bytes(bs)) - if response.cmd in self.responses: - self.responses[response.cmd].put_nowait( - response.data, - ) - except Exception as e: - raise Exception("Failed to parse response: %s" % e) - - @staticmethod - def _parse_response(res: bytes): - code = int.from_bytes(res[:1], byteorder='big') - code = ResponseCode(code) - - cmd = int.from_bytes(res[1:2], byteorder='big') - cmd = Command(cmd) - - data = res[2:] - - return Response( - code=code, - cmd=cmd, - data=data, - ) - - async def get_protocol_version(self): - buf = await self._send_request(Request( - cmd=Command.GET_PROTOCOL_VERSION, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_feature_map(self): - buf = await self._send_request(Request( - cmd=Command.GET_FEATURE_MAP, - has_res=True, - )) - return int.from_bytes(buf, byteorder='big') # TODO: check if this is correct - - async def get_device_name(self): - buf = await self._send_request(Request( - cmd=Command.GET_DEVICE_NAME, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_firmware_revision(self): - buf = await self._send_request(Request( - cmd=Command.GET_FW_REVISION, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_hardware_revision(self): - buf = await self._send_request(Request( - cmd=Command.GET_HW_REVISION, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_model_number(self): - buf = await self._send_request(Request( - cmd=Command.GET_MODEL_NUMBER, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_serial_number(self): - buf = await self._send_request(Request( - cmd=Command.GET_SERIAL_NUMBER, - has_res=True, - )) - return buf.decode('utf-8') - - async def get_manufacturer_name(self): - buf = await self._send_request(Request( - cmd=Command.GET_MANUFACTURER_NAME, - has_res=True, - )) - - return buf.decode('utf-8') - - async def get_bootloader_version(self): - buf = await self._send_request(Request( - cmd=Command.GET_BOOTLOADER_VERSION, - has_res=True, - )) - - return buf.decode('utf-8') - - async def get_battery_level(self): - buf = await self._send_request(Request( - cmd=Command.GET_BATTERY_LEVEL, - has_res=True, - )) - return int.from_bytes(buf, byteorder='big') - - async def get_temperature(self): - buf = await self._send_request(Request( - cmd=Command.GET_TEMPERATURE, - has_res=True, - )) - return int.from_bytes(buf, byteorder='big') - - async def power_off(self): - await self._send_request(Request( - cmd=Command.POWEROFF, - has_res=False, - )) - - async def switch_to_oad(self): - await self._send_request(Request( - cmd=Command.SWITCH_TO_OAD, - has_res=False, - )) - - async def system_reset(self): - await self._send_request(Request( - cmd=Command.SYSTEM_RESET, - has_res=False, - )) - - async def switch_service(self): - await self._send_request(Request( - cmd=Command.SWITCH_SERVICE, - has_res=False, - )) - - async def set_motor(self): # TODO: check if this works and what it does - await self._send_request(Request( - cmd=Command.MOTOR_CONTROL, - has_res=True, - )) - - async def set_led(self): # TODO: check if this works and what it does - await self._send_request(Request( - cmd=Command.LED_CONTROL_TEST, - has_res=True, - )) - - async def set_log_level(self): - await self._send_request(Request( - cmd=Command.SET_LOG_LEVEL, - has_res=False, - )) - - async def set_log_module(self): - await self._send_request(Request( - cmd=Command.SET_LOG_MODULE, - has_res=False, - )) - - async def print_kernel_msg(self): - await self._send_request(Request( - cmd=Command.PRINT_KERNEL_MSG, - has_res=True, - )) - - async def set_package_id(self): - await self._send_request(Request( - cmd=Command.PACKAGE_ID_CONTROL, - has_res=False, - )) - - async def send_training_package(self): - await self._send_request(Request( - cmd=Command.SEND_TRAINING_PACKAGE, - has_res=False, - )) - - async def set_emg_raw_data_config(self, cfg=EmgRawDataConfig()): - body = cfg.to_bytes() - await self._send_request(Request( - cmd=Command.SET_EMG_RAWDATA_CONFIG, - body=body, - has_res=True, - )) - self.resolution = cfg.resolution - - async def get_emg_raw_data_config(self): - buf = await self._send_request(Request( - cmd=Command.GET_EMG_RAWDATA_CONFIG, - has_res=True, - )) - return EmgRawDataConfig.from_bytes(buf) - - async def set_subscription(self, subscription: DataSubscription): - body = [0xFF & subscription, 0xFF & (subscription >> 8), 0xFF & (subscription >> 16), - 0xFF & (subscription >> 24)] - body = bytes(body) - await self._send_request(Request( - cmd=Command.SET_DATA_NOTIF_SWITCH, - body=body, - has_res=True, - )) - - async def start_streaming(self): - q = Queue() - await self.client.start_notify( - DATA_NOTIFY_CHAR_UUID, - lambda _, data: self._on_data_response(q, data), - ) - return q - - async def stop_streaming(self): - exceptions = [] - try: - await self.set_subscription(DataSubscription.OFF) - except Exception as e: - exceptions.append(e) - try: - await self.client.stop_notify(DATA_NOTIFY_CHAR_UUID) - except Exception as e: - exceptions.append(e) - try: - await self.client.stop_notify(CMD_NOTIFY_CHAR_UUID) - except Exception as e: - exceptions.append(e) - - if len(exceptions) > 0: - raise Exception("Failed to stop streaming: %s" % exceptions) - - async def disconnect(self): - with suppress(asyncio.CancelledError): - await self.client.disconnect() - - def _get_response_channel(self, cmd: Command): - q = Queue() - self.responses[cmd] = q - return q - - async def _send_request(self, req: Request): - q = None - if req.has_res: - q = self._get_response_channel(req.cmd) - - bs = bytes([req.cmd]) - if req.body is not None: - bs += req.body - await self.client.write_gatt_char(CMD_NOTIFY_CHAR_UUID, bs) - - if not req.has_res: - return None - - return await asyncio.wait_for(q.get(), 5) - - def run(self): - asyncio.run(self.start_stream()) - - async def start_stream(self): - for item in self.shared_memory_items: - self.smm.create_variable(*item) - await self.connect() - await self.set_emg_raw_data_config(self.emg_conf) - await self.set_subscription( - DataSubscription.EMG_RAW - ) - print("Connected to Oymotion Cuff!") - - q = await self.start_streaming() - while True: - if self.signal.is_set(): - self.cleanup() - break - try: - for e in await q.get(): - emg = np.expand_dims(np.array(e),0) - self.smm.modify_variable("emg", lambda x: np.vstack((emg, x))[:x.shape[0],:]) - self.smm.modify_variable("emg_count", lambda x: x + emg.shape[0]) - - except: - print("Worker Stopped.") - quit() - - def cleanup(self): - self.disconnect() - print("Oymotion has disconnected.") \ No newline at end of file diff --git a/libemg/_streamers/_sifi_bridge_streamer.py b/libemg/_streamers/_sifi_bridge_streamer.py index eea7735b..9d4c2a3b 100644 --- a/libemg/_streamers/_sifi_bridge_streamer.py +++ b/libemg/_streamers/_sifi_bridge_streamer.py @@ -1,15 +1,10 @@ -import os -import requests -from libemg.shared_memory_manager import SharedMemoryManager -from multiprocessing import Process, Event, Lock -import subprocess -import json +from multiprocessing import Process, Event import numpy as np -import shutil -import json -from semantic_version import Version from collections.abc import Callable -from platform import system + +import sifi_bridge_py as sbp + +from libemg.shared_memory_manager import SharedMemoryManager class SiFiBridgeStreamer(Process): @@ -21,9 +16,8 @@ class SiFiBridgeStreamer(Process): Parameters ---------- - - version : str - The version of the devie ('1_1 for bioarmband, 1_2 or 1_3 for biopoint). + name : sifi_bridge_py.DeviceType | None + The name of the devie (eg BioArmband, BioPoint_v1_2, BioPoint_v1_3, etc.). None to auto-connect to any device. shared_memory_items : list Shared memory configuration parameters for the streamer in format: ["tag", (size), datatype, Lock()]. @@ -37,53 +31,44 @@ class SiFiBridgeStreamer(Process): Turn IMU modality on or off. ppg : bool Turn PPG modality on or off - notch_on : bool - Turn on-system EMG notch filtering on or off. - notch_freq : int - Specify the frequency of the on-system notch filter. - emgfir_on : bool - Turn on-system EMG bandpass filter on or off. - emg_fir : list - The cutoff frequencies of the on-system bandpass filter. - eda_cfg : bool - Turn EDA into high sampling frequency mode (Bioimpedance at high frequency). - fc_lp : int - Bioimpedance bandpass low cutoff frequency. - fc_hp : int - Bioimpedance bandpass upper cutoff frequency. - freq : int - EDA/Bioimpedance sampling frequency. + filtering, default = True + Enable on-device filtering, including bandpass filters and notch filters. + emg_notch_freq, default = 60 + EMG notch filter frequency, useful for eliminating Mains power interference. Can be {None, 50, 60} Hz. + emg_bandpass: tuple + The (lower, higher) cutoff frequencies of the on-system EMG bandpass filter. + eda_bandpass: tuple + The (lower, higher) cutoff frequencies of the on-system EDA/BIOZ bandpass filter. + eda_freq : int + EDA/Bioimpedance injected signal frequency. 0 for DC. streaming : bool Reduce latency by joining packets of different modalities together. - bridge_version : str - Version of sifi bridge to use for PIPE. - mac : str + mac : str | None MAC address of the device to be connected with. - + """ - def __init__(self, - version: str = '1_2', - shared_memory_items: list = [], - ecg: bool = False, - emg: bool = True, - eda: bool = False, - imu: bool = False, - ppg: bool = False, - notch_on: bool = True, - notch_freq: int = 60, - emgfir_on: bool = True, - emg_fir: list = [20, 450], - eda_cfg: bool = True, - fc_lp: int = 0, # low pass eda - fc_hp: int = 5, # high pass eda - freq: int = 250,# eda sampling frequency - streaming: bool = False, - bridge_version: str | None = None, - mac: str | None = None): - + + def __init__( + self, + name: str | None = None, + shared_memory_items: list = [], + ecg: bool = False, + emg: bool = True, + eda: bool = False, + imu: bool = False, + ppg: bool = False, + filtering: bool = True, + emg_notch_freq: int = 60, + emg_bandpass: tuple = (20, 450), + eda_bandpass: tuple = (0, 5), + eda_freq: int = 0, + streaming: bool = False, + mac: str | None = None, + ): + Process.__init__(self, daemon=True) - self.connected=False + self.connected = False self.signal = Event() self.shared_memory_items = shared_memory_items @@ -92,306 +77,248 @@ def __init__(self, self.eda_handlers = [] self.ecg_handlers = [] self.ppg_handlers = [] + + + self.device_name = name + self.ecg = ecg + self.emg = emg + self.eda = eda + self.imu = imu + self.ppg = ppg + self.filtering = filtering + self.emg_notch_freq = emg_notch_freq + self.emg_bandpass = emg_bandpass + self.eda_bandpass = eda_bandpass + self.eda_freq = eda_freq + self.streaming = streaming + self.mac = mac + # connecting to device can either have a string for the name, device class, or mac address. If None is provided, it autoconnects. + self.handle = self.mac if self.mac is not None else self.device_name + + def configure( + self, + ecg: bool = False, + emg: bool = True, + eda: bool = False, + imu: bool = False, + ppg: bool = False, + filtering: bool = True, + notch_freq: int = 60, + emg_bandpass: tuple = (20, 450), + eda_bandpass: tuple = (0, 5), + eda_freq: int = 0, + streaming: bool = False, + ): + self.sb.configure_sensors(ecg, emg, eda, imu, ppg) + + if ecg: + self.sb.configure_ecg(fs=500, + dc_notch=filtering, + mains_notch=notch_freq, + bandpass=filtering, + flo=0, + fhi=30) + + if emg: + self.sb.configure_emg(fs=1600, + dc_notch=filtering, + mains_notch=notch_freq, + bandpass=filtering, + flo=emg_bandpass[0], + fhi=emg_bandpass[1]) - self.prepare_config_message(ecg, emg, eda, imu, ppg, - notch_on, notch_freq, emgfir_on, emg_fir, - eda_cfg, fc_lp, fc_hp, freq, streaming) - self.prepare_connect_message(version, mac) - self.prepare_executable(bridge_version) - - def prepare_config_message(self, - ecg: bool = False, - emg: bool = True, - eda: bool = False, - imu: bool = False, - ppg: bool = False, - notch_on: bool = True, - notch_freq: int = 60, - emgfir_on: bool = True, - emg_fir: list = [20, 450], - eda_cfg: bool = True, - fc_lp: int = 0, # low pass eda - fc_hp: int = 5, # high pass eda - freq: int = 250,# eda sampling frequency - streaming: bool = False,): - self.config_message = "-s ch " + str(int(ecg)) +","+str(int(emg))+","+str(int(eda))+","+str(int(imu))+","+str(int(ppg)) - if notch_on or emgfir_on: - self.config_message += " enable_filters 1 " - if notch_on: - self.config_message += " emg_notch " + str(notch_freq) - else: - self.config_message += " emg_notch 0" - if emgfir_on: - self.config_message += " emg_fir " + str(emg_fir[0]) + "," + str(emg_fir[1]) + "" - else: - self.config_message += " enable_filters 0" - - if eda_cfg: - self.config_message += " eda_cfg " + str(int(fc_lp)) + "," + str(int(fc_hp)) + "," + str(int(freq)) - - if streaming: - self.config_message += " data_mode 1" + if eda: + self.sb.configure_eda(fs=50, + dc_notch=filtering, + mains_notch=notch_freq, + bandpass=filtering, + flo=eda_bandpass[0], + fhi=eda_bandpass[1],) - self.config_message += " tx_power 2" - self.config_message += "\n" - self.config_message = bytes(self.config_message,"UTF-8") - - def prepare_connect_message(self, - version: str, - mac : str): - if mac is not None: - self.connect_message = '-c ' + str(mac) + '\n' - else: - self.connect_message = '-c BioPoint_v' + str(version) + '\n' - self.connect_message = bytes(self.connect_message,"UTF-8") - - def prepare_executable(self, - bridge_version: str): - pltfm = system() - self.executable = f"sifi_bridge%s-{pltfm.lower()}" + ( - ".exe" if pltfm == "Windows" else "" - ) - if bridge_version is None: - # Find the latest upstream version - try: - releases = requests.get( - "https://api.github.com/repos/sifilabs/sifi-bridge-pub/releases", - timeout=5, - ).json() - bridge_version = str( - max([Version(release["tag_name"]) for release in releases]) - ) - except Exception: - # Probably some network error, so try to find an existing version - # Expected to find sifi_bridge-V.V.V-platform in the current directory - for file in os.listdir(): - if not file.startswith("sifi_bridge"): - continue - bridge_version = file.split("-")[1].replace(".exe", "") - if bridge_version is None: - raise ValueError( - "Could not fetch from upstream nor find a version of sifi_bridge to use in the current directory." - ) + if imu: + self.sb.configure_imu(fs=100) + + if ppg: + self.sb.configure_ppg(sps=50) - self.executable = self.executable % ("-" + bridge_version) - - if self.executable not in os.listdir(): - ext = ".zip" if pltfm == "Windows" else ".tar.gz" - arch = None - if pltfm == "Linux": - arch = "x86_64-unknown-linux-gnu" - print( - "Please run in the terminal to indicate this is an executable file! You only need to do this once." - ) - elif pltfm == "Darwin": - arch = "aarch64-apple-darwin" - elif pltfm == "Windows": - arch = "x86_64-pc-windows-gnu" - - # Get Github releases - releases = requests.get( - "https://api.github.com/repos/sifilabs/sifi-bridge-pub/releases", - timeout=5, - ).json() - - # Extract the release matching the requested version - release_idx = [release["tag_name"] for release in releases].index( - bridge_version - ) - assets = releases[release_idx]["assets"] - - # Find the asset that matches the architecture - archive_url = None - for asset in assets: - asset_name = asset["name"] - if arch not in asset_name: - continue - archive_url = asset["browser_download_url"] - if not archive_url: - ValueError(f"No upstream version found for {self.executable}") - print(f"Fetching sifi_bridge from {archive_url}") - - # Fetch and write to disk as a zip file - r = requests.get(archive_url) - zip_path = "sifi_bridge" + ext - with open(zip_path, "wb") as file: - file.write(r.content) - - # Unpack & delete the archive - shutil.unpack_archive(zip_path, "./") - os.remove(zip_path) - extracted_path = f"sifi_bridge-{bridge_version}-{arch}/" - for file in os.listdir(extracted_path): - if not file.startswith("sifi_bridge"): - continue - shutil.move(extracted_path + file, f"./{self.executable}") - shutil.rmtree(extracted_path) + self.sb.configure_temperature(fs=1) - + self.sb.set_low_latency_mode(True) + self.sb.set_ble_power(sbp.BleTxPower.HIGH) + self.sb.set_memory_mode(sbp.MemoryMode.STREAMING) - def start_pipe(self): - # note, for linux you may need to use sudo chmod +x sifi_bridge_linux - self.proc = subprocess.Popen( - ["./" + self.executable], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, + def connect(self): + + while not self.sb.connect(self.handle): + print(f"Could not connect to {self.handle}. Retrying.") + + + self.connected = True + print("Connected to Sifi device.") + + self.configure( + self.ecg, + self.emg, + self.eda, + self.imu, + self.ppg, + self.filtering, + self.emg_notch_freq, + self.emg_bandpass, + self.eda_bandpass, + self.eda_freq, + self.streaming, ) - assert self.proc is not None + self.sb.stop() + self.sb.start() + self.sb.clear_data_buffer() - - def connect(self): - while not self.connected: - self.proc.stdin.write(self.connect_message) - self.proc.stdin.flush() - - ret = self.proc.stdout.readline().decode() - - dat = json.loads(ret) - - if dat["connected"] == 1: - self.connected = True - print("Connected to Sifi device.") - else: - print("Could not connect. Retrying.") - # Setup channels - self.proc.stdin.write(self.config_message) - self.proc.stdin.flush() - - self.proc.stdin.write(b'-cmd 1\n') - self.proc.stdin.flush() - self.proc.stdin.write(b'-cmd 0\n') - self.proc.stdin.flush() - - def add_emg_handler(self, - closure: Callable): + def add_emg_handler(self, closure: Callable): self.emg_handlers.append(closure) - def add_imu_handler(self, - closure: Callable): + def add_imu_handler(self, closure: Callable): self.imu_handlers.append(closure) - def add_ppg_handler(self, - closure: Callable): + def add_ppg_handler(self, closure: Callable): self.ppg_handlers.append(closure) - - def add_ecg_handler(self, - closure: Callable): + + def add_ecg_handler(self, closure: Callable): self.ecg_handlers.append(closure) - - def add_eda_handler(self, - closure: Callable): + + def add_eda_handler(self, closure: Callable): self.eda_handlers.append(closure) - def process_packet(self, - data: str): - packet = np.zeros((14,8)) - if data == "" or data.startswith("sending cmd"): - return - data = json.loads(data) - + def process_packet(self, data: dict): if "data" in list(data.keys()): - if "emg0" in list(data["data"].keys()): # this is multi-channel (armband) emg - emg = np.stack((data["data"]["emg0"], - data["data"]["emg1"], - data["data"]["emg2"], - data["data"]["emg3"], - data["data"]["emg4"], - data["data"]["emg5"], - data["data"]["emg6"], - data["data"]["emg7"] - )).T + if "emg0" in list( + data["data"].keys() + ): # this is multi-channel (armband) emg + emg = np.stack( + ( + data["data"]["emg0"], + data["data"]["emg1"], + data["data"]["emg2"], + data["data"]["emg3"], + data["data"]["emg4"], + data["data"]["emg5"], + data["data"]["emg6"], + data["data"]["emg7"], + ) + ).T + if emg.dtype != 'float64': + # Remove None rows while preserving 2D structure + emg = emg.astype('float64') + emg = emg[[any(~np.isnan(row)) for row in emg]] + for h in self.emg_handlers: h(emg) # print(data['sample_rate']) - if "emg" in list(data["data"].keys()): # This is the biopoint emg - emg = np.expand_dims(np.array(data['data']["emg"]),0).T + if "emg" in list(data["data"].keys()): # This is the biopoint emg + # print(data["data"]["emg"]) + emg = np.expand_dims(np.array(data["data"]["emg"]), 0).T for h in self.emg_handlers: h(emg) - if "acc_x" in list(data["data"].keys()): - imu = np.stack((data["data"]["acc_x"], - data["data"]["acc_y"], - data["data"]["acc_z"], - data["data"]["w"], - data["data"]["x"], - data["data"]["y"], - data["data"]["z"] - )).T + if "ax" in list(data["data"].keys()): + imu = np.stack( + ( + data["data"]["ax"], + data["data"]["ay"], + data["data"]["az"], + data["data"]["qw"], + data["data"]["qx"], + data["data"]["qy"], + data["data"]["qz"], + ) + ).T for h in self.imu_handlers: h(imu) if "eda" in list(data["data"].keys()): - eda = np.expand_dims(np.array(data['data']['eda']),0).T + eda = np.expand_dims(np.array(data["data"]["eda"]), 0).T for h in self.eda_handlers: h(eda) if "ecg" in list(data["data"].keys()): - ecg = np.stack((data["data"]["ecg"], - )).T + ecg = np.stack((data["data"]["ecg"],)).T for h in self.ecg_handlers: h(ecg) - if "b" in list(data["data"].keys()): - if self.old_ppg_packet is None: - self.old_ppg_packet = data - else: - ppg = np.stack((data["data"]["b"] + self.old_ppg_packet["data"]["b"], - data["data"]["g"] + self.old_ppg_packet["data"]["g"], - data["data"]["r"] + self.old_ppg_packet["data"]["r"], - data["data"]["ir"] + self.old_ppg_packet["data"]["ir"] - )).T - self.old_ppg_packet = None - for h in self.ppg_handlers: - h(ppg) - + if "ir" in list(data["data"].keys()): + ppg = np.array([data["data"]["ir"], data["data"]["r"], data["data"]["g"], data["data"]["b"]]).T + for h in self.ppg_handlers: + h(ppg) + def run(self): # process is started beyond this point! + self.sb = sbp.SifiBridge() + self.sb._DEFAULT_REQUEST_TIMEOUT = 10.0 + + self.connect() + + + self.smm = SharedMemoryManager() for item in self.shared_memory_items: self.smm.create_variable(*item) - self.start_pipe() + def write_emg(emg): # update the samples in "emg" - self.smm.modify_variable("emg", lambda x: np.vstack((np.flip(emg,0), x))[:x.shape[0],:]) + self.smm.modify_variable( + "emg", lambda x: np.vstack((np.flip(emg, 0), x))[: x.shape[0], :] + ) # update the number of samples retrieved self.smm.modify_variable("emg_count", lambda x: x + emg.shape[0]) + self.add_emg_handler(write_emg) def write_imu(imu): # update the samples in "imu" - self.smm.modify_variable("imu", lambda x: np.vstack((np.flip(imu,0), x))[:x.shape[0],:]) + self.smm.modify_variable( + "imu", lambda x: np.vstack((np.flip(imu, 0), x))[: x.shape[0], :] + ) # update the number of samples retrieved self.smm.modify_variable("imu_count", lambda x: x + imu.shape[0]) # sock.sendto(data_arr, (self.ip, self.port)) + self.add_imu_handler(write_imu) def write_eda(eda): # update the samples in "eda" - self.smm.modify_variable("eda", lambda x: np.vstack((np.flip(eda,0), x))[:x.shape[0],:]) + self.smm.modify_variable( + "eda", lambda x: np.vstack((np.flip(eda, 0), x))[: x.shape[0], :] + ) # update the number of samples retrieved self.smm.modify_variable("eda_count", lambda x: x + eda.shape[0]) + self.add_eda_handler(write_eda) def write_ppg(ppg): # update the samples in "ppg" - self.smm.modify_variable("ppg", lambda x: np.vstack((np.flip(ppg,0), x))[:x.shape[0],:]) + self.smm.modify_variable( + "ppg", lambda x: np.vstack((np.flip(ppg, 0), x))[: x.shape[0], :] + ) # update the number of samples retrieved self.smm.modify_variable("ppg_count", lambda x: x + ppg.shape[0]) + self.add_ppg_handler(write_ppg) def write_ecg(ecg): # update the samples in "ecg" - self.smm.modify_variable("ecg", lambda x: np.vstack((np.flip(ecg,0), x))[:x.shape[0],:]) + self.smm.modify_variable( + "ecg", lambda x: np.vstack((np.flip(ecg, 0), x))[: x.shape[0], :] + ) # update the number of samples retrieved self.smm.modify_variable("ecg_count", lambda x: x + ecg.shape[0]) + self.add_ecg_handler(write_ecg) - self.connect() - - self.old_ppg_packet = None # required for now since ppg sends non-uniform packet length + self.old_ppg_packet = ( + None # required for now since ppg sends non-uniform packet length + ) while True: try: - data_from_processess = self.proc.stdout.readline().decode() - self.process_packet(data_from_processess) + new_packet = self.sb.get_data() + self.process_packet(new_packet) except Exception as e: print("Error Occurred: " + str(e)) continue @@ -401,47 +328,28 @@ def write_ecg(ecg): print("LibEMG -> SiFiBridgeStreamer (process ended).") def stop_sampling(self): - self.proc.stdin.write(b'-cmd 1\n') - self.proc.stdin.flush() + self.sb.stop() return def turnoff(self): - self.proc.stdin.write(b'-cmd 13\n') - self.proc.stdin.flush() + self.sb.send_command(sbp.DeviceCommand.POWER_OFF) return - + def disconnect(self): - self.proc.stdin.write(b'-d\n') - self.proc.stdin.flush() - while self.connected: - ret = self.proc.stdout.readline().decode() - dat = json.loads(ret) - if 'connected' in dat.keys(): - if dat["connected"] == 0: - self.connected = False + self.connected = self.sb.disconnect()["connected"] return self.connected def deep_sleep(self): - self.proc.stdin.write(b'-cmd 14\n') - self.proc.stdin.flush() + self.sb.send_command(sbp.DeviceCommand.POWER_DEEP_SLEEP) def cleanup(self): - self.stop_sampling() # stop sampling print("LibEMG -> SiFiBridgeStreamer (sampling stopped).") - self.deep_sleep() # stops status packets + self.deep_sleep() # stops status packets print("LibEMG -> SiFiBridgeStreamer (device sleeped).") - self.disconnect() # disconnect + self.disconnect() # disconnect print("LibEMG -> SiFiBridgeStreamer (device disconnected).") - self.proc.kill() + self.sb._bridge.kill() print("LibEMG -> SiFiBridgeStreamer (bridge killed).") self.smm.cleanup() print("LibEMG -> SiFiBridgeStreamer (SMM cleaned up).") - - def __del__(self): - # self.proc.stdin.write(b"-d -q\n") - # print("LibEMG -> SiFiBridgeStreamer (device disconnected).") - # print("LibEMG -> SiFiBridgeStreamer (bridge killed).") - # self.smm.cleanup() - # print("LibEMG -> SiFiBridgeStreamer (SMM cleaned up).") - pass diff --git a/libemg/adaptation/__init__.py b/libemg/adaptation/__init__.py new file mode 100644 index 00000000..4d02e755 --- /dev/null +++ b/libemg/adaptation/__init__.py @@ -0,0 +1 @@ +from libemg.adaptation import _base, managers, memory \ No newline at end of file diff --git a/libemg/adaptation/_base.py b/libemg/adaptation/_base.py new file mode 100644 index 00000000..3645834e --- /dev/null +++ b/libemg/adaptation/_base.py @@ -0,0 +1,223 @@ +import numpy as np +from multiprocessing import Lock +from libemg.output_writer import SharedMemoryOutputWriter, SocketOutputWriter + +def get_edil_adaptation_objects(num_features, num_outputs): + """ + Get the shared memory items necessary for adaptation. This is configured for environment-dependent incremental learning. + + This config prepares the model_inputs and model_outputs to be available via shared memory, and broadcast model_outputs over a UDP port. + + Parameters + ---------- + num_features : int + Number of features for input in the model. + num_outputs : int + Number of outputs of the model (and returned by the environment). + + Returns + ------- + smi + Shared memory items necessary for adaptation. Pass this to the OnlineStreamer (classifier or regressor) + """ + + # Every sharedmemoryoutputwriter should have a mod_fn that describes how to modify its data. + + # there are two places to grab the model input (pre scaler and post scaler) + # for the setup of my experiments, I've worked with the pre-scaler (non-normalized) model inputs. + def mod_fn_input(self, data, info): + new_slice = np.hstack((info['timestamp'],info['model_input_raw'][-1,:])) + input_size = self.smm.variables['model_input']["shape"][0] + data[:] = np.vstack((new_slice, data))[:input_size, :] + return data + + def mod_fn_output(self, data, info): + new_slice = np.hstack((info['timestamp'],info['model_output'])) + input_size = self.smm.variables['model_output']["shape"][0] + data[:] = np.vstack((new_slice, data))[:input_size, :] + return data + + def mod_fn_env(self, data, info): + new_slice = np.hstack((info['timestamp'], info['trial'], info['environment_feedback'])) + input_size = self.smm.variables['environment_feedback']["shape"][0] + data[:] = np.vstack((new_slice, data))[:input_size, :] + return data + + def mod_fn_flags(self, data, number): + data[:] = number + return data + + def mod_fn_flags_count(self, data, number): + data[:] = data[:] + 1 + return data + + + + adapt_flag_smow = SharedMemoryOutputWriter('adapt_flag', (1,1), np.int32, Lock(), mod_fn=mod_fn_flags, mod_fn_count=mod_fn_flags_count) + memory_flag_smow = SharedMemoryOutputWriter('memory_flag', (1,1), np.int32, Lock(), mod_fn=mod_fn_flags, mod_fn_count=mod_fn_flags_count) + active_flag_smow = SharedMemoryOutputWriter('active_flag', (1,1), np.int8, Lock(), mod_fn=mod_fn_flags, mod_fn_count=mod_fn_flags_count) + environment_flag_smow = SharedMemoryOutputWriter('environment_flag', (1,1), np.int32, Lock(), mod_fn=mod_fn_flags, mod_fn_count=mod_fn_flags_count) + + model_output_smow = SharedMemoryOutputWriter('model_output', (100, 1+num_outputs), np.float64, Lock(), mod_fn=mod_fn_output, mod_fn_count=mod_fn_flags_count) + model_input_smow = SharedMemoryOutputWriter('model_input', (100, 1+num_features), np.float64, Lock(), mod_fn=mod_fn_input, mod_fn_count=mod_fn_flags_count) + environment_feedback_smow = SharedMemoryOutputWriter('environment_feedback', (100, 1+1+num_outputs), np.float64, Lock(), mod_fn=mod_fn_env, mod_fn_count=mod_fn_flags_count) + + model_output_sow = SocketOutputWriter("model_output") + + # initial values for flags + adapt_flag_smow.write(0) + memory_flag_smow.write(0) + active_flag_smow.write(1) + environment_flag_smow.write(1) + + model_output_smow.reset() + model_input_smow.reset() + environment_feedback_smow.reset() + + """ + The model receives messages over shared memory via the adapt tag (to notify when a new model is ready to be loaded). + The model receives messages over the active tag (to notify when a model should halt running). + The model writes inputs out via the model_output and model_input tags via shared memory. + model_output contains a timestamp and DoF outputs + model_input contains a timestamp and model inputs (i.e., features in most cases). + The model also writes out the model_output to a socket (UDP) for the environment to read -- this can be changed in the future to use sharedmemory, but the + RegressionController and ClassifierController are not set up to read from shared memory yet. + """ + model_smi = [ + adapt_flag_smow.smm.get_shared_memory_items()[0], # notify the onlinestreamer to load a new model by this int + active_flag_smow.smm.get_shared_memory_items()[0], # notify the online streamer to pause running with this flag + ] + model_ow = [ + model_output_smow, # <- timestamp, DOF1, DOF2 -> + model_input_smow, # <- timestamp, ---INPUTS--- -> + model_output_sow, + ] + + """ + The environment receives messages over the UDP port from the model to let the user take action during the game loop. This is handled pretty manually within the + environment, but could be made more elegant in the future. The environment_smi is then empty, but provided to keep the interface consistent and extensible. + + For adaptation, the environment writes out messages to shared memory via the environment_feedback tag. This contains a timestamp, trial number, and environment feedback (i.e., reward or pseudolabels). + The environment also has a + """ + environment_smi = [] + environment_ow = [ + environment_feedback_smow, # <- timestamp, TRIAL, ENVIRONMENT FEEDBACK -> + environment_flag_smow, # indicate when the environment is alive + ] + + """ + The adaptation manager receives messages from the memory manager to indicate a new slice of memory is ready (default upon a trial being complete). + The adaptation manager also receives messages from the environment to indicate when the environment is alive (i.e., when it is useful to continue adaptation). + + The adaptation manager writes to the model when a new model is ready to be loaded (i.e., when the model should be updated). + """ + adaptation_manager_smi = [ + memory_flag_smow.smm.get_shared_memory_items()[0], + environment_flag_smow.smm.get_shared_memory_items()[0] + ] + adaptation_manager_ow = [ + adapt_flag_smow + ] + + """ + The memory manager receives messages from the model containing the inputs (e.g., EMG features), that will be used for adaptation later. + The memory manager receives messages from the environment in response to its actions to provide feedback to ends up being a pseudo-label. + The memory manager receives messages from the environment to indicate when the environment is alive (i.e., when it is useful to continue adaptation). + + The memory manager outputs a message to the adaptation_manager which specifies the number of memory slices that have been saved thus far (written to .pkl files). + """ + memory_manager_smi = [ + model_input_smow.smm.get_shared_memory_items()[0], + environment_feedback_smow.smm.get_shared_memory_items()[0], # the feedback itself + environment_feedback_smow.smm.get_shared_memory_items()[1], # the number of times the feedback has been written to (useful for only querying the new stuff to be appended to memory) + environment_flag_smow.smm.get_shared_memory_items()[0], + ] + memory_manager_ow = [ + memory_flag_smow + ] + + return model_smi, model_ow, environment_smi, environment_ow, adaptation_manager_smi, adaptation_manager_ow, memory_manager_smi, memory_manager_ow + +def get_pdil_adaptation_items(): + ... + +def get_uil_adaptation_items(): + ... + +def get_dodr_adaptation_items(): + ... + +def produce_tciil_feedback(current_location: list[int, int], + current_direction: list[int, int], + target_location: list[int, int], + target_size: int, + trial_distance: int): + """ + Example function handle to produce a tolerant context informed incremental learning pseudo-label to be used as feedback from a 2-DoF environment. + This is used in the default implementation of libemg.environment.curricular_environment.CurricularFittsEnvironment. This uses the procedure described + in "Context-Informed Incremental Learning Improves Throughput and Reduces Drift in Regression-Based Myoelectric Control", Morrell et al 2025. + + Parameters + ---------- + current location : list + A 2DoF location where the cursor is located. + target location : list + A 2DoF location where the target is located. + target size : int + The size of the target. + trial_distance : int + The distance between the cursor and target at the start of the trial. + """ + + optimal_direction = get_optimal_direction(current_location, target_location) + + PC = distance_to_proportional_control(current_location, target_location, current_direction, target_size, trial_distance) + quadrant_check = check_quadrants(current_location, current_direction, target_location) + + # silence bad directions + pseudo_label = [val if outcome else 0 for val, outcome in zip(optimal_direction, quadrant_check)] + # scale to correct value + pseudo_label_scale = np.linalg.norm(pseudo_label) + pseudo_label = [val * PC / pseudo_label_scale for val in pseudo_label] + # pseudo_label = [0 if np.isnan(i) else i for i in pseudo_label] # remove NaNs (completely wrong quadrant is set to 0,0) + + return pseudo_label + +def get_optimal_direction(current_location: list[int, int], + target_location: list[int, int]) -> list[int, int]: + return [i - j for i, j in zip(target_location, current_location)] + +def distance_to_proportional_control(current_location, target_location, current_direction, target_size, trial_distance) -> float: + distance = np.linalg.norm(np.array(current_location) - np.array(target_location)) + in_target = distance < target_size + if in_target: + # step_in_dir = [ x + 0.01*y for x,y in zip(current_location, current_direction)] + # if np.linalg.norm(np.array(step_in_dir) - np.array(target_location)) < distance: + # PC = 0.05 # if we're still approaching the center of the target, output speed is 0.1 + # else: + PC = 0 # if we're in the circle, but not approaching the center, output speed is 0 + else: + # trial distance makes sense as the normalizer for most shooting tasks, but sometimes with random distance targets, the new target can be very close + # in which case, the user wouldn't hit the max proportinal control value for that trial. + # it probably makes more sense to just normalize by a consant value + #PC = min(1.41, np.sqrt((distance - target_size)/trial_distance)) + calculated_percentile = (distance - target_size) / 300 + calculated_pc = 0.1+0.9/(1+np.exp(-10*(calculated_percentile-0.5))) + PC = min(1., calculated_pc) + # the gameplay region in CurricularFittsLaw is about 1000 pixels, so any distance greater than half the playable + # area should evoke max speed. + return PC + +def check_quadrants(current_location, current_direction, target_location) -> list[bool, bool]: + margins = [abs(i - j) for i, j in zip(target_location, current_location)] + del_margins = [abs(i - (0.01 * k + j)) for i, j, k in zip(target_location, current_location, current_direction)] + outcome = [] + for i, j in zip(margins, del_margins): + if i > j : + # better after the step + outcome.append(True) + else: + # worse after the step + outcome.append(False) + return outcome diff --git a/libemg/adaptation/managers.py b/libemg/adaptation/managers.py new file mode 100644 index 00000000..5b54665c --- /dev/null +++ b/libemg/adaptation/managers.py @@ -0,0 +1,246 @@ +from multiprocessing import Process, Lock, Event +import libemg +from abc import abstractmethod +from typing import Any, Tuple +import pickle +import numpy as np +import time + +class MemoryManager(Process): + """ + This object is part of the adaptation suite provided by LibEMG. The memory manager is responsible for managing the data (both the inputs and pseudolabels) that arise from + a user-in-the-loop setting and storing them in pickled memory classes. These memory classes may represent segments of data that have been collected (e.g., a trial of Fitts' Law). + The memory manager contains a signal to trigger the adaptation manager to load these slices of memory, which is the other half of the adaptation suite provided by LibEMG. + + The memory slices are composed by monitoring shared memory output writers from the environment (which provide a timestamp and pseudo-label), and monitoring shared memory output writers from + the model (which provide an identical timestamp and model inputs such as EMG features). + + Parameters + ---------- + memory: Any + A custom object that defines the memory class. It should have a .append(), .save(), .load(), .reset(), and __add__ operator overload. + smi: list + a list containing information to construct shared memory managers that receive all data necessary to compile memories. Consult libemg.adaptation._base.get__adaptation_items() for more information. + ow: list[libemg.output_writer.OutputWriter] + A list of LibEMG output writers that are used by this class to write information out to other processes (i.e., the adpatation manager). Consult libemg.adaptation._base.get__adaptation_items() for more information. + save_dir: str + The location that memory slices will be saved. If adaptation is actually running, this should be same as the load_dir argument of the libemg.adaptation.managers.AdaptationManager. + """ + def __init__(self, + memory: Any, + smi: list, + ow: list[libemg.output_writer.OutputWriter], + save_dir: str): + Process.__init__(self, daemon=True) + + self.signal = Event() + + self.memory = memory + self.smi = smi + self.ow = ow + self.save_dir = save_dir + + self.environment_feedback_count = 0 + self.trial_counter = 1 + ensure_directory(self.save_dir) + + def run(self): + self.smm = libemg.shared_memory_manager.SharedMemoryManager() + for smi in self.smi: + self.smm.create_variable(*smi) + self.ow[0].write(0) # start at 0 + + self.memory.reset() + while True: + if self.signal.is_set(): + self.memory.save(self.save_dir + "memory_"+str(self.trial_counter) + ".pkl") + break + + # check if there is data to process, and if so process it + self.process_data() + + + def process_data(self): + # Snapshot the feedback buffer and its counter together (they share a + # lock) so num_to_grab is always sliced against the exact buffer state + # it was measured from. Reading them separately lets the writer append + # rows in between, which misaligns the slice and splices in wrong rows. + feedback = self.smm.get_variables(["environment_feedback", "environment_feedback_count"]) + environment_feedback_count = feedback["environment_feedback_count"][0,0] + # if there has been no new environment feedback, just continue + if environment_feedback_count == self.environment_feedback_count: + return + num_to_grab = environment_feedback_count - self.environment_feedback_count + feedback_data = feedback["environment_feedback"] + feedback_data = feedback_data[:num_to_grab,:] + + # model_input is joined to feedback rows by timestamp (not by index), so + # it doesn't need to be part of the atomic snapshot; reading it here also + # keeps it as fresh as possible so matching timestamps are present. + input_data = self.smm.get_variable('model_input') + # for every row in data: + for i in range(feedback_data.shape[0]): + feedback_row = feedback_data[i,:] + trial = feedback_row[1] + if trial != self.trial_counter: + # Save the memory + self.memory.save(self.save_dir + "memory_"+str(int(self.trial_counter)) + ".pkl") + self.ow[0].write(self.trial_counter) + self.trial_counter = trial + # tell the adaptation manager a new slice is ready + + # Start a fresh memory + self.memory.reset() + # Append the data to the memory object + row_timestamp = feedback_row[0] + # find timestamp in classifier_input + timestamp_id = np.where(input_data[:,0]== row_timestamp)[0] + input_row = input_data[timestamp_id,1:] # start at 2nd column to remove timestamp + self.append_to_memory(feedback_row[2:], input_row, trial-1) + + self.environment_feedback_count = environment_feedback_count + + def save_memory(self, loc: str): + with open(loc, 'wb') as f: + pickle.dump(self.memory, f) + + def run_helper(self, block=True): + """ + Helper function to run the process. This is used to avoid blocking the main thread. + """ + if block: + self.run() + else: + self.start() + + @abstractmethod + def setup_memorymanager() -> None: + pass + + @abstractmethod + def get_data() -> Tuple[Any, Any]: + pass + + @abstractmethod + def append_to_memory(self, feedback, input, trial) -> None: + self.memory.append(feedback, input, trial) + + + +class AdaptationManager(Process): + """ + This object is part of the adaptation suite provided by LibEMG. The adaptation manager is responsible for aggregating the slices of data that are packaged by the memory manager, and running + the adaptation of the model given this data. + + Parameters + ---------- + model: Any + A custom object that defines the model. It should have a .adapt, .save(), .load() and .predict() methods. + smi: list + a list containing information to construct shared memory managers that receive messages from the memory manager to update the aggregated data for adptation. Consult libemg.adaptation._base.get__adaptation_items() for more information. + ow: list[libemg.output_writer.OutputWriter] + A list of LibEMG output writers that are used by this class to write information out to other processes (i.e., the OnlineStreamer to load the updated model). Consult libemg.adaptation._base.get__adaptation_items() for more information. + initial_memory_loc: str + The location of the initial memory slice. A typical use for this would be to load data from screen guided training prior to gather user-in-the-loop data. Not entirely necessary, but very beneficial for stability. + load_dir: str + The location that the memory slices will be loaded. This should be the save_dir argument of the libemg.adaptation.managers.MemoryManager. + save_dir: str + The location that the adapted models will be saved. This should be the file_path argument of the libemg.emg_predictor.OnlineStreamer (the live model for the user-in-the-loop setting.) + """ + def __init__(self, + model, + smi: list, + ow: list[libemg.output_writer.OutputWriter], + initial_memory_loc: str|None, + load_dir: str, + save_dir: str, + stop_condition: callable = lambda x: True, + notify: bool = True): + + Process.__init__(self, daemon=True) + + self.signal = Event() + + self.model = model + self.smi = smi + self.ow = ow + self.initial_memory_loc = initial_memory_loc + self.load_dir = load_dir + self.save_dir = save_dir + self.stop_condition = stop_condition + self.notify = notify + + self.memory = self.load_memory(initial_memory_loc) + + self.memory_count = 0 + self.adaptation_count = 0 + + # ensure save and load directories exist + ensure_directory(self.save_dir) + ensure_directory(self.load_dir) + + def run_helper(self, block=True): + """ + Helper function to run the process. This is used to avoid blocking the main thread. + NOTE: if you're training on the GPU, you need to run this in the main loop. This is a CUDA thing and + not something that can be worked around without sacrificing multiprocessing latency elsewhere (streaming). + """ + if block: + self.run() + else: + self.start() + + def run(self): + start_time = time.time() + self.smm = libemg.shared_memory_manager.SharedMemoryManager() + for smi in self.smi: + self.smm.create_variable(*smi) + + while not self.stop_condition(self.memory_count): + + if self.signal.is_set(): + break + + memory_count = self.smm.get_variable("memory_flag")[0,0] + if memory_count > self.memory_count: + num_memories_to_load = memory_count - self.memory_count + for m in range(num_memories_to_load): + # load the next memory + self.memory_count += 1 + # Load memory + new_memory = self.load_memory(self.load_dir + "memory_" + str(self.memory_count) + ".pkl") + print(f"{self.memory_count} : {new_memory.processed_data[0].shape}") + self.memory = self.memory + new_memory + + # Adapt the model + loss_list = self.model.adapt(self.memory) + with open(self.save_dir + "losses.txt", 'a') as f: + f.write(str(time.time() - start_time) + "\t" + str(loss_list) + "\n") + self.adaptation_count += 1 + self.model.save(self.save_dir + "mdl" + str(self.adaptation_count) + ".pkl") + if self.notify: + self.ow[0].write(self.adaptation_count) + + self.save_model(self.save_dir + "model_final.pkl") + + + def load_memory(self, loc: str): + with open(loc, 'rb') as f: + return pickle.load(f) + + def save_model(self, loc: str): + with open(loc, 'wb') as f: + pickle.dump(self.model, f) + +def ensure_directory(directory: str) -> None: + """ + Ensure that a directory exists. If it does not exist, create it. + + Parameters + ---------- + directory : str + The directory to ensure exists. + """ + import os + if not os.path.exists(directory): + os.makedirs(directory) \ No newline at end of file diff --git a/libemg/adaptation/memory.py b/libemg/adaptation/memory.py new file mode 100644 index 00000000..9c0c46f7 --- /dev/null +++ b/libemg/adaptation/memory.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod + +class Memory(ABC): + + @abstractmethod + def append(self, data): + ... + + @abstractmethod + def reset(self): + ... + + @abstractmethod + def save(self): + ... + + @abstractmethod + def load(self): + ... + + @abstractmethod + def __add__(self, other): + ... \ No newline at end of file diff --git a/libemg/animator.py b/libemg/animator.py index 80e2716d..4f53d8d2 100644 --- a/libemg/animator.py +++ b/libemg/animator.py @@ -1,3 +1,4 @@ +from pathlib import Path import os from typing import Callable, Sequence import warnings @@ -53,7 +54,7 @@ def save_mp4(self, frames: Sequence[Image.Image]): frames: list List of frames, where each element is a PIL.Image object. """ - fourcc = cv2.VideoWriter_fourcc(*'avc1') + fourcc = cv2.VideoWriter_fourcc(*'mp4v') video = cv2.VideoWriter(self.output_filepath, fourcc, fps=self.fps, frameSize=frames[0].size) for frame in frames: @@ -178,7 +179,7 @@ def __init__(self, output_filepath: str ='libemg.gif', fps: int = 24, show_direc self.fpd = fps * self.tpd # number of frames to generate to travel a distance of 1 - def convert_distance_to_frames(self, coordinates1: npt.NDArray[np.float_], coordinates2: npt.NDArray[np.float_]): + def convert_distance_to_frames(self, coordinates1: npt.NDArray[np.float64], coordinates2: npt.NDArray[np.float64]): """Calculate the number of frames needed to move from coordinates1 to coordinates2. Parameters @@ -198,7 +199,7 @@ def convert_distance_to_frames(self, coordinates1: npt.NDArray[np.float_], coord return int(distance * self.fpd) @staticmethod - def _normalize_to_unit_distance(x: npt.NDArray[np.float_], y: npt.NDArray[np.float_]): + def _normalize_to_unit_distance(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]): """Normalize coordinates to a unit circle distance. Parameters @@ -257,7 +258,7 @@ def _format_figure(self): ax = plt.gca() return fig, ax - def _preprocess_coordinates(self, coordinates: npt.NDArray[np.float_]): + def _preprocess_coordinates(self, coordinates: npt.NDArray[np.float64]): """Modify coordinates before plotting (e.g., normalization). Parameters @@ -272,7 +273,7 @@ def _show_boundary(self): """Plot boundary to axis.""" pass # going to be different for each implementation, so don't implement it here - def _show_countdown(self, coordinates: npt.NDArray[np.float_], text: str): + def _show_countdown(self, coordinates: npt.NDArray[np.float64], text: str): """Show a countdown based on the current coordinates and frame index. Parameters @@ -284,7 +285,7 @@ def _show_countdown(self, coordinates: npt.NDArray[np.float_], text: str): """ plt.text(coordinates[0], coordinates[1], text, fontweight='bold', c='red', ha='center', va='center') - def _show_direction(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0): + def _show_direction(self, coordinates: npt.NDArray[np.float64], alpha: float = 1.0): """Show the direction of the next part of the movement. Parameters @@ -298,7 +299,7 @@ def _show_direction(self, coordinates: npt.NDArray[np.float_], alpha: float = 1. self.plot_icon(coordinates, alpha=alpha, colour='green') - def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, colour: str = 'black'): + def plot_icon(self, coordinates: npt.NDArray[np.float64], alpha: float = 1.0, colour: str = 'black'): """Plot target / icon on axis. Parameters @@ -313,7 +314,7 @@ def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, col plt.plot(coordinates[0], coordinates[1], alpha=alpha, c=colour) - def save_plot_video(self, coordinates: npt.NDArray[np.float_], title: str = '', xlabel: str = '', ylabel: str = '', save_coordinates: bool = False, verbose: bool = False): + def save_plot_video(self, coordinates: npt.NDArray[np.float64], title: str = '', xlabel: str = '', ylabel: str = '', save_coordinates: bool = False, verbose: bool = False): """Save a video file of an icon moving around a 2D plane. Parameters @@ -340,8 +341,8 @@ def save_plot_video(self, coordinates: npt.NDArray[np.float_], title: str = '', """ if save_coordinates: # Save coordinates in .txt file - filename_no_extension = os.path.splitext(self.output_filepath)[0] - labels_filepath = filename_no_extension + '.txt' + labels_filepath = Path(self.output_filepath).with_suffix('.txt') + labels_filepath.parent.mkdir(parents=True, exist_ok=True) np.savetxt(labels_filepath, coordinates, delimiter=',') # Format figure @@ -351,16 +352,23 @@ def save_plot_video(self, coordinates: npt.NDArray[np.float_], title: str = '', fig.suptitle(title) fig.tight_layout() - # Calculate steady states - diff = np.diff(coordinates, axis=0) - max_diff = np.max(np.abs(diff), axis=1) - all_steady_state_indices = np.where(max_diff < 0.01)[0] # find where the differences at each frame is less than some threshold - # Only take starts and ends of each segment - split_steady_state_indices = np.split(all_steady_state_indices, np.where(np.diff(all_steady_state_indices) != 1)[0] + 1) # add 1 to align diff with original - split_steady_state_indices.append(np.array([coordinates.shape[0] - 1])) # append last frame - steady_state_start_indices = np.array([segment[0] for segment in split_steady_state_indices]) - steady_state_end_indices = np.array([segment[-1] for segment in split_steady_state_indices]) - + split_steady_state_indices = None + steady_state_start_indices = None + steady_state_end_indices = None + if self.show_direction or self.show_countdown: + # Calculate steady states + diff = np.diff(coordinates, axis=0) + max_diff = np.max(np.abs(diff), axis=1) + all_steady_state_indices = np.where(max_diff < 0.01)[0] # find where the differences at each frame is less than some threshold + try: + # Only take starts and ends of each segment + split_steady_state_indices = np.split(all_steady_state_indices, np.where(np.diff(all_steady_state_indices) != 1)[0] + 1) # add 1 to align diff with original + split_steady_state_indices.append(np.array([coordinates.shape[0] - 1])) # append last frame + steady_state_start_indices = np.array([segment[0] for segment in split_steady_state_indices]) + steady_state_end_indices = np.array([segment[-1] for segment in split_steady_state_indices]) + except IndexError as e: + raise IndexError('Could not find steady state frames. If these features are desired, please pass in steady state frames (i.e., consecutive frames with the same value).') from e + # Adjust coordinates if desired coordinates = self._preprocess_coordinates(coordinates) @@ -371,31 +379,33 @@ def save_plot_video(self, coordinates: npt.NDArray[np.float_], title: str = '', if verbose and frame_idx % 10 == 0: print(f'Frame {frame_idx} / {coordinates.shape[0]}') - # Calculate next steady state frame - next_steady_state_idx = min(current_steady_state_idx, len(steady_state_end_indices) - 1) # limit max index - if frame_idx > steady_state_end_indices[current_steady_state_idx]: - current_steady_state_idx += 1 - target_alpha = 0.05 # reset alpha # Plot additional information if self.show_boundary: # Show boundaries self._show_boundary() - if self.show_direction: - # Show path until a change in direction - next_steady_state_idx = current_steady_state_idx + 1 if frame_idx > steady_state_start_indices[current_steady_state_idx] else current_steady_state_idx - next_steady_state_start = steady_state_start_indices[next_steady_state_idx] - target_alpha += 0.01 # add in fade - target_alpha = min(0.4, target_alpha) # limit alpha to 0.4 - self._show_direction(coordinates[next_steady_state_start], alpha=target_alpha) + if (self.show_direction or self.show_countdown) and split_steady_state_indices is not None and steady_state_start_indices is not None and steady_state_end_indices is not None: + # Calculate next steady state frame + next_steady_state_idx = min(current_steady_state_idx, len(steady_state_end_indices) - 1) # limit max index + if frame_idx > steady_state_end_indices[current_steady_state_idx]: + current_steady_state_idx += 1 + target_alpha = 0.05 # reset alpha - if self.show_countdown: - # Show countdown during steady state - steady_state_end = steady_state_end_indices[current_steady_state_idx] - time_until_movement = (steady_state_end - frame_idx) * self.duration / 1000 # convert from frames to seconds - if time_until_movement >= 0.25 and frame_idx in split_steady_state_indices[current_steady_state_idx]: - # Only show countdown if the steady state is longer than 1 second - self._show_countdown(frame_coordinates, str(int(time_until_movement))) + if self.show_direction: + # Show path until a change in direction + next_steady_state_idx = current_steady_state_idx + 1 if frame_idx > steady_state_start_indices[current_steady_state_idx] else current_steady_state_idx + next_steady_state_start = steady_state_start_indices[next_steady_state_idx] + target_alpha += 0.01 # add in fade + target_alpha = min(0.4, target_alpha) # limit alpha to 0.4 + self._show_direction(coordinates[next_steady_state_start], alpha=target_alpha) + + if self.show_countdown: + # Show countdown during steady state + steady_state_end = steady_state_end_indices[current_steady_state_idx] + time_until_movement = (steady_state_end - frame_idx) * self.duration / 1000 # convert from frames to seconds + if time_until_movement >= 0.25 and frame_idx in split_steady_state_indices[current_steady_state_idx]: + # Only show countdown if the steady state is longer than 1 second + self._show_countdown(frame_coordinates, str(int(time_until_movement))) # Plot icon self.plot_icon(frame_coordinates) @@ -474,7 +484,7 @@ def _format_figure(self): ax.set(xlim=axis_limits, ylim=axis_limits) return fig, ax - def _preprocess_coordinates(self, coordinates: npt.NDArray[np.float_]): + def _preprocess_coordinates(self, coordinates: npt.NDArray[np.float64]): coordinates = super()._preprocess_coordinates(coordinates) if self.normalize_distance: @@ -486,7 +496,7 @@ def _show_boundary(self): an = np.linspace(0, 2 * np.pi, 100) plt.plot(np.cos(an), np.sin(an), 'b--', alpha=0.7) - def _show_countdown(self, coordinates: npt.NDArray[np.float_], text: str): + def _show_countdown(self, coordinates: npt.NDArray[np.float64], text: str): x = coordinates[0] y = coordinates[1] - 0.2 return super()._show_countdown((x, y), text) @@ -558,7 +568,7 @@ def __init__(self, output_filepath: str = 'libemg.gif', fps: int = 24, show_dire self.plot_line = plot_line - def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, colour: str = 'black'): + def plot_icon(self, coordinates: npt.NDArray[np.float64], alpha: float = 1.0, colour: str = 'black'): # Parse coordinates x = coordinates[0] y = coordinates[1] @@ -572,7 +582,7 @@ def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, col class ArrowPlotAnimator(CartesianPlotAnimator): - def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, colour: str = 'black'): + def plot_icon(self, coordinates: npt.NDArray[np.float64], alpha: float = 1.0, colour: str = 'black'): # Parse coordinates x_tail = coordinates[0] y_tail = coordinates[1] @@ -597,7 +607,7 @@ def _plot_circle(xy: tuple[float, float], radius: float, edgecolor: str, facecol circle = Circle(xy, radius=radius, edgecolor=edgecolor, facecolor=facecolor, alpha=alpha) plt.gca().add_patch(circle) - def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1.0, colour: str = 'black'): + def plot_icon(self, coordinates: npt.NDArray[np.float64], alpha: float = 1.0, colour: str = 'black'): # Parse coordinates x = coordinates[0] y = coordinates[1] @@ -658,13 +668,13 @@ def _format_figure(self): ax.set(ylim=axis_limits) return fig, ax - def _plot_border(self, coordinates: npt.NDArray[np.float_], edgecolor: str = 'black'): + def _plot_border(self, coordinates: npt.NDArray[np.float64], edgecolor: str = 'black'): plt.bar(self.bar_labels, coordinates, color='none', edgecolor=edgecolor, linewidth=2, width=self.bar_width) - def _show_direction(self, coordinates: npt.NDArray[np.float_], alpha: float = 1): + def _show_direction(self, coordinates: npt.NDArray[np.float64], alpha: float = 1): self._plot_border(coordinates, edgecolor='green') - def _show_countdown(self, coordinates: npt.NDArray[np.float_], text: str): + def _show_countdown(self, coordinates: npt.NDArray[np.float64], text: str): adjustment = 0.05 for label, dof_value in zip(self.bar_labels, coordinates): modifier = -adjustment if dof_value < 0 else adjustment @@ -674,7 +684,7 @@ def _show_boundary(self): self._plot_border(-1) self._plot_border(1) - def plot_icon(self, coordinates: npt.NDArray[np.float_], alpha: float = 1, colour: str = 'black'): + def plot_icon(self, coordinates: npt.NDArray[np.float64], alpha: float = 1, colour: str = 'black'): plt.bar(self.bar_labels, coordinates, alpha=alpha, color=colour, width=self.bar_width) axis_limits = plt.gca().get_ylim() diff --git a/libemg/data_handler.py b/libemg/data_handler.py index ec057c98..be4d8135 100644 --- a/libemg/data_handler.py +++ b/libemg/data_handler.py @@ -5,16 +5,13 @@ import pandas as pd import os import re -import socket -import csv -import pickle import time import math import wfdb import copy import matplotlib.pyplot as plt import matplotlib.cm as cm -from sklearn.decomposition import PCA +from matplotlib.axes import Axes from scipy.ndimage import zoom from scipy.signal import decimate from matplotlib import pyplot @@ -29,20 +26,23 @@ from libemg.utils import get_windows, _get_fn_windows, _get_mode_windows, make_regex class RegexFilter: - def __init__(self, left_bound: str, right_bound: str, values: Sequence, description: str): - """Filters files based on filenames that match the associated regex pattern and grabs metadata based on the regex pattern. + """ + Filters files based on filenames that match the associated regex pattern and grabs metadata based on the regex pattern. - Parameters - ---------- - left_bound: str - The left bound of the regex. - right_bound: str - The right bound of the regex. - values: list - The values between the two regexes. - description: str - Description of filter - used to name the metadata field. - """ + Parameters + ---------- + left_bound: str + The left bound of the regex. + right_bound: str + The right bound of the regex. + values: list + The values between the two regexes. + description: str + Description of filter - used to name the metadata field. Pass in an empty string to filter files without storing the values as metadata. + """ + def __init__(self, left_bound: str, right_bound: str, values: Sequence[str], description: str): + if values is None: + raise ValueError('Expected a list of values for RegexFilter, but got None. Using regex wildcard is not supported with the RegexFilter.') self.pattern = make_regex(left_bound, right_bound, values) self.values = values self.description = description @@ -83,19 +83,21 @@ def get_metadata(self, filename: str): class MetadataFetcher(ABC): - def __init__(self, description: str): - """Describes a type of metadata and implements a method to fetch it. + """ + Describes a type of metadata and implements a method to fetch it. - Parameters - ---------- - description: str - Description of metadata. - """ + Parameters + ---------- + description: str + Description of metadata. + """ + def __init__(self, description: str): self.description = description @abstractmethod - def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[str]): + def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[str]) -> npt.NDArray: """Fetch metadata. Must return a (N x M) numpy.ndarray, where N is the number of samples in the EMG data and M is the number of columns in the metadata. + If a single value array is returned (0D or 1D), it will be cast to a N x 1 array where all values are the original value. Parameters ---------- @@ -115,32 +117,54 @@ def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[st class FilePackager(MetadataFetcher): - def __init__(self, regex_filter: RegexFilter, package_function: Callable[[str, str], bool], align_method: str | Callable[[npt.NDArray, npt.NDArray], npt.NDArray] = 'zoom', load = None, column_mask = None): - """Package data file with another file that contains relevant metadata (e.g., a labels file). Cycles through all files - that match the RegexFilter and packages a data file with a metadata file based on a packaging function. + """Package data file with another file that contains relevant metadata (e.g., a labels file). Cycles through all files + that match the RegexFilter and packages a data file with a metadata file based on a packaging function. - Parameters - ---------- - regex_filter: RegexFilter - Used to find the type of metadata files. - package_function: callable - Function handle used to determine if two files should be packaged together (i.e., found the metadata file that goes with the data file). - Takes in the filename of a metadata file and the filename of the data file. Should return True if the files should be packaged together and False if not. - align_method: str or callable, default='zoom' - Method for aligning the samples of the metadata file and data file. Pass in 'zoom' for the metadata file to be zoomed using spline interpolation to the size of the data file or - pass in a callable that takes in the metadata and the EMG data and returns the aligned metadata. - load: callable or None, default=None - Custom loading function for metadata file. If None is passed, the metadata is loaded based on the file extension (only .csv and .txt are supported). - column_mask: list or None, default=None - List of integers corresponding to the indices of the columns that should be extracted from the raw file data. If None is passed, all columns are extracted. - """ + Parameters + ---------- + regex_filter: RegexFilter + Used to find the type of metadata files. The description of this RegexFilter is used to assign the name of the field for this metadata in the OfflineDataHandler. + package_function: callable or Sequence[RegexFilter] + Function handle used to determine if two files should be packaged together (i.e., found the metadata file that goes with the data file). + Takes in the filename of a metadata file and the filename of the data file. Should return True if the files should be packaged together and False if not. + Alternatively, a list of RegexFilters can be passed in and a function will be created that packages files only if the regex metadata of the data filename + and metadata filename match. + align_method: str or callable, default='zoom' + Method for aligning the samples of the metadata file and data file. Pass in 'zoom' for the metadata file to be zoomed using spline interpolation to the size of the data file or + pass in a callable that takes in the metadata and the EMG data and returns the aligned metadata. + load: callable, str, or None, default=None + Determines how metadata file is loaded. If a custom loading function, should take in the filename and return an array. If a string, + it is assumed to be the MRDF key of a .hea file. If None is passed, the metadata is loaded based on the file extension (only .csv and .txt are supported). + column_mask: list or None, default=None + List of integers corresponding to the indices of the columns that should be extracted from the raw file data. If None is passed, all columns are extracted. + """ + def __init__(self, regex_filter: RegexFilter, package_function: Callable[[str, str], bool] | Sequence[RegexFilter], + align_method: str | Callable[[npt.NDArray, npt.NDArray], npt.NDArray] = 'zoom', load: Callable[[str], npt.NDArray] | str | None = None, column_mask: Sequence[int] | None = None): super().__init__(regex_filter.description) self.regex_filter = regex_filter + self.package_filters = None + + if isinstance(package_function, Sequence): + # Create function to ensure metadata matches + self.package_filters = copy.deepcopy(package_function) + package_function = self._match_regex_patterns self.package_function = package_function self.align_method = align_method self.load = load self.column_mask = column_mask + def _match_regex_patterns(self, metadata_file: str, data_file: str): + assert self.package_filters is not None, 'Attempting to match package filters, but None found.' + for filter in self.package_filters: + if len(filter.get_matching_files([metadata_file])) == 0: + # Doesn't match filters + return False + matching_metadata = filter.get_metadata(metadata_file) == filter.get_metadata(data_file) + if not matching_metadata: + return False + return True + + def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[str]): potential_files = self.regex_filter.get_matching_files(all_files) packaged_files = [Path(potential_file) for potential_file in potential_files if self.package_function(potential_file, filename)] @@ -148,13 +172,19 @@ def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[st # I think it's easier to enforce a single file per FilePackager, but we could build in functionality to allow multiple files then just vstack all the data if there's a use case for that. raise ValueError(f"Found {len(packaged_files)} files to be packaged with {filename} when trying to package {self.regex_filter.description} file (1 file should be found). Please check filter and package functions.") packaged_file = packaged_files[0] + suffix = packaged_file.suffix + packaged_file = packaged_file.as_posix() if callable(self.load): # Passed in a custom loading function packaged_file_data = self.load(packaged_file) - elif packaged_file.suffix == '.txt': + elif isinstance(self.load, str): + # Passed in a MRDF key + assert suffix == '.hea', f"Provided string for load parameter, but packaged file doesn't have extension .hea. Please pass in a custom load function and/or ensure the correct file is packaged." + packaged_file_data = (wfdb.rdrecord(packaged_file.replace('.hea', ''))).__getattribute__(self.load) + elif suffix == '.txt': packaged_file_data = np.loadtxt(packaged_file, delimiter=',') - elif packaged_file.suffix == '.csv': + elif suffix == '.csv': packaged_file_data = pd.read_csv(packaged_file) packaged_file_data = packaged_file_data.to_numpy() else: @@ -163,7 +193,7 @@ def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[st # Align with EMG data if self.align_method == 'zoom': zoom_rate = file_data.shape[0] / packaged_file_data.shape[0] - zoom_factor = [zoom_rate if idx == 0 else 1 for idx in range(packaged_file_data.shape[1])] # only align the 0th axis (samples) + zoom_factor = (zoom_rate, 1) # only align the 0th axis (samples) packaged_file_data = zoom(packaged_file_data, zoom=zoom_factor) elif callable(self.align_method): packaged_file_data = self.align_method(packaged_file_data, file_data) @@ -181,19 +211,20 @@ def __call__(self, filename: str, file_data: npt.NDArray, all_files: Sequence[st return packaged_file_data -class ColumnFetch(MetadataFetcher): - def __init__(self, description: str, column_mask: Sequence[int] | int, values: Sequence | None = None): - """Fetch metadata from columns within data file. +class ColumnFetcher(MetadataFetcher): + """ + Fetch metadata from columns within data file. - Parameters - ---------- - description: str - Description of metadata. - column_mask: list or int - Integers corresponding to indices of columns that should be fetched. - values: list or None, default=None - List of potential values within metadata column. If a list is passed in, the metadata will be stored as the location (index) of the value within the provided list. If None, the value within the columns will be stored. - """ + Parameters + ---------- + description: str + Description of metadata. + column_mask: list or int + Integers corresponding to indices of columns that should be fetched. + values: list or None, default=None + List of potential values within metadata column. If a list is passed in, the metadata will be stored as the location (index) of the value within the provided list. If None, the value within the columns will be stored. + """ + def __init__(self, description: str, column_mask: Sequence[int] | int, values: Sequence | None = None): super().__init__(description) self.column_mask = column_mask self.values = values @@ -294,6 +325,9 @@ def get_data(self, folder_location: str, regex_filters: Sequence[RegexFilter], m Raises ValueError if folder_location is not a valid directory. """ def append_to_attribute(name, value): + if name == '': + # Don't want this data saved to data handler, so skip it + return if not hasattr(self, name): setattr(self, name, []) self.extra_attributes.append(name) @@ -345,6 +379,9 @@ def append_to_attribute(name, value): # Fetch remaining metadata for metadata_fetcher in metadata_fetchers: metadata = metadata_fetcher(file, file_data, all_files) + if metadata.ndim == 0 or metadata.shape[0] == 1: + # Cast to array with the same # of samples as EMG data + metadata = np.full((file_data.shape[0], 1), fill_value=metadata) if metadata.ndim == 1: # Ensure that output is always 2D array metadata = np.expand_dims(metadata, axis=1) @@ -395,6 +432,13 @@ def parse_windows(self, window_size, window_increment, metadata_operations=None) The number of samples in a window. window_increment: int The number of samples that advances before next window. + metadata_operations: dict or None (optional),default=None + Specifies which operations should be performed on metadata attributes when performing windowing. By default, + all metadata is stored as its mode in a window. To change this behaviour, specify the metadata attribute as the key and + the operation as the value in the dictionary. The operation (value) should either be an accepted string (mean, median, last_sample) or + a function handle that takes in an ndarray of size (window_size, ) and returns a single value to represent the metadata for that window. Passing in a string + will map from that string to the specified operation. The windowing of only the attributes specified in this dictionary will be modified - all other + attributes will default to the mode. If None, all attributes default to the mode. Defaults to None. Returns ---------- @@ -407,34 +451,40 @@ def parse_windows(self, window_size, window_increment, metadata_operations=None) return self._parse_windows_helper(window_size, window_increment, metadata_operations) def _parse_windows_helper(self, window_size, window_increment, metadata_operations): - metadata_ = {} + common_metadata_operations = { + 'mean': np.mean, + 'median': np.median, + 'last_sample': lambda x: x[-1] + } + window_data = [] + metadata = {k: [] for k in self.extra_attributes} for i, file in enumerate(self.data): # emg data windowing - windows = get_windows(file,window_size,window_increment) - if "windows_" in locals(): - windows_ = np.concatenate((windows_, windows)) - else: - windows_ = windows - # metadata windowing + window_data.append(get_windows(file,window_size,window_increment)) + for k in self.extra_attributes: if type(getattr(self,k)[i]) != np.ndarray: - file_metadata = np.ones((windows.shape[0])) * getattr(self, k)[i] + file_metadata = np.ones((window_data[-1].shape[0])) * getattr(self, k)[i] else: if metadata_operations is not None: if k in metadata_operations.keys(): # do the specified operation - file_metadata = _get_fn_windows(getattr(self,k)[i], window_size, window_increment, metadata_operations[k]) + operation = metadata_operations[k] + + if isinstance(operation, str): + try: + operation = common_metadata_operations[operation] + except KeyError as e: + raise KeyError(f"Unexpected metadata operation string. Please pass in a function or an accepted string {tuple(common_metadata_operations.keys())}. Got: {operation}.") + file_metadata = _get_fn_windows(getattr(self,k)[i], window_size, window_increment, operation) else: file_metadata = _get_mode_windows(getattr(self,k)[i], window_size, window_increment) else: file_metadata = _get_mode_windows(getattr(self,k)[i], window_size, window_increment) - if k not in metadata_.keys(): - metadata_[k] = file_metadata - else: - metadata_[k] = np.concatenate((metadata_[k], file_metadata)) - + + metadata[k].append(file_metadata) - return windows_, metadata_ + return np.vstack(window_data), {k: np.concatenate(metadata[k], axis=0) for k in metadata.keys()} def isolate_channels(self, channels): @@ -461,7 +511,7 @@ def isolate_channels(self, channels): new_odh.data[i] = new_odh.data[i][:,channels] return new_odh - def isolate_data(self, key, values): + def isolate_data(self, key, values, fast=True): """Entry point for isolating a single key of data within the offline data handler. First, error checking is performed within this method, then if it passes, the isolate_data_helper is called to make a new OfflineDataHandler that contains only that data. @@ -471,6 +521,8 @@ def isolate_data(self, key, values): The metadata key that will be used to filter (e.g., "subject", "rep", "class", "set", whatever you'd like). values: list A list of values that you want to isolate. (e.g. [0,1,2,3]). Indexing starts at 0. + fast: Boolean (default=False) + If true, it iterates over the median value for each EMG element. This should be used when parsing on things like reps, subjects, classes, etc. Returns ---------- @@ -479,47 +531,31 @@ def isolate_data(self, key, values): """ assert key in self.extra_attributes assert type(values) == list - return self._isolate_data_helper(key,values) + return self._isolate_data_helper(key,values,fast) - def _isolate_data_helper(self, key, values): + def _isolate_data_helper(self, key, values,fast): new_odh = OfflineDataHandler() setattr(new_odh, "extra_attributes", self.extra_attributes) key_attr = getattr(self, key) - - # if these end up being ndarrays, it means that the metadata was IN the csv file. - - if type(key_attr[0]) == np.ndarray: - # for every file (list element) - data = [] - for f in range(len(key_attr)): - # get the keep_mask + for e in self.extra_attributes: + setattr(new_odh, e, []) + + for f in range(len(key_attr)): + if fast: + if key_attr[f][0][0] in values: + keep_mask = [True] * len(key_attr[f]) + else: + keep_mask = [False] * len(key_attr[f]) + else: keep_mask = list([i in values for i in key_attr[f]]) - # append the valid data - if self.data[f][keep_mask,:].shape[0]> 0: - data.append(self.data[f][keep_mask,:]) - setattr(new_odh, "data", data) + + if self.data[f][keep_mask,:].shape[0]> 0: + new_odh.data.append(self.data[f][keep_mask,:]) + for e in self.extra_attributes: + updated_arr = getattr(new_odh, e) + updated_arr.append(getattr(self, e)[f][keep_mask]) + setattr(new_odh, e, updated_arr) - for k in self.extra_attributes: - key_value = getattr(self, k) - if type(key_value[0]) == np.ndarray: - # the other metadata that is in the csv file should be sliced the same way as the ndarray - key = [] - for f in range(len(key_attr)): - keep_mask = list([i in values for i in key_attr[f]]) - if key_value[f][keep_mask,:].shape[0]>0: - key.append(key_value[f][keep_mask,:]) - setattr(new_odh, k, key) - - else: - assert False # we should never get here - # # if the other metadata was not in the csv file (i.e. subject label in filename but classes in csv), then just keep it - # setattr(new_odh, k, key_value) - else: - assert False # we should never get here - # keep_mask = list([i in values for i in key_attr]) - # setattr(new_odh, "data", list(compress(self.data, keep_mask))) - # for k in self.extra_attributes: - # setattr(new_odh, k,list(compress(getattr(self, k), keep_mask))) return new_odh def visualize(): @@ -536,13 +572,17 @@ class OnlineDataHandler(DataHandler): ---------- shared_memory_items: Object The shared memory object returned from the streamer. + channel_mask: list or None (optional), default=None + Mask of active channels to use online. Allows certain channels to be ignored when streaming in real-time. If None, all channels are used. + Defaults to None. """ - def __init__(self, shared_memory_items): + def __init__(self, shared_memory_items, channel_mask = None): self.shared_memory_items = shared_memory_items self.prepare_smm() self.log_signal = Event() self.visualize_signal = Event() self.fi = None + self.channel_mask = channel_mask def prepare_smm(self): self.modalities = [] @@ -584,6 +624,17 @@ def install_filter(self, fi): """ self.fi = fi + def install_channel_mask(self, mask): + """Install a channel mask to isolate certain channels for online streaming. + + Parameters + ---------- + mask: list or None (optional), default=None + Mask of active channels to use online. Allows certain channels to be ignored when streaming in real-time. If None, all channels are used. + Defaults to None. + """ + self.channel_mask = mask + def analyze_hardware(self, analyze_time=10): """Analyzes several metrics from the hardware: @@ -641,7 +692,7 @@ def visualize(self, num_samples=500, block=True): if block: self._visualize(num_samples) else: - p = Process(target=self._visualize, kwargs={"num_samples":num_samples}) + p = Process(target=self._visualize, kwargs={"num_samples":num_samples}, daemon=True) p.start() def _visualize(self, num_samples): @@ -771,7 +822,8 @@ def extract_data(): # Extract features along each channel windows = data[np.newaxis].transpose(0, 2, 1) # add axis and tranpose to convert to (windows x channels x samples) fe = FeatureExtractor() - feature_set_dict = fe.extract_features(feature_list, windows) + feature_set_dict = fe.extract_features(feature_list, windows, array=False) + assert isinstance(feature_set_dict, dict), f"Expected dictionary of features. Got: {type(feature_set_dict)}." if remap_function is not None: # Remap raw data to image format for key in feature_set_dict: @@ -798,6 +850,8 @@ def extract_data(): # Format figure sample_data = extract_data() # access sample data to determine heatmap size fig, axs = plt.subplots(len(sample_data.keys()), 1) + if isinstance(axs, Axes): + axs = np.array([axs]) fig.suptitle(f'HD-EMG Heatmap') plots = [] for (feature_key, feature_data), ax in zip(sample_data.items(), axs): @@ -940,7 +994,11 @@ def get_data(self, N=0, filter=True): val = {} count = {} for mod in self.modalities: - data = self.smm.get_variable(mod) + # Read the buffer and its sample counter as a single atomic + # snapshot. They share a lock (see assign_shared_memory_locks), so + # the count can never run ahead of the data copied alongside it. + snapshot = self.smm.get_variables([mod, mod + "_count"]) + data = snapshot[mod] if filter: if self.fi is not None: if mod == "emg": # TODO: enable filter for each modality @@ -949,7 +1007,9 @@ def get_data(self, N=0, filter=True): val[mod] = data[:N,:] else: val[mod] = data[:,:] - count[mod] = self.smm.get_variable(mod+"_count") + if self.channel_mask is not None: + val[mod] = val[mod][:, self.channel_mask] + count[mod] = snapshot[mod + "_count"] return val,count def reset(self, modality=None): @@ -987,7 +1047,7 @@ def log_to_file(self, block=False, file_path='', timestamps=True): self._log_to_file() print("ODH->log_to_file ended.") else: - p = Process(target=self._log_to_file) + p = Process(target=self._log_to_file, daemon=True) p.start() def _log_to_file(self): @@ -997,17 +1057,20 @@ def _log_to_file(self): self.smm = SharedMemoryManager() for item in self.shared_memory_items: self.smm.find_variable(*item) - # initialize sample count for all modalities + # initialize sample count for all modalities by reading actual current counts from shared memory last_count = {} + _, counts = self.get_data(N=0, filter=False) for m in self.modalities: - last_count[m] = 0 + last_count[m] = counts[m][0,0] while True: timestamp = time.time() vals, counts = self.get_data(N=0, filter=False) for m in vals.keys(): new_count = counts[m][0,0] num_new_samples = new_count - last_count[m] - new_samples = vals[m][:num_new_samples,:] + # Shared-memory buffers are newest-first. Restore chronological + # order before appending each batch to the log file. + new_samples = np.flip(vals[m][:num_new_samples,:], axis=0) last_count[m] = new_count if num_new_samples: if not m in files.keys(): @@ -1019,6 +1082,8 @@ def _log_to_file(self): np.savetxt(files[m], new_samples) if self.log_signal.is_set(): print("ODH->log_to_file ended.") + for file in files.values(): + file.close() break def _check_streaming(self, timeout=15): diff --git a/libemg/datasets.py b/libemg/datasets.py index 04f67e8c..fe6f95ce 100644 --- a/libemg/datasets.py +++ b/libemg/datasets.py @@ -1,490 +1,495 @@ -import os +from libemg._datasets._3DC import _3DCDataset +from libemg._datasets.one_subject_myo import OneSubjectMyoDataset +from libemg._datasets.one_subject_emager import OneSubjectEMaGerDataset +from libemg._datasets.emg_epn612 import EMGEPN_UserDependent, EMGEPN_UserIndependent +from libemg._datasets.ciil import CIIL_MinimalData, CIIL_ElectrodeShift, CIIL_WeaklySupervised +from libemg._datasets.grab_myo import GRABMyoBaseline, GRABMyoCrossDay +from libemg._datasets.continous_transitions import ContinuousTransitions +from libemg._datasets.nina_pro import NinaproDB2, NinaproDB8 +from libemg._datasets.user_compliance import UserComplianceDataset +from libemg._datasets.fors_emg import FORSEMG +from libemg._datasets.radmand_lp import RadmandLP +from libemg._datasets.fougner_lp import FougnerLP +from libemg._datasets.intensity import ContractionIntensity +from libemg._datasets.hyser import Hyser1DOF, HyserNDOF, HyserRandom, HyserPR, HyserMVC # HyserMVC is not used in this script but is imported so it's public in libemg API +from libemg._datasets.kaufmann_md import KaufmannMD +from libemg._datasets.tmr_shirleyryanabilitylab import TMR_Post, TMR_Pre +from libemg.feature_extractor import FeatureExtractor +from libemg.emg_predictor import EMGClassifier, EMGRegressor +from libemg.offline_metrics import OfflineMetrics +from libemg.filtering import Filter +from libemg._datasets.emg2pose import EMG2POSEUD, EMG2POSECU +from sklearn.preprocessing import StandardScaler +import pickle import numpy as np -import zipfile -import scipy.io as sio -from libemg.data_handler import ColumnFetch, MetadataFetcher, OfflineDataHandler, RegexFilter, FilePackager -from libemg.utils import make_regex -from glob import glob -from os import walk -from pathlib import Path -from datetime import datetime -# this assumes you have git downloaded (not pygit, but the command line program git) - -class Dataset: - def __init__(self, save_dir='.', redownload=False): - self.save_dir = save_dir - self.redownload=redownload - - def download(self, url, dataset_name): - clone_command = "git clone " + url + " " + dataset_name - os.system(clone_command) + +def get_dataset_list(type='CLASSIFICATION', cross_user=False): + """Gets a list of all available datasets. + + Parameters + ---------- + type: str (default='CLASSIFICATION') + The type of datasets to return. Valid Options: 'CLASSIFICATION', 'REGRESSION', 'WEAKLYSUPERVISED', and 'ALL'. + cross_user: bool (default=False) + + + Returns + ---------- + dictionary + A dictionary with the all available datasets and their respective classes. + """ + type = type.upper() + if type not in ['CLASSIFICATION', 'REGRESSION', 'WEAKLYSUPERVISED', 'CROSSUSER', 'ALL']: + print('Valid Options for type parameter: \'CLASSIFICATION\', \'REGRESSION\', or \'ALL\'.') + return {} + + cross_user_classification = { + 'EMGEPN612': EMGEPN_UserIndependent, + } + + cross_user_regression = { + 'EMG2POSE': EMG2POSECU, + } - def remove_dataset(self, dataset_folder): - remove_command = "rm -rf " + dataset_folder - os.system(remove_command) - - def check_exists(self, dataset_folder): - return os.path.exists(dataset_folder) - - def prepare_data(self, format=OfflineDataHandler): - pass - - -class _3DCDataset(Dataset): - def __init__(self, save_dir='.', redownload=False, dataset_name="_3DCDataset"): - Dataset.__init__(self, save_dir, redownload) - self.url = "https://github.com/libemg/3DCDataset" - self.dataset_name = dataset_name - self.dataset_folder = os.path.join(self.save_dir , self.dataset_name) - self.class_list = ["Neutral", "Radial Deviation", "Wrist Flexion", "Ulnar Deviation", "Wrist Extension", "Supination", - "Pronation", "Power Grip", "Open Hand", "Chuck Grip", "Pinch Grip"] - - if (not self.check_exists(self.dataset_folder)): - self.download(self.url, self.dataset_folder) - elif (self.redownload): - self.remove_dataset(self.dataset_folder) - self.download(self.url, self.dataset_folder) - - - - def prepare_data(self, format=OfflineDataHandler, subjects_values = [str(i) for i in range(1,23)], - sets_values = ["train", "test"], - reps_values = ["0","1","2","3"], - classes_values = [str(i) for i in range(11)]): - if format == OfflineDataHandler: - regex_filters = [ - RegexFilter(left_bound = "/", right_bound="/EMG", values = sets_values, description='sets'), - RegexFilter(left_bound = "_", right_bound=".txt", values = classes_values, description='classes'), - RegexFilter(left_bound = "EMG_gesture_", right_bound="_", values = reps_values, description='reps'), - RegexFilter(left_bound="Participant", right_bound="/",values=subjects_values, description='subjects') - ] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") - return odh - -class Ninapro(Dataset): - def __init__(self, save_dir='.', dataset_name="Ninapro"): - # downloading the Ninapro dataset is not supported (no permission given from the authors)' - # however, you can download it from http://ninapro.hevs.ch/DB8 - # the subject zip files should be placed at: /NinaproDB8/DB8_s#.zip - Dataset.__init__(self, save_dir) - self.dataset_name = dataset_name - self.dataset_folder = os.path.join(self.save_dir , self.dataset_name, "") - self.exercise_step = [] + classification = { + 'OneSubjectMyo': OneSubjectMyoDataset, + '3DC': _3DCDataset, + 'MinimalTrainingData': CIIL_MinimalData, + 'ElectrodeShift': CIIL_ElectrodeShift, + 'GRABMyoBaseline': GRABMyoBaseline, + 'GRABMyoCrossDay': GRABMyoCrossDay, + 'ContinuousTransitions': ContinuousTransitions, + 'NinaProDB2': NinaproDB2, + 'FORS-EMG': FORSEMG, + 'EMGEPN612': EMGEPN_UserDependent, + 'ContractionIntensity': ContractionIntensity, + 'RadmandLP': RadmandLP, + 'FougnerLP': FougnerLP, + 'KaufmannMD': KaufmannMD, + 'TMR_Post' : TMR_Post, + 'TMR_Pre': TMR_Pre, + 'HyserPR': HyserPR, + } + + regression = { + 'OneSubjectEMaGer': OneSubjectEMaGerDataset, + 'NinaProDB8': NinaproDB8, + 'Hyser1DOF': Hyser1DOF, + 'HyserNDOF': HyserNDOF, + 'HyserRandom': HyserRandom, + 'UserCompliance': UserComplianceDataset, + 'EMG2POSE': EMG2POSEUD + } + + weaklysupervised = { + 'WeaklySupervised': CIIL_WeaklySupervised + } - def convert_to_compatible(self): - # get the zip files (original format they're downloaded in) - zip_files = find_all_files_of_type_recursively(self.dataset_folder,".zip") - # unzip the files -- if any are there (successive runs skip this) - for zip_file in zip_files: - with zipfile.ZipFile(zip_file, 'r') as zip_ref: - zip_ref.extractall(zip_file[:-4]+'/') - os.remove(zip_file) - # get the mat files (the files we want to convert to csv) - mat_files = find_all_files_of_type_recursively(self.dataset_folder,".mat") - for mat_file in mat_files: - self.convert_to_csv(mat_file) + if type == 'CLASSIFICATION': + if cross_user: + return cross_user_classification + return classification + elif type == 'REGRESSION': + if cross_user: + return cross_user_regression + return regression + elif type == "WEAKLYSUPERVISED": + return weaklysupervised + else: + # Concatenate all datasets + classification.update(regression) + classification.update(weaklysupervised) + return classification - def convert_to_csv(self, mat_file): - # read the mat file - mat_file = mat_file.replace("\\", "/") - mat_dir = mat_file.split('/') - mat_dir = os.path.join(*mat_dir[:-1],"") - mat = sio.loadmat(mat_file) - # get the data - exercise = int(mat_file.split('_')[3][1]) - exercise_offset = self.exercise_step[exercise-1] # 0 reps already included - data = mat['emg'] - restimulus = mat['restimulus'] - rerepetition = mat['rerepetition'] - if data.shape[0] != restimulus.shape[0]: # this happens in some cases - min_shape = min([data.shape[0], restimulus.shape[0]]) - data = data[:min_shape,:] - restimulus = restimulus[:min_shape,] - rerepetition = rerepetition[:min_shape,] - # remove 0 repetition - collection buffer - remove_mask = (rerepetition != 0).squeeze() - data = data[remove_mask,:] - restimulus = restimulus[remove_mask] - rerepetition = rerepetition[remove_mask] - # important little not here: - # the "rest" really is only the rest between motions, not a dedicated rest class. - # there will be many more rest repetitions (as it is between every class) - # so usually we really care about classifying rest as its important (most of the time we do nothing) - # but for this dataset it doesn't make sense to include (and not its just an offline showcase of the library) - # I encourage you to plot the restimulus to see what I mean. -> plt.plot(restimulus) - # so we remove the rest class too - remove_mask = (restimulus != 0).squeeze() - data = data[remove_mask,:] - restimulus = restimulus[remove_mask] - rerepetition = rerepetition[remove_mask] - tail = 0 - while tail < data.shape[0]-1: - rep = rerepetition[tail][0] # remove the 1 offset (0 was the collection buffer) - motion = restimulus[tail][0] # remove the 1 offset (0 was between motions "rest") - # find head - head = np.where(rerepetition[tail:] != rep)[0] - if head.shape == (0,): # last segment of data - head = data.shape[0] -1 +def get_dataset_info(dataset): + """Prints out the information about a certain dataset. + + Parameters + ---------- + dataset: string + The name of the dataset you want the information of. + """ + if dataset in get_dataset_list(): + get_dataset_list()[dataset]().get_info() + else: + print("ERROR: Invalid dataset name") + +def evaluate(model, window_size, window_inc, feature_list=['MAV'], feature_dic={}, included_datasets=['OneSubjectMyo', '3DC'], output_file='out.pkl', regression=False, metrics=['CA'], normalize_data=False, normalize_features=False): + """Evaluates an algorithm against all included datasets. + + Parameters + ---------- + window_size: int + The window size (**in ms**). + window_inc: int + The window increment (**in ms**). + feature_list: list (default=['MAV']) + A list of features. Pass in None for CNN. + feature_dic: dic (default={}) + A dictionary of parameters for the passed in features. + included_datasets: list (str) or list (DataSets) + The name of the datasets you want to evaluate your model on. Either pass in strings (e.g., '3DC') for names or the dataset objects (e.g., _3DCDataset()). + output_file: string (default='out.pkl') + The name of the directory you want to incrementally save the results to (it will be a pickle file). + regression: boolean (default=False) + If True, will create an EMGRegressor object. Otherwise creates an EMGClassifier object. + metrics: list (default=['CA']/['MSE']) + The metrics to extract from each dataset. + normalize_data: boolean (default=False) + If True, the data will be normalized. + normalize_features: boolean (default=False) + If True, features will get normalized. + Returns + ---------- + dictionary + A dictionary with a set of accuracies for different datasets + """ + + # -------------- Setup ------------------- + if metrics == ['CA'] and regression: + metrics = ['MSE'] + + metadata_operations = None + label_val = 'classes' + if regression: + metadata_operations = {'labels': 'last_sample'} + label_val = 'labels' + + om = OfflineMetrics() + + # --------------- Run ----------------- + accuracies = {} + for d_i, d in enumerate(included_datasets): + print(f"Evaluating {d} dataset...") + if isinstance(d, str): + dataset = get_dataset_list('ALL')[d]() + else: + dataset = d + + # Get feature dic + if isinstance(feature_dic, list): + f_dic = feature_dic[d_i] + else: + f_dic = feature_dic + + accs = [] + for s_i in range(0, dataset.num_subjects): + data = dataset.prepare_data(split=True, subjects=[s_i]) + + if data == None: + print('Skipping Subject... No data found.') + continue + + s_train_dh = data['Train'] + s_test_dh = data['Test'] + + print(str(s_i) + '/' + str(dataset.num_subjects) + ' completed.') + + # Normalize Data + if normalize_data: + filter = Filter(dataset.sampling) + filter.install_filters({'name': 'standardize', 'data': s_train_dh}) + filter.filter(s_train_dh) + filter.filter(s_test_dh) + + train_windows, train_meta = s_train_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + test_windows, test_meta = s_test_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + + if feature_list is not None: + fe = FeatureExtractor() + if normalize_features: + train_feats, scaler = fe.extract_features(feature_list, train_windows, feature_dic=f_dic, normalize=True, fix_feature_errors=True) + test_feats, _ = fe.extract_features(feature_list, test_windows, feature_dic=f_dic, normalize=True, normalizer=scaler, fix_feature_errors=True) + else: + train_feats = fe.extract_features(feature_list, train_windows, feature_dic=f_dic, fix_feature_errors=True) + test_feats = fe.extract_features(feature_list, test_windows, feature_dic=f_dic, fix_feature_errors=True) else: - head = head[0] + tail - # downsample to 1kHz from 2kHz using decimation - data_for_file = data[tail:head,:] - data_for_file = data_for_file[::2, :] - # write to csv - csv_file = mat_dir + 'C' + str(motion-1) + 'R' + str(rep-1 + exercise_offset) + '.csv' - np.savetxt(csv_file, data_for_file, delimiter=',') - tail = head - os.remove(mat_file) - -class NinaproDB8(Ninapro): - def __init__(self, save_dir='.', dataset_name="NinaproDB8"): - Ninapro.__init__(self, save_dir, dataset_name) - self.class_list = ["Thumb Flexion/Extension", "Thumb Abduction/Adduction", "Index Finger Flexion/Extension", "Middle Finger Flexion/Extension", "Combined Ring and Little Fingers Flexion/Extension", - "Index Pointer", "Cylindrical Grip", "Lateral Grip", "Tripod Grip"] - self.exercise_step = [0,10,20] - - def prepare_data(self, format=OfflineDataHandler, subjects_values = [str(i) for i in range(1,13)], - reps_values = [str(i) for i in range(22)], - classes_values = [str(i) for i in range(9)]): + train_feats = train_windows + test_feats = test_windows + + ds = { + 'training_features': train_feats, + 'training_labels': train_meta[label_val] + } + + if not regression: + clf = EMGClassifier(model) + else: + clf = EMGRegressor(model) + clf.fit(ds) + + if regression: + preds = clf.run(test_feats) + else: + preds, _ = clf.run(test_feats) + + metrics = om.extract_offline_metrics(metrics, test_meta[label_val], preds) + accs.append(metrics) + + print(metrics) + accuracies[str(d)] = accs + + with open(output_file, 'wb') as handle: + pickle.dump(accuracies, handle, protocol=pickle.HIGHEST_PROTOCOL) + + +def evaluate_crossuser(model, window_size, window_inc, feature_list=['MAV'], feature_dic={}, included_datasets=['EMGEPN612'], output_file='out_cross.pkl', regression=False, metrics=['CA'], normalize_data=False, normalize_features=False, memory_efficient=False): + """Evaluates an algorithm against all the cross-user datasets. + + Parameters + ---------- + window_size: int + The window size (**in ms**). + window_inc: int + The window increment (**in ms**). + feature_list: list (default=['MAV']) + A list of features. + feature_dic: dic or list (default={}) + A dictionary or list of dictionaries of parameters for the passed in features. + included_datasets: list (str) or list (DataSets) + The name of the datasets you want to evaluate your model on. Either pass in strings (e.g., '3DC') for names or the dataset objects (e.g., _3DCDataset()). + output_file: string (default='out.pkl') + The name of the directory you want to incrementally save the results to (it will be a pickle file). + regression: boolean (default=False) + If True, will create an EMGRegressor object. Otherwise creates an EMGClassifier object. + metrics: list (default=['CA']) + The metrics to extract from each dataset. + normalize_data: boolean (default=False) + If True, the data will be normalized. + normalize_features: boolean (default=False) + If True, features will get normalized. + memory_efficient: boolean (default=false) + If True, features will be extracted in the prepare method. You wont have access to the raw data and normalization won't be possible. + Returns + ---------- + dictionary + A dictionary with a set of accuracies for different datasets + """ + # -------------- Setup ------------------- + if metrics == ['CA'] and regression: + metrics = ['MSE'] + + metadata_operations = None + label_val = 'classes' + if regression: + metadata_operations = {'labels': 'last_sample'} + label_val = 'labels' + + om = OfflineMetrics() + fe = FeatureExtractor() + + # --------------- Run ----------------- + accuracies = {} + for d_i, d in enumerate(included_datasets): + print(f"Evaluating {d} dataset...") + if isinstance(d, str): + if regression: + dataset = get_dataset_list('REGRESSION', True)[d]() + else: + dataset = get_dataset_list('CLASSIFICATION', True)[d]() + else: + dataset = d + + # Get feature dic + if isinstance(feature_dic, list): + f_dic = feature_dic[d_i] + else: + f_dic = feature_dic - if format == OfflineDataHandler: - regex_filters = [ - RegexFilter(left_bound = "/C", right_bound="R", values = classes_values, description='classes'), - RegexFilter(left_bound = "R", right_bound=".csv", values = reps_values, description='reps'), - RegexFilter(left_bound="DB8_s", right_bound="/",values=subjects_values, description='subjects') - ] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") - return odh - -class NinaproDB2(Ninapro): - def __init__(self, save_dir='.', dataset_name="NinaproDB2"): - Ninapro.__init__(self, save_dir, dataset_name) - self.class_list = ["TODO"] - self.exercise_step = [0,0,0] - - def prepare_data(self, format=OfflineDataHandler, subjects_values = [str(i) for i in range(1,41)], - reps_values = [str(i) for i in range(6)], - classes_values = [str(i) for i in range(50)]): + if memory_efficient: + data = dataset.prepare_data(split=True, feature_list=feature_list, feature_dic=f_dic, window_size=int(dataset.sampling/1000 * window_size), window_inc=int(dataset.sampling/1000 * window_inc)) + else: + data = dataset.prepare_data(split=True) - if format == OfflineDataHandler: - regex_filters = [ - RegexFilter(left_bound = "/C", right_bound="R", values = classes_values, description='classes'), - RegexFilter(left_bound = "R", right_bound=".csv", values = reps_values, description='reps'), - RegexFilter(left_bound="DB2_s", right_bound="/",values=subjects_values, description='subjects') - ] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") - return odh - -# given a directory, return a list of files in that directory matching a format -# can be nested -# this is just a handly utility -def find_all_files_of_type_recursively(dir, terminator): - files = os.listdir(dir) - file_list = [] - for file in files: - if file.endswith(terminator): - file_list.append(dir+file) + train_data = data['Train'] + test_data = data['Test'] + + # Normalize Data + if normalize_data and not memory_efficient: + filter = Filter(dataset.sampling) + filter.install_filters({'name': 'standardize', 'data': train_data}) + filter.filter(train_data) + filter.filter(test_data) + + if memory_efficient: + train_feats = np.vstack(train_data.data) + train_labels = np.vstack(getattr(train_data, label_val)).squeeze() + if normalize_features: + normalizer = StandardScaler() + train_feats = normalizer.fit_transform(train_feats) else: - if os.path.isdir(dir+file): - file_list += find_all_files_of_type_recursively(dir+file+'/',terminator) - return file_list - - -class OneSubjectMyoDataset(Dataset): - def __init__(self, save_dir='.', redownload=False, dataset_name="OneSubjectMyoDataset"): - Dataset.__init__(self, save_dir, redownload) - self.url = "https://github.com/libemg/OneSubjectMyoDataset" - self.dataset_name = dataset_name - self.dataset_folder = os.path.join(self.save_dir , self.dataset_name) - - if (not self.check_exists(self.dataset_folder)): - self.download(self.url, self.dataset_folder) - elif (self.redownload): - self.remove_dataset(self.dataset_folder) - self.download(self.url, self.dataset_folder) - - def prepare_data(self, format=OfflineDataHandler): - if format == OfflineDataHandler: - sets_values = ["1","2","3","4","5","6"] - classes_values = ["0","1","2","3","4"] - reps_values = ["0","1"] - regex_filters = [ - RegexFilter(left_bound = "/trial_", right_bound="/", values = sets_values, description='sets'), - RegexFilter(left_bound = "C_", right_bound=".csv", values = classes_values, description='classes'), - RegexFilter(left_bound = "R_", right_bound="_", values = reps_values, description='reps') - ] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, delimiter=",") - return odh - - -class _SessionFetcher(MetadataFetcher): - def __init__(self): - super().__init__('sessions') - - def __call__(self, filename, file_data, all_files): - def split_filename(f): - # Split date and name into separate variables - date_idx = f.find('2018') - date = datetime.strptime(Path(f[date_idx:]).stem, '%Y-%m-%d-%H-%M-%S-%f') - description = f[:date_idx] - return date, description - - data_file_date, data_file_description = split_filename(filename) - - # Grab the other file of a different date. Return the index of which session it is - same_subject_files = [f for f in all_files if data_file_description in f] - file_dates = [split_filename(subject_filename)[0] for subject_filename in same_subject_files] - file_dates.sort() - session_idx = file_dates.index(data_file_date) - return session_idx * np.ones((file_data.shape[0], 1), dtype=int) - - -class _RepFetcher(ColumnFetch): - def __call__(self, filename, file_data, all_files): - column_data = super().__call__(filename, file_data, all_files) + train_windows, train_meta = train_data.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + if normalize_features: + train_feats, normalizer = fe.extract_features(feature_list, train_windows, feature_dic=f_dic, normalize=True, fix_feature_errors=True) + else: + train_feats = fe.extract_features(feature_list, train_windows, feature_dic=f_dic, fix_feature_errors=True) + del train_windows + train_labels = train_meta[label_val] + + ds = { + 'training_features': train_feats, + 'training_labels': train_labels + } - # Get rep transitions - diff = np.diff(column_data, axis=0) - rep_end_row_mask, rep_end_col_mask = np.nonzero((diff < 0) & (column_data[1:] == 0)) - unique_rep_end_row_mask = np.unique(rep_end_row_mask) # remove duplicate start indices (for combined movements) - # rest_end_row_mask = np.nonzero(np.diff(np.nonzero(column_data == 0)[0]) > 1)[0] - # rest_end_row_mask = np.nonzero(np.diff(np.nonzero(np.all(column_data == 0, axis=1))[0]) > 1)[0] - # unique_rep_end_row_mask = np.concatenate((unique_rep_end_row_mask, rest_end_row_mask)) - # unique_rep_end_row_mask = np.sort(unique_rep_end_row_mask) - - - # Populate metadata array - metadata = np.empty((column_data.shape[0], 1), dtype=np.int16) - rep_counters = [0 for _ in range(5)] # 5 different press types - previous_rep_start = 0 - for idx, rep_start in enumerate(unique_rep_end_row_mask): - movement_idx = 4 if np.sum(rep_end_row_mask == rep_start) > 1 else rep_end_col_mask[idx] # if multiple columns are nonzero then it's a combined movement - rep = rep_counters[movement_idx] - metadata[previous_rep_start:rep_start] = rep - rep_counters[movement_idx] += 1 - previous_rep_start = rep_start - - # Fill in final samples - metadata[rep_start:] = rep - - return metadata - - -class PutEMGForceDataset(Dataset): - def __init__(self, save_dir = '.', dataset_name = 'PutEMGForceDataset', data_filetype = None): - """Dataset wrapper for putEMG-Force dataset. Used for regression of finger forces. - - Parameters - ---------- - save_dir : str, default='.' - Base data directory. - dataset_name : str, default='PutEMGForceDataset' - Name of dataset. Looks for dataset in filepath created by appending save_dir and dataset_name. - data_filetype : list or None, default=None - Type of data file to use. Accepted values are 'repeats_long', 'repeats_short', 'sequential', or any combination of those. If None is passed, all will be used. - """ - # TODO: Implement downloading dataset using .sh or .py file - super().__init__(save_dir) - self.dataset_name = dataset_name - self.dataset_folder = os.path.join(self.save_dir, self.dataset_name) - if data_filetype is None: - data_filetype = ['repeats_short', 'repeats_long', 'sequential'] - elif not isinstance(data_filetype, list): - data_filetype = [data_filetype] - self.data_filetype = data_filetype - - def prepare_data(self, format=OfflineDataHandler, subjects = None, sessions = None, reps = None, labels = 'forces', label_dof_mask = None): - if subjects is None: - subjects = [str(idx).zfill(2) for idx in range(60)] - - if labels == 'forces': - column_mask = np.arange(25, 35) - elif labels == 'trajectories': - column_mask = np.arange(36, 40) + if not regression: + clf = EMGClassifier(model) else: - raise ValueError(f"Expected either 'forces' or trajectories' for labels parameter, but received {labels}.") - - if label_dof_mask is not None: - column_mask = column_mask[label_dof_mask] - - if format == OfflineDataHandler: - regex_filters = [ - RegexFilter(left_bound='/emg_force-', right_bound='-', values=subjects, description='subjects'), - RegexFilter(left_bound='-', right_bound='-', values=self.data_filetype, description='data_filetype'), - ] - metadata_fetchers = [ - _SessionFetcher(), - ColumnFetch('labels', column_mask), - _RepFetcher('reps', list(range(36, 40))) - ] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers, delimiter=',', skiprows=1, data_column=list(range(1, 25))) - if sessions is not None: - odh = odh.isolate_data('sessions', sessions) - if reps is not None: - odh = odh.isolate_data('reps', reps) - return odh - - -class OneSubjectEMaGerDataset(Dataset): - def __init__(self, save_dir = '.', redownload = False, dataset_name = 'OneSubjectEMaGerDataset'): - super().__init__(save_dir, redownload) - self.url = 'https://github.com/LibEMG/OneSubjectEMaGerDataset' - self.dataset_name = dataset_name - self.dataset_folder = os.path.join(self.save_dir, self.dataset_name) - - if (not self.check_exists(self.dataset_folder)): - self.download(self.url, self.dataset_folder) - elif (self.redownload): - self.remove_dataset(self.dataset_folder) - self.download(self.url, self.dataset_folder) - - def prepare_data(self, format=OfflineDataHandler): - if format == OfflineDataHandler: - regex_filters = [ - RegexFilter(left_bound='/', right_bound='/', values=['open-close', 'pro-sup'], description='movements'), - RegexFilter(left_bound='_R_', right_bound='_emg.csv', values=[str(idx) for idx in range(5)], description='reps') - ] - package_function = lambda x, y: Path(x).parent.absolute() == Path(y).parent.absolute() - metadata_fetchers = [FilePackager(RegexFilter(left_bound='/', right_bound='.txt', values=['labels'], description='labels'), package_function)] - odh = OfflineDataHandler() - odh.get_data(folder_location=self.dataset_folder, regex_filters=regex_filters, metadata_fetchers=metadata_fetchers) - return odh - + clf = EMGRegressor(model) + clf.fit(ds) -# class GRABMyo(Dataset): -# def __init__(self, save_dir='.', redownload=False, subjects=list(range(1,44)), sessions=list(range(1,4)), dataset_name="GRABMyo"): -# Dataset.__init__(self, save_dir, redownload) -# self.url = "https://physionet.org/files/grabmyo/1.0.2/" -# self.dataset_name = dataset_name -# self.dataset_folder = os.path.join(self.save_dir , self.dataset_name) -# self.subjects = subjects -# self.sessions = sessions - -# if (not self.check_exists(self.dataset_folder)): -# self.download_data() -# elif (self.redownload): -# self.remove_dataset(self.dataset_folder) -# self.download_data() -# else: -# print("Data Already Downloaded.") - -# def download_data(self): -# curl_command = "curl --create-dirs" + " -O --output-dir " + str(self.dataset_folder) + "/ " -# # Download files -# print("Starting download...") -# files = ['readme.txt', 'subject-info.csv', 'MotionSequence.txt'] -# for f in files: -# os.system(curl_command + self.url + f) -# for session in self.sessions: -# curl_command = "curl --create-dirs" + " -O --output-dir " + str(self.dataset_folder) + "/" + "Session" + str(session) + "/ " -# for p in self.subjects: -# for t in range(1,8): -# for g in range(1,18): -# endpoint = self.url + "Session" + str(session) + "/session" + str(session) + "_participant" + str(p) + "/session" + str(session) + "_participant" + str(p) + "_gesture" + str(g) + "_trial" + str(t) -# os.system(curl_command + endpoint + '.hea') -# os.system(curl_command + endpoint + '.dat') -# print("Download complete.") - -# def prepare_data(self, format=OfflineDataHandler, subjects=[str(i) for i in range(1,44)], sessions=["1","2","3"]): -# if format == OfflineDataHandler: -# sets_regex = make_regex(left_bound = "session", right_bound="_", values = sessions) -# classes_values = ["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17"] -# classes_regex = make_regex(left_bound = "_gesture", right_bound="_", values = classes_values) -# reps_values = ["1","2","3","4","5","6","7"] -# reps_regex = make_regex(left_bound = "trial", right_bound=".hea", values = reps_values) -# subjects_regex = make_regex(left_bound="participant", right_bound="_",values=subjects) -# dic = { -# "sessions": sessions, -# "sessions_regex": sets_regex, -# "reps": reps_values, -# "reps_regex": reps_regex, -# "classes": classes_values, -# "classes_regex": classes_regex, -# "subjects": subjects, -# "subjects_regex": subjects_regex -# } -# odh = OfflineDataHandler() -# odh.get_data(folder_location=self.dataset_folder, filename_dic=dic, delimiter=",") -# return odh - -# def print_info(self): -# print('Reference: https://www.physionet.org/content/grabmyo/1.0.2/') -# print('Name: ' + self.dataset_name) -# print('Gestures: 17') -# print('Trials: 7') -# print('Time Per Rep: 5s') -# print('Subjects: 43') -# print("Forearm EMG (16): Columns 0-15\nWrist EMG (12): 18-23 and 26-31\nUnused (4): 16,23,24,31") - - -# class NinaDB1(Dataset): -# def __init__(self, dataset_dir, subjects): -# Dataset.__init__(self, dataset_dir) -# self.dataset_folder = dataset_dir -# self.subjects = subjects - -# if (not self.check_exists(self.dataset_folder)): -# print("The dataset does not currently exist... Please download it from: http://ninaweb.hevs.ch/data1") -# exit(1) -# else: -# filenames = next(walk(self.dataset_folder), (None, None, []))[2] -# if not any("csv" in f for f in filenames): -# self.setup(filenames) -# print("Extracted and set up repo.") -# self.prepare_data() - -# def setup(self, filenames): -# for f in filenames: -# if "zip" in f: -# file_path = os.path.join(self.dataset_folder, f) -# with zipfile.ZipFile(file_path, 'r') as zip_ref: -# zip_ref.extractall(self.dataset_folder) -# self.convert_data() - -# def convert_data(self): -# mat_files = [y for x in os.walk(self.dataset_folder) for y in glob(os.path.join(x[0], '*.mat'))] -# for f in mat_files: -# mat_dict = sio.loadmat(f) -# output_ = np.concatenate((mat_dict['emg'], mat_dict['restimulus'], mat_dict['rerepetition']), axis=1) -# mask_ids = output_[:,11] != 0 -# output_ = output_[mask_ids,:] -# np.savetxt(f[:-4]+'.csv', output_,delimiter=',') - -# def cleanup_data(self): -# mat_files = [y for x in os.walk(self.dataset_folder) for y in glob(os.path.join(x[0], '*.mat'))] -# zip_files = [y for x in os.walk(self.dataset_folder) for y in glob(os.path.join(x[0], '*.zip'))] -# files = mat_files + zip_files -# for f in files: -# os.remove(f) + del train_feats + del ds + + unique_subjects = np.unique(np.hstack([t.flatten() for t in test_data.subjects])) + + accs = [] + for s_i, s in enumerate(unique_subjects): + print(str(s_i) + '/' + str(len(unique_subjects)) + ' completed.') + s_test_dh = test_data.isolate_data('subjects', [s]) + + if memory_efficient: + test_feats = np.vstack(s_test_dh.data) + test_labels = np.vstack(getattr(s_test_dh, label_val)).squeeze() + if normalize_features: + test_feats = normalizer.transform(test_feats) + else: + test_windows, test_meta = s_test_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + if normalize_features: + test_feats, _ = fe.extract_features(feature_list, test_windows, feature_dic=f_dic, normalize=True, normalizer=normalizer, fix_feature_errors=True) + else: + test_feats = fe.extract_features(feature_list, test_windows, feature_dic=f_dic, fix_feature_errors=True) + test_labels = test_meta[label_val] + + if regression: + preds = clf.run(test_feats) + else: + preds, _ = clf.run(test_feats) + + metrics = om.extract_offline_metrics(metrics, test_labels, preds) + accs.append(metrics) + + print(metrics) + accuracies[str(d)] = accs + + with open(output_file, 'wb') as handle: + pickle.dump(accuracies, handle, protocol=pickle.HIGHEST_PROTOCOL) + +def evaluate_weaklysupervised(model, window_size, window_inc, feature_list=['MAV'], feature_dic={}, included_datasets=['CIIL_WeaklySupervised'], output_file='out.pkl', regression=False, metrics=['CA'], normalize_data=False, normalize_features=False): + """Evaluates an algorithm against all included datasets. -# def prepare_data(self, format=OfflineDataHandler): -# if format == OfflineDataHandler: -# classes_values = list(range(1,24)) -# classes_column = [10] -# classset_values = [str(i) for i in list(range(1,4))] -# classset_regex = make_regex(left_bound="_E", right_bound=".csv", values=classset_values) -# reps_values = list(range(1,11)) - -# reps_column = [11] -# subjects_values = [str(s) for s in self.subjects] -# subjects_regex = make_regex(left_bound="S", right_bound="_A", values=subjects_values) -# data_column = list(range(0,10)) -# dic = { -# "reps": reps_values, -# "reps_column": reps_column, -# "classes": classes_values, -# "classes_column": classes_column, -# "subjects": subjects_values, -# "subjects_regex": subjects_regex, -# "classset": classset_values, -# "classset_regex": classset_regex, -# "data_column": data_column -# } -# odh = OfflineDataHandler() -# odh.get_data(folder_location=self.dataset_folder, filename_dic=dic, delimiter=",") -# return odh + Parameters + ---------- + window_size: int + The window size (**in ms**). + window_inc: int + The window increment (**in ms**). + feature_list: list (default=['MAV']) + A list of features. Pass in None for CNN. + feature_dic: dic (default={}) + A dictionary of parameters for the passed in features. + included_datasets: list (str) or list (DataSets) + The name of the datasets you want to evaluate your model on. Either pass in strings (e.g., '3DC') for names or the dataset objects (e.g., _3DCDataset()). + output_file: string (default='out.pkl') + The name of the directory you want to incrementally save the results to (it will be a pickle file). + regression: boolean (default=False) + If True, will create an EMGRegressor object. Otherwise creates an EMGClassifier object. + metrics: list (default=['CA']/['MSE']) + The metrics to extract from each dataset. + normalize_data: boolean (default=False) + If True, the data will be normalized. + normalize_features: boolean (default=False) + If True, features will get normalized. + Returns + ---------- + dictionary + A dictionary with a set of accuracies for different datasets + """ + + # -------------- Setup ------------------- + if metrics == ['CA'] and regression: + metrics = ['MSE'] + + metadata_operations = None + label_val = 'classes' + if regression: + metadata_operations = {'labels': 'last_sample'} + label_val = 'labels' + + om = OfflineMetrics() + + # --------------- Run ----------------- + accuracies = {} + for d in included_datasets: + print(f"Evaluating {d} dataset...") + if isinstance(d, str): + dataset = get_dataset_list('WEAKLYSUPERVISED')[d]() + else: + dataset = d + + accs = [] + for s_i in range(0, dataset.num_subjects): + data = dataset.prepare_data(split=True, subjects=[s_i]) + + if data == None: + print('Skipping Subject... No data found.') + continue + + s_pretrain_dh = data['Pretrain'] + s_pretrain_dh.extra_attributes.remove('classes') + delattr(s_pretrain_dh,"classes") + s_train_dh = data['Train'] + s_test_dh = data['Test'] + + # Normalize Data + if normalize_data: + filter = Filter(dataset.sampling) + filter.install_filters({'name': 'standardize', 'data': s_pretrain_dh}) + filter.filter(s_pretrain_dh) + filter.filter(s_train_dh) + filter.filter(s_test_dh) + + pretrain_windows, pretrain_meta = s_pretrain_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + train_windows, train_meta = s_train_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + test_windows, test_meta = s_test_dh.parse_windows(int(dataset.sampling/1000 * window_size), int(dataset.sampling/1000 * window_inc), metadata_operations=metadata_operations) + + if feature_list is not None: + fe = FeatureExtractor() + if normalize_features: + pretrain_feats, scaler = fe.extract_features(feature_list, pretrain_windows, feature_dic=feature_dic, normalize=True, fix_feature_errors=True) + train_feats, _ = fe.extract_features(feature_list, train_windows, feature_dic=feature_dic, normalize=True, normalizer=scaler, fix_feature_errors=True) + test_feats, _ = fe.extract_features(feature_list, test_windows, feature_dic=feature_dic, normalize=True, normalizer=scaler, fix_feature_errors=True) + else: + pretrain_feats = fe.extract_features(feature_list, pretrain_windows, feature_dic=feature_dic, fix_feature_errors=True) + train_feats = fe.extract_features(feature_list, train_windows, feature_dic=feature_dic, fix_feature_errors=True) + test_feats = fe.extract_features(feature_list, test_windows, feature_dic=feature_dic, fix_feature_errors=True) + else: + pretrain_feats = pretrain_windows + train_feats = train_windows + test_feats = test_windows + + ds = { + 'training_features': train_feats, + 'training_labels': train_meta[label_val], + 'pretraining_features': pretrain_feats + } + + model.fit(ds) + + if not regression: + clf = EMGClassifier(model) + else: + clf = EMGRegressor(model) + + if regression: + preds = clf.run(test_feats) + else: + preds, _ = clf.run(test_feats) + + metrics = om.extract_offline_metrics(metrics, test_meta[label_val], preds) + accs.append(metrics) + print(str(s_i) + '/' + str(dataset.num_subjects) + ' completed.') + print(metrics) + accuracies[str(dataset)] = accs + + with open(output_file, 'wb') as handle: + pickle.dump(accuracies, handle, protocol=pickle.HIGHEST_PROTOCOL) \ No newline at end of file diff --git a/libemg/emg_predictor.py b/libemg/emg_predictor.py index 2528861c..654d60fb 100644 --- a/libemg/emg_predictor.py +++ b/libemg/emg_predictor.py @@ -7,9 +7,8 @@ from sklearn.naive_bayes import GaussianNB from sklearn.neural_network import MLPClassifier, MLPRegressor from sklearn.svm import SVC, SVR -from libemg.feature_extractor import FeatureExtractor -from libemg.shared_memory_manager import SharedMemoryManager -from multiprocessing import Process, Lock +from sklearn.preprocessing import StandardScaler +from multiprocessing import Process, Lock, Event import numpy as np import pickle import socket @@ -23,36 +22,51 @@ from scipy import stats import csv from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import re +from matplotlib.animation import FuncAnimation +from functools import partial +from libemg.feature_extractor import FeatureExtractor +from libemg.shared_memory_manager import SharedMemoryManager from libemg.utils import get_windows +from libemg.environments.controllers import RegressorController, ClassifierController +from libemg.data_handler import OnlineDataHandler class EMGPredictor: - def __init__(self, model, model_parameters = None, random_seed = 0, fix_feature_errors = False, silent = False) -> None: - """Base class for EMG prediction. + """Base class for EMG prediction. Parent class that shares common functionality between classifiers and regressors. - Parameters - ---------- - model: custom model (must have fit, predict and predict_proba functions) - Object that will be used to fit and provide predictions. - model_parameters: dictionary, default=None - Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. - random_seed: int, default=0 - Constant value to control randomization seed. - fix_feature_errors: bool (default=False) - If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. - silent: bool (default=False) - If True, the outputs from the fix_feature_errors parameter will be silenced. - """ + Parameters + ---------- + model: custom model (must have fit, predict and predict_proba functions) + Object that will be used to fit and provide predictions. + model_parameters: dictionary, default=None + Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. + random_seed: int, default=0 + Constant value to control randomization seed. + fix_feature_errors: bool (default=False) + If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. + silent: bool (default=False) + If True, the outputs from the fix_feature_errors parameter will be silenced. + """ + def __init__(self, + model, + model_parameters: Optional[Dict[str, Any]] = None, + random_seed: int = 0, + fix_feature_errors: bool = False, + silent: bool = False) -> None: + self.model = model self.model_parameters = model_parameters - # default for feature parameters self.feature_params = {} self.fix_feature_errors = fix_feature_errors self.silent = silent random.seed(random_seed) - def fit(self, feature_dictionary = None, dataloader_dictionary = None, training_parameters = None): + def fit(self, + feature_dictionary: Optional[Dict[str, Any]] = None, + dataloader_dictionary: Optional[Dict[str, Any]] = None, + training_parameters: Optional[Dict[str, Any]] = None) -> None: """The fit function for the EMG Prediction class. This is the method called that actually optimizes model weights for the dataset. This method presents a fork for two @@ -85,7 +99,8 @@ def fit(self, feature_dictionary = None, dataloader_dictionary = None, training_ raise ValueError("Incorrect combination of values passed to fit method. A feature dictionary is needed for statistical models and a dataloader dictionary is needed for deep models.") @classmethod - def from_file(self, filename): + def from_file(self, + filename: str) -> "EMGPredictor": """Loads a classifier - rather than creates a new one. After saving a statistical model, you can recreate it by running EMGClassifier.from_file(). By default @@ -109,19 +124,48 @@ def from_file(self, filename): model = pickle.load(f) return model - def _predict(self, data): + def _predict(self, + data: Any) -> Any: + """ + Predict using the model. + + Parameters + ---------- + data: np.ndarray or torch.tensor + The input to be processed + + Returns + ---------- + prediction: int + the output prediction (categorical) + """ try: return self.model.predict(data) except AttributeError as e: raise AttributeError("Attempted to perform prediction when model doesn't have a predict() method. Please ensure model has a valid predict() method.") from e - def _predict_proba(self, data): + def _predict_proba(self, + data: Any) -> Any: + """ + Predict probabilities using the model. + + Parameters + ---------- + data: np.ndarray or torch.tensor + The input to be processed + + Returns + ---------- + probabilities: np.ndarray or torch.tensor + the output probabilities (continuous valued) + """ try: return self.model.predict_proba(data) except AttributeError as e: raise AttributeError("Attempted to perform prediction when model doesn't have a predict_proba() method. Please ensure model has a valid predict_proba() method.") from e - def save(self, filename): + def save(self, + filename: str) -> None: """Saves (pickles) the EMGClassifier object to a file. Use this save function if you want to load the object later using the from_file function. Note that @@ -135,7 +179,8 @@ def save(self, filename): with open(filename, 'wb') as f: pickle.dump(self, f) - def install_feature_parameters(self, feature_params): + def install_feature_parameters(self, + feature_params: Dict[str, Any]) -> None: """Installs the feature parameters for the classifier. This function is used to install the feature parameters for the classifier. This is necessary for the classifier @@ -149,7 +194,13 @@ def install_feature_parameters(self, feature_params): self.feature_params = feature_params @staticmethod - def _validate_model_parameters(model, model_parameters, model_config): + def _validate_model_parameters(model, + model_parameters: Optional[Dict[str, Any]], + model_config: Dict[str, Any]) -> Any: + """ + Provide a string representing a sklearn model and this function will validate if the model parameter dictionary is valid + by checking the sklearn model constructor arguments. + """ if not isinstance(model, str): # Custom model return model @@ -170,8 +221,12 @@ def _validate_model_parameters(model, model_parameters, model_config): valid_model = model_reference(**valid_parameters) return valid_model - def _format_data(self, feature_dictionary): - if not isinstance(feature_dictionary, np.ndarray): + def _format_data(self, + feature_dictionary: Union[Dict[str, Any], Any]) -> Any: + """ + Format dictionary format of features into a single np.ndarray. + """ + if isinstance(feature_dictionary, dict): # Loop through each element and stack arr = None for feat in feature_dictionary: @@ -187,7 +242,11 @@ def _format_data(self, feature_dictionary): arr = np.nan_to_num(arr, neginf=0, nan=0, posinf=0) return arr - def _fit_statistical_model(self, feature_dictionary): + def _fit_statistical_model(self, + feature_dictionary: Dict[str, Any]) -> None: + """ + Fit the model using a feature dictionary. + """ assert 'training_features' in feature_dictionary.keys() assert 'training_labels' in feature_dictionary.keys() # convert dictionary of features format to np.ndarray for test/train set (NwindowxNfeature) @@ -195,33 +254,42 @@ def _fit_statistical_model(self, feature_dictionary): # self._set_up_classifier(model, feature_dictionary, parameters) self.model.fit(feature_dictionary['training_features'], feature_dictionary['training_labels']) - def _fit_deeplearning_model(self, dataloader_dictionary, parameters): + def _fit_deeplearning_model(self, + dataloader_dictionary: Dict[str, Any], + parameters: Dict[str, Any]) -> None: + """ + Fit a deep learning model using a dataloader dictionary. + """ assert 'training_dataloader' in dataloader_dictionary.keys() assert 'validation_dataloader' in dataloader_dictionary.keys() self.model.fit(dataloader_dictionary, **parameters) - class EMGClassifier(EMGPredictor): - def __init__(self, model, model_parameters = None, random_seed = 0, fix_feature_errors = False, silent = False): - """The Offline EMG Classifier. + """The Offline EMG Classifier. - This is the base class for any offline EMG classification. + This is the base class for any offline EMG classification. - Parameters - ---------- - model: string or custom classifier (must have fit, predict and predic_proba functions) - The type of machine learning model. Valid options include: 'LDA', 'QDA', 'SVM', 'KNN', 'RF' (Random Forest), - 'NB' (Naive Bayes), 'GB' (Gradient Boost), 'MLP' (Multilayer Perceptron). Note, these models are all default sklearn - models with no hyperparameter tuning and may not be optimal. Pass in custom classifiers or parameters for more control. - model_parameters: dictionary, default=None - Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. - random_seed: int, default=0 - Constant value to control randomization seed. - fix_feature_errors: bool (default=False) - If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. - silent: bool (default=False) - If True, the outputs from the fix_feature_errors parameter will be silenced. - """ + Parameters + ---------- + model: string or custom classifier (must have fit, predict and predic_proba functions) + The type of machine learning model. Valid options include: 'LDA', 'QDA', 'SVM', 'KNN', 'RF' (Random Forest), + 'NB' (Naive Bayes), 'GB' (Gradient Boost), 'MLP' (Multilayer Perceptron). Note, these models are all default sklearn + models with no hyperparameter tuning and may not be optimal. Pass in custom classifiers or parameters for more control. + model_parameters: dictionary, default=None + Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. + random_seed: int, default=0 + Constant value to control randomization seed. + fix_feature_errors: bool (default=False) + If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. + silent: bool (default=False) + If True, the outputs from the fix_feature_errors parameter will be silenced. + """ + def __init__(self, + model: Union[str, Any], + model_parameters: Optional[Dict[str, Any]] = None, + random_seed: int = 0, + fix_feature_errors: bool = False, + silent: bool = False): model_config = { 'LDA': (LinearDiscriminantAnalysis, {}), 'KNN': (KNeighborsClassifier, {"n_neighbors": 5}), @@ -245,7 +313,8 @@ def __init__(self, model, model_parameters = None, random_seed = 0, fix_feature_ - def run(self, test_data): + def run(self, + test_data: Any) -> Tuple[np.ndarray, np.ndarray]: """Runs the classifier on a pre-defined set of training data. Parameters @@ -261,7 +330,6 @@ def run(self, test_data): A list of the probabilities (for each prediction), based on the passed in testing features. """ test_data = self._format_data(test_data) - prob_predictions = self._predict_proba(test_data) # Default @@ -280,7 +348,8 @@ def run(self, test_data): # Accumulate Metrics return predictions, probabilities - def add_rejection(self, threshold=0.9): + def add_rejection(self, + threshold: float=0.9) -> None: """Adds the rejection post-processing block onto a classifier. Parameters @@ -291,7 +360,8 @@ def add_rejection(self, threshold=0.9): self.rejection = True self.rejection_threshold = threshold - def add_majority_vote(self, num_samples=5): + def add_majority_vote(self, + num_samples: int=5) -> None: """Adds the majority voting post-processing block onto a classifier. Parameters @@ -301,9 +371,11 @@ def add_majority_vote(self, num_samples=5): """ self.majority_vote = num_samples - def add_velocity(self, train_windows, train_labels, - velocity_metric_handle = None, - velocity_mapping_handle = None): + def add_velocity(self, + train_windows: np.ndarray, + train_labels: np.ndarray, + velocity_metric_handle: Optional[Callable[[Any], Any]] = None, + velocity_mapping_handle: Optional[Callable[[Any], Any]] = None): """Adds velocity (i.e., proportional) control where a multiplier is generated for the level of contraction intensity. Note, that when using this optional, ramp contractions should be captured for training. @@ -314,7 +386,6 @@ def add_velocity(self, train_windows, train_labels, self.velocity_metric_handle = velocity_metric_handle self.velocity_mapping_handle = velocity_mapping_handle self.velocity = True - self.th_min_dic, self.th_max_dic = self._set_up_velocity_control(train_windows, train_labels) @@ -322,7 +393,21 @@ def add_velocity(self, train_windows, train_labels, ''' ---------------------- Private Helper Functions ---------------------- ''' - def _prediction_helper(self, predictions): + def _prediction_helper(self, + predictions: Any) -> Tuple[np.ndarray, np.ndarray]: + """ + Helper function to extract prediction and probability. + + Parameters + ---------- + predictions : Any + Raw predictions. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + Tuple of predicted classes and probabilities. + """ probabilities = [] prediction_vals = [] for i in range(0, len(predictions)): @@ -331,7 +416,12 @@ def _prediction_helper(self, predictions): probabilities.append(pred_list[pred_list.index(max(pred_list))]) return np.array(prediction_vals), np.array(probabilities) - def _rejection_helper(self, prediction, prob): + def _rejection_helper(self, + prediction: Any, + prob: Any) -> Any: + """ + Helper function for rejection. + """ if self.rejection: if prob > self.rejection_threshold: return prediction @@ -339,7 +429,11 @@ def _rejection_helper(self, prediction, prob): return -1 return prediction - def _majority_vote_helper(self, predictions): + def _majority_vote_helper(self, + predictions: np.ndarray) -> np.ndarray: + """ + Helper function for majority voting. + """ updated_predictions = [] for i in range(0, len(predictions)): idxs = np.array(range(i-self.majority_vote+1, i+1)) @@ -348,7 +442,24 @@ def _majority_vote_helper(self, predictions): updated_predictions.append(stats.mode(group, keepdims=False)[0]) return np.array(updated_predictions) - def _get_velocity(self, window, c): + def _get_velocity(self, + window: Dict[str, Any], + c: Any) -> str: + """ + Compute velocity output based on window data. + + Parameters + ---------- + window : dict + Window data. + c : Any + Class or index. + + Returns + ------- + str + Formatted velocity. + """ mod = "emg" # todo: specify another way to do this is needed if self.th_max_dic and self.th_min_dic: @@ -362,7 +473,17 @@ def _get_velocity(self, window, c): velocity_output = self.velocity_mapping_handle(velocity_output) return '{0:.2f}'.format(min([1, max([velocity_output, 0])])) - def _set_up_velocity_control(self, train_windows, train_labels): + def _set_up_velocity_control(self, + train_windows: np.ndarray, + train_labels: np.ndarray) -> Tuple[Dict[Any, float], Dict[Any, float]]: + """ + Sets up velocity control thresholds. + + Returns + ------- + Tuple[dict, dict] + Dictionaries for min and max thresholds. + """ # Extract classes th_min_dic = {} th_max_dic = {} @@ -385,7 +506,10 @@ def _set_up_velocity_control(self, train_windows, train_labels): th_max_dic[c] = th_max return th_min_dic, th_max_dic - def visualize(self, test_labels, predictions, probabilities): + def visualize(self, + test_labels: np.ndarray, + predictions: np.ndarray, + probabilities: np.ndarray) -> None: """Visualize the decision stream of the classifier on the testing data. You can call this visualize function to get a visual output of what the decision stream of what @@ -434,35 +558,35 @@ def visualize(self, test_labels, predictions, probabilities): plt.legend(loc='lower right') plt.show() - class EMGRegressor(EMGPredictor): """The Offline EMG Regressor. This is the base class for any offline EMG regression. + Parameters + ---------- + model: string or custom regressor (must have fit and predict functions) + The type of machine learning model. Valid options include: 'LR' (Linear Regression), 'SVM' (Support Vector Machine), 'RF' (Random Forest), + 'GB' (Gradient Boost), 'MLP' (Multilayer Perceptron). Note, these models are all default sklearn + models with no hyperparameter tuning and may not be optimal. Pass in custom regressors or parameters for more control. + model_parameters: dictionary, default=None + Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. + random_seed: int, default=0 + Constant value to control randomization seed. + fix_feature_errors: bool (default=False) + If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. + silent: bool (default=False) + If True, the outputs from the fix_feature_errors parameter will be silenced. + deadband_threshold: float, default=0.0 + Threshold that controls deadband around 0 for output predictions. Values within this deadband will be output as 0 instead of their original prediction. """ - def __init__(self, model, model_parameters = None, random_seed = 0, fix_feature_errors = False, silent = False, deadband_threshold = 0.): - """The Offline EMG Regressor. - - This is the base class for any offline EMG regression. - - Parameters - ---------- - model: string or custom regressor (must have fit and predict functions) - The type of machine learning model. Valid options include: 'LR' (Linear Regression), 'SVM' (Support Vector Machine), 'RF' (Random Forest), - 'GB' (Gradient Boost), 'MLP' (Multilayer Perceptron). Note, these models are all default sklearn - models with no hyperparameter tuning and may not be optimal. Pass in custom regressors or parameters for more control. - model_parameters: dictionary, default=None - Mapping from parameter name to value based on the constructor of the specified model. Only used when a string is passed in for model. - random_seed: int, default=0 - Constant value to control randomization seed. - fix_feature_errors: bool (default=False) - If True, the model will update any feature errors (INF, -INF, NAN) using the np.nan_to_num function. - silent: bool (default=False) - If True, the outputs from the fix_feature_errors parameter will be silenced. - deadband_threshold: float, default=0.0 - Threshold that controls deadband around 0 for output predictions. Values within this deadband will be output as 0 instead of their original prediction. - """ + def __init__(self, + model: Union[str, Any], + model_parameters: Optional[Dict[str, Any]] = None, + random_seed: int = 0, + fix_feature_errors: bool = False, + silent: bool = False, + deadband_threshold: float = 0.): model_config = { 'LR': (LinearRegression, {}), 'SVM': (SVR, {"kernel": "linear"}), @@ -478,16 +602,18 @@ def __init__(self, model, model_parameters = None, random_seed = 0, fix_feature_ super().__init__(model, model_parameters, random_seed=random_seed, fix_feature_errors=fix_feature_errors, silent=silent) - def run(self, test_data): + def run(self, + test_data: Any) -> np.ndarray: """Runs the regressor on a pre-defined set of training data. Parameters ---------- test_data: list A dictionary, np.ndarray of inputs appropriate for the model of the EMGRegressor. + Returns ---------- - list + np.ndarray A list of predictions, based on the passed in testing features. """ test_data = self._format_data(test_data) @@ -499,15 +625,20 @@ def run(self, test_data): return predictions - def visualize(self, test_labels, predictions): + def visualize(self, + test_labels: np.ndarray, + predictions: np.ndarray) -> None: """Visualize the decision stream of the regressor on test data. You can call this visualize function to get a visual output of what the decision stream looks like. - :param test_labels: np.ndarray - :type test_labels: N x M array, where N = # samples and M = # DOFs, containing the labels for the test data. - :param predictions: np.ndarray - :type predictions: N x M array, where N = # samples and M = # DOFs, containing the predictions for the test data. + Parameters + ---------- + test_labels: np.ndarray + N x M array, where N = # samples and M = # DOFs, containing the labels for the test data. + predictions: np.ndarray + N x M array, where N = # samples and M = # DOFs, containing the predictions for the test data. + """ assert len(predictions) > 0, 'Empty list passed in for predictions to visualize.' @@ -534,7 +665,8 @@ def visualize(self, test_labels, predictions): plt.show() - def add_deadband(self, threshold): + def add_deadband(self, + threshold: float) -> None: """Add a deadband around regressor predictions that will instead be output as 0. Parameters @@ -544,7 +676,6 @@ def add_deadband(self, threshold): """ self.deadband_threshold = threshold - class OnlineStreamer(ABC): """OnlineStreamer. @@ -568,148 +699,104 @@ class OnlineStreamer(ABC): A location that the inputs and output of the classifier will be saved to. file: bool (optional) A toggle for activating the saving of inputs and outputs of the classifier. - smm: bool (optional) + enable_smm: bool (optional) A toggle for activating the storing of inputs and outputs of the classifier in the shared memory manager. smm_items: list (optional) A list of lists containing the tag, size, and multiprocessing locks for shared memory. parameters: dict (optional) A dictionary including all of the parameters for the sklearn models. These parameters should match those found in the sklearn docs for the given model. - port: int (optional), default = 12346 - The port used for streaming predictions over UDP. - ip: string (optional), default = '127.0.0.1' - The ip used for streaming predictions over UDP. velocity: bool (optional), default = False If True, the classifier will output an associated velocity (used for velocity/proportional based control). std_out: bool (optional), default = False If True, prints predictions to std_out. - tcp: bool (optional), default = False - If True, will stream predictions over TCP instead of UDP. """ def __init__(self, - offline_predictor, - window_size, - window_increment, - online_data_handler, - file_path, file, - smm, smm_items, - features, - port, ip, - std_out, - tcp): + offline_predictor: EMGPredictor, + window_size: int, + window_increment: int, + online_data_handler: OnlineDataHandler, + file_path: str, + file: bool, + enable_smm: bool, + smm_items: List[List[Any]], + features: Optional[List[Any]], + std_out: bool, + output_writers: Optional[List[Any]] = None): + + # setting arguments as class attributes self.window_size = window_size self.window_increment = window_increment self.odh = online_data_handler self.features = features - self.port = port - self.ip = ip self.predictor = offline_predictor + self.file = file + self.file_path = file_path + self.std_out = std_out + self.scaler = None + self.output_writers = output_writers if output_writers is not None else [] + self.signal = Event() - self.options = {'file': file, 'file_path': file_path, 'std_out': std_out} - - required_smm_items = [ # these tags are also required - ["adapt_flag", (1,1), np.int32], - ["active_flag", (1,1), np.int8] - ] - smm_items.extend(required_smm_items) - self.smm = smm + self.enable_smm = enable_smm self.smm_items = smm_items - self.files = {} - self.tcp = tcp - if not tcp: - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - else: - print("Waiting for TCP connection...") - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.sock.bind((ip, port)) - self.sock.listen() - self.conn, addr = self.sock.accept() - print(f"Connected by {addr}") + self.smm = None + self.model_smm_writes = 0 self.process = Process(target=self._run_helper, daemon=True,) - def start_stream(self, block=True): + def stop(self): + self.signal.set() + + def start_stream(self, + block: bool =True) -> None: + """ + Start the streaming process. + + Parameters + ---------- + block : bool, default=True + Whether to run in blocking mode. + """ if block: self._run_helper() else: self.process.start() - - def write_output(self, prediction, probabilities, probability, calculated_velocity, model_input): - time_stamp = time.time() - if calculated_velocity == "": - printed_velocity = "-1" - else: - printed_velocity = float(calculated_velocity) - if self.options['std_out']: - print(f"{int(prediction)} {printed_velocity} {time.time()}") - # Write classifier output: - if self.options['file']: - if not 'file_handle' in self.files.keys(): - self.files['file_handle'] = open(self.options['file_path'] + 'classifier_output.txt', "a", newline="") - writer = csv.writer(self.files['file_handle']) - feat_str = str(model_input[0]).replace('\n','')[1:-1] - row = [f"{time_stamp} {prediction} {probability[0]} {printed_velocity} {feat_str}"] - writer.writerow(row) - self.files['file_handle'].flush() - if "smm" in self.options.keys(): - # assumed to have "classifier_input" and "classifier_output" keys - # these are (1+) - def insert_classifier_input(data): - input_size = self.options['smm'].variables['classifier_input']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, model_input[0]]), data))[:input_size,:] - return data - def insert_classifier_output(data): - output_size = self.options['smm'].variables['classifier_output']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, prediction, probability[0], float(printed_velocity)]), data))[:output_size,:] - return data - self.options['smm'].modify_variable("classifier_input", - insert_classifier_input) - self.options['smm'].modify_variable("classifier_output", - insert_classifier_output) - self.options['classifier_smm_writes'] += 1 - - if self.output_format == "predictions": - message = str(prediction) + calculated_velocity + '\n' - elif self.output_format == "probabilities": - message = ' '.join([f'{i:.2f}' for i in probabilities[0]]) + calculated_velocity + " " + str(time_stamp) - if not self.tcp: - self.sock.sendto(bytes(message, 'utf-8'), (self.ip, self.port)) - else: - self.conn.sendall(str.encode(message)) - def prepare_smm(self): - for i in self.smm_items: - if len(i) == 3: - i.append(Lock()) + def prepare_smm(self) -> None: + """ + Prepare shared memory by creating required variables. + """ smm = SharedMemoryManager() for item in self.smm_items: smm.create_variable(*item) - self.options['smm'] = smm - self.options['classifier_smm_writes'] = 0 + self.smm = smm + self.model_smm_writes = 0 - def analyze_predictor(self, analyze_time=10): + def analyze_predictor(self, + ip: str="127.0.0.1", + port: int=12346, + analyze_time: int=10) -> None: """Analyzes the latency of the designed predictor. - - Parameters - ---------- - analyze_time: int (optional), default=10 (seconds) - The time in seconds that you want to analyze the model for. - port: int (optional), default = 12346 - The port used for streaming predictions over UDP. - ip: string (optional), default = '127.0.0.1' - The ip used for streaming predictions over UDP. (1) Time Between Prediction (Average): The average time between subsequent predictions. (2) STD Between Predictions (Standard Deviation): The standard deviation between predictions. (3) Total Number of Predictions: The number of predictions that were made. Sometimes if the increment is too small, samples will get dropped and this may be less than expected. + + Parameters + ---------- + ip: str (optional), default=localhost + The ip address to listen to for model outputs. + port: int (optional), default=12346 + The port to listen to for model outputs. + analyze_time: int (optional), default=10 (seconds) + The time in seconds that you want to analyze the model for. """ print("Starting analysis of predictor " + "(" + str(analyze_time) + "s)...") sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind((self.ip, self.port)) + sock.bind((ip, port)) st = time.time() times = [] while(time.time() - st < analyze_time): @@ -723,7 +810,11 @@ def analyze_predictor(self, analyze_time=10): print("Total Number of Predictions: " + str(len(times) + 1)) self.stop_running() - def _format_data_sample(self, data): + def _format_data_sample(self, + data: Dict[str, Any]) -> np.ndarray: + """ + Stack data from a dictionary into one array. In this case 'data' is the feature dictionary. + """ arr = None for feat in data: if arr is None: @@ -732,84 +823,232 @@ def _format_data_sample(self, data): arr = np.hstack((arr, data[feat])) return arr - def _get_data_helper(self): + def _get_data_helper(self) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """ + Retrieve and reverse data. + """ data, counts = self.odh.get_data(N=self.window_size) + # TODO: this probably adds latency and isn't really needed + # ODH has index 0=most recent sample + # if you trained using typical csv formats, the most recent sample is at the end of the list + # This inversion makes them the same, but is unnecessary like 99% of the time. for key in data.keys(): data[key] = data[key][::-1] return data, counts - def get_interaction_items(self): + def get_interaction_items(self) -> List[List[Any]]: + """ + Return the shared memory items. + """ return self.smm_items - def load_emg_predictor(self, number): - with open(self.options['file_path'] + 'mdl' + str(number) + '.pkl', 'rb') as handle: - self.predictor = pickle.load(handle) - print(f"Loaded model #{number}.") + def load_emg_predictor(self, + number: int) -> None: + """ + Load a predictor from a file. Assumes that the files is treated as an EMGPredictor (LibEMG object) or the underlying model. + + Parameters + ---------- + number : int + Model number. + """ + filepath = self.file_path + 'mdl' + str(number) + '.pkl' + loaded_content = self._load_emg_predictor_helper(filepath) + if type(loaded_content) == EMGPredictor: + self.predictor = loaded_content + else: + self.predictor.model = loaded_content + print(f"Loaded model #{number}.") + def _load_emg_predictor_helper(self, file_path: str) -> Any: + try: + with open(file_path,'rb') as handle: + return pickle.load(handle) + except pickle.UnpicklingError as e: + #Only catch the specific "persistent id" error that comes from pytorch + msg = str(e) + if 'persistent id' not in msg: + raise + try: + import torch + except ImportError: + raise RuntimeError("File to load contains pytorch tensor but torch is not installed.") + return torch.load(file_path) + - def _run_helper(self): - if self.smm: + + # ----- Default functions for the streaming pipeline ----- + def default_startup(self) -> None: + """ + Default startup: prepare shared memory and reset online data handler. + """ + if self.enable_smm: self.prepare_smm() - self.options['smm'].modify_variable("active_flag", lambda x:1) - self.options["smm"].modify_variable("adapt_flag", lambda x: -1) - + self.smm.modify_variable("active_flag", lambda x: 1) + self.smm.modify_variable("adapt_flag", lambda x: -1) self.odh.prepare_smm() + self.expected_count = {mod: self.window_size for mod in self.odh.modalities} + self.odh.reset() - if self.features is not None: - fe = FeatureExtractor() + def default_model_flag_handler(self) -> bool: + """ + Checks and handles the shared memory flags: if the active flag is not set, + returns False immediately. Also checks if the adapt flag is set and if so, + loads a new predictor. + + Returns + ------- + bool + True if flags are acceptable to run model; False otherwise. + """ + if self.enable_smm: + if not self.smm.get_variable("active_flag")[0, 0]: + return False + if self.smm.get_variable("adapt_flag")[0][0] != -1: + self.load_emg_predictor(self.smm.get_variable("adapt_flag")[0][0]) + self.smm.modify_variable("adapt_flag", lambda x: -1) + return True + + + def default_window_trigger(self) -> bool: + """ + Check whether enough data samples are collected. - self.expected_count = {mod:self.window_size for mod in self.odh.modalities} - # todo: deal with different sampling frequencies for different modalities - self.odh.reset() + Returns + ------- + bool + True if window is ready. + """ + val, count = self.odh.get_data(N=self.window_size) + modality_ready = [count[mod] > self.expected_count[mod] for mod in self.odh.modalities] + return all(modality_ready) - files = {} - while True: - if self.smm: - if not self.options["smm"].get_variable("active_flag")[0,0]: - continue - - if not (self.options["smm"].get_variable("adapt_flag")[0][0] == -1): - self.load_emg_predictor(self.options["smm"].get_variable("adapt_flag")[0][0]) - self.options["smm"].modify_variable("adapt_flag", lambda x: -1) - - val, count = self.odh.get_data(N=self.window_size) - modality_ready = [count[mod] > self.expected_count[mod] for mod in self.odh.modalities] - - if all(modality_ready): - data, count = self._get_data_helper() - - # Extract window and predict sample - window = {mod:get_windows(data[mod], self.window_size, self.window_increment) for mod in self.odh.modalities} - - # Dealing with the case for CNNs when no features are used - if self.features: - model_input = None - for mod in self.odh.modalities: - # todo: features for each modality can be different - mod_features = fe.extract_features(self.features, window[mod], self.predictor.feature_params) - mod_features = self._format_data_sample(mod_features) - if model_input is None: - model_input = mod_features - else: - model_input = np.hstack((model_input, mod_features)) - + def default_on_window(self) -> Tuple[Any, Dict[str, Any]]: + """ + Extract window and prepare model input. This is the same for OnlineEMGRegressors or OnlineEMGClassifiers. + + Returns + ------- + Tuple[Any, dict] + The model input (processed single window ready for model, optionally scaled) and the raw window (raw samples pre scaling). + """ + data, count = self._get_data_helper() + window = {mod: get_windows(data[mod], self.window_size, self.window_increment) for mod in self.odh.modalities} + fe = FeatureExtractor() + if self.features is not None: + model_input_raw = None + for mod in self.odh.modalities: + mod_features = fe.extract_features(self.features, window[mod], feature_dic=self.predictor.feature_params, array=True) + if model_input_raw is None: + model_input_raw = mod_features else: - model_input = window[list(window.keys())[0]] #TODO: Change this - - for mod in self.odh.modalities: - self.expected_count[mod] += self.window_increment - - self.write_output(model_input, window) + model_input_raw = np.hstack((model_input_raw, mod_features)) + if self.scaler is not None: + model_input = self.scaler.transform(model_input_raw) + else: + model_input = model_input_raw + else: + model_input_raw = window[list(window.keys())[0]] + if self.scaler is not None: + model_input = self.scaler.transform(model_input_raw) + else: + model_input = model_input_raw + # TODO: This should be adding a per modality increment since they don't typically have the same Fs + for mod in self.odh.modalities: + self.expected_count[mod] += self.window_increment + return model_input, model_input_raw, window + + def run(self, + block: bool=True): + """Runs the streamer. + + Parameters + ---------- + block: bool (optional), default = True + If True, the run function blocks the main thread. Otherwise it runs in a + seperate process. + """ + self.start_stream(block) + + def _run_helper(self) -> None: + """ + Main loop for online streaming. + """ + # Startup stage + self.on_startup_function_handle() + + while True: + + if self.signal.is_set(): + self.cleanup() + break + # Check flags + if not self.model_flag_handle(): + continue + # Window trigger stage + if not self.window_trigger_function_handle(): + continue + + # Window processing stage + model_input, model_input_raw, window = self.on_window_function_handle() + if model_input is None: + continue + + # Prediction/Postprocessing stage + raw = self.prediction_function_handle(model_input) + processed = self.postprocessing_function_handle(raw, model_input, window) + info = self.format_output_info(processed, model_input, model_input_raw, window) + for writer in self.output_writers: + writer.write(info) + + def cleanup(self) -> None: + self.smm.cleanup() + print("LibEMG -> OnlineStreamer (smm cleaned up).") + print("LibEMG -> OnlineStreamer (process ended).") + + def install_standardization(self, + standardization: np.ndarray | StandardScaler) -> None: + """Install standardization to online model. Standardizes each feature based on training data (i.e., standardizes across windows). + Standardization is only applied when features are extracted and is applied before feature queueing (i.e., features are standardized then queued) + if relevant. To standardize data, use the standardize Filter. + + standardization : np.ndarray or StandardScaler + Data or pre-fit scaler for standardization. + """ + scaler = standardization + + if not isinstance(scaler, StandardScaler): + # Fit scaler to provided data + scaler = StandardScaler().fit(np.array(standardization)) + + self.scaler = scaler + + def stop_running(self) -> None: + """Kills the process streaming decisions. + """ + self.process.terminate() # ----- All of these are unique to each online streamer ---------- - def run(self): - pass + + @abstractmethod + def default_prediction_function(self, model_input: np.ndarray) -> Tuple[Any, Any]: + """ + Default prediction routine. + """ + pass - def stop_running(self): + @abstractmethod + def default_postprocessing_function(self, raw: Any, model_input: np.ndarray, window: Dict[str, Any]) -> Tuple: + """ + Default prediction routine. + """ pass @abstractmethod - def write_output(self, model_input, window): + def format_output_info(self, processed: Tuple[Any, Any, Any], model_input: Any, model_input_raw: Any, window: Dict[str, Any]) -> Dict[str, Any]: + """ + Format output info as dictionary. + """ pass @@ -846,149 +1085,111 @@ class OnlineEMGClassifier(OnlineStreamer): ["classifier_output", (100,4), np.double], #timestamp, class prediction, confidence, velocity ['classifier_input', (100, 1 + 32), np.double], # timestamp <- features -> ] - port: int (optional), default = 12346 - The port used for streaming predictions over UDP. - ip: string (optional), default = '127.0.0.1' - The ip used for streaming predictions over UDP. std_out: bool (optional), default = False If True, prints predictions to std_out. - tcp: bool (optional), default = False - If True, will stream predictions over TCP instead of UDP. - output_format: str (optional), default=predictions - If predictions, it will broadcast an integer of the prediction, if probabilities it broacasts the posterior probabilities + output_writers: OutputWriter, default = None + A list of OutputWriters. This defines what is typically done with the output of the OnlineStreamer. """ - def __init__(self, offline_classifier, window_size, window_increment, online_data_handler, features, - file_path = '.', file=False, smm=False, - smm_items= None, - port=12346, ip='127.0.0.1', std_out=False, tcp=False, - output_format="predictions"): + def __init__(self, + offline_classifier: EMGClassifier, + window_size: int, + window_increment: int, + online_data_handler: Any, + features: Optional[List[Any]], + file_path: str = '.', + file: bool=False, + smm: bool=False, + smm_items: Optional[List[List[Any]]]= None, + std_out: bool=False, + output_writers: Optional[List[Any]]=None) -> None: + - if smm_items is None: - smm_items = [ - ["classifier_output", (100,4), np.double], #timestamp, class prediction, confidence, velocity - ["classifier_input", (100,1+32), np.double], # timestamp, <- features -> - ] - assert 'classifier_input' in [item[0] for item in smm_items], f"'model_input' tag not found in smm_items. Got: {smm_items}." - assert 'classifier_output' in [item[0] for item in smm_items], f"'model_output' tag not found in smm_items. Got: {smm_items}." super(OnlineEMGClassifier, self).__init__(offline_classifier, window_size, window_increment, online_data_handler, - file_path, file, smm, smm_items, features, port, ip, std_out, tcp) - self.output_format = output_format + file_path, file, smm, smm_items, features, std_out, output_writers) self.previous_predictions = deque(maxlen=self.predictor.majority_vote) self.smi = smm_items - - def run(self, block=True): - """Runs the classifier - continuously streams predictions over UDP. - - Parameters - ---------- - block: bool (optional), default = True - If True, the run function blocks the main thread. Otherwise it runs in a - seperate process. - """ - self.start_stream(block) - - def stop_running(self): - """Kills the process streaming classification decisions. - """ - self.process.terminate() - - def write_output(self, model_input, window): - # Make prediction - probabilities = self.predictor.model.predict_proba(model_input) - prediction, probability = self.predictor._prediction_helper(probabilities) - prediction = prediction[0] - # Check for rejection + # Set the streaming pipeline function handles in the classifier subclass. + self.on_startup_function_handle = self.default_startup + self.window_trigger_function_handle = self.default_window_trigger + self.model_flag_handle = self.default_model_flag_handler + self.on_window_function_handle = self.default_on_window + self.prediction_function_handle = self.default_prediction_function + self.postprocessing_function_handle = self.default_postprocessing_function + + def default_prediction_function(self, model_input: np.ndarray) -> Tuple[Any, Any]: + probabilities = self.predictor._predict_proba(model_input) + prediction, _ = self.predictor._prediction_helper(probabilities) + return (prediction[0], probabilities[0]) + + def default_postprocessing_function(self, raw: Any, model_input: np.ndarray, window: Dict[str, Any]): + prediction, probabilities = raw if self.predictor.rejection: - #TODO: Right now this will default to -1 - prediction = self.predictor._rejection_helper(prediction, probability) + prediction = self.predictor._rejection_helper(prediction, probabilities[prediction]) self.previous_predictions.append(prediction) - - # Check for majority vote if self.predictor.majority_vote: values, counts = np.unique(list(self.previous_predictions), return_counts=True) prediction = values[np.argmax(counts)] - - # Check for velocity based control calculated_velocity = "" if self.predictor.velocity: calculated_velocity = " 0" - # Dont check if rejected if prediction >= 0: calculated_velocity = " " + str(self.predictor._get_velocity(window, prediction)) + return (prediction, probabilities, calculated_velocity) + + def format_output_info(self, + processed: Tuple[Any, Any, Any], + model_input: Any, + model_input_raw: Any, + window: Dict[str, Any]) -> Dict[str, Any]: + # Compose a dictionary with all information you wish to send. + prediction, probabilities, calculated_velocity = processed + if isinstance(prediction, np.ndarray): + prediction = prediction.item() + timestamp = time.time() + message = ' '.join([f'{i:.2f}' for i in probabilities]) + calculated_velocity + " " + str(timestamp) + + info = { + "timestamp": timestamp, + "model_output": prediction, + "probability": probabilities, + "velocity": calculated_velocity, + "model_input": model_input, + "model_input_raw": model_input_raw, + "window": window, + "message": message + } + return info - - time_stamp = time.time() - if calculated_velocity == "": - printed_velocity = "-1" - else: - printed_velocity = float(calculated_velocity) - if self.options['std_out']: - print(f"{int(prediction)} {printed_velocity} {time.time()}") - - # Write classifier output: - if self.options['file']: - if not 'file_handle' in self.files.keys(): - self.files['file_handle'] = open(self.options['file_path'] + 'classifier_output.txt', "a", newline="") - writer = csv.writer(self.files['file_handle']) - feat_str = str(model_input[0]).replace('\n','')[1:-1] - row = [f"{time_stamp} {prediction} {probability[0]} {printed_velocity} {feat_str}"] - writer.writerow(row) - self.files['file_handle'].flush() - if "smm" in self.options.keys(): - #assumed to have "classifier_input" and "classifier_output" keys - # these are (1+) - def insert_classifier_input(data): - input_size = self.options['smm'].variables['classifier_input']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, model_input[0]]), data))[:input_size,:] - return data - def insert_classifier_output(data): - output_size = self.options['smm'].variables['classifier_output']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, prediction, probability[0], float(printed_velocity)]), data))[:output_size,:] - return data - self.options['smm'].modify_variable("classifier_input", - insert_classifier_input) - self.options['smm'].modify_variable("classifier_output", - insert_classifier_output) - self.options['classifier_smm_writes'] += 1 - - if self.output_format == "predictions": - message = str(prediction) + calculated_velocity + '\n' - elif self.output_format == "probabilities": - message = ' '.join([f'{i:.2f}' for i in probabilities[0]]) + calculated_velocity + " " + str(time_stamp) - else: - raise ValueError(f"Unexpected value for output_format. Accepted values are 'predictions' and 'probabilities'. Got: {self.output_format}.") - if not self.tcp: - self.sock.sendto(bytes(message, 'utf-8'), (self.ip, self.port)) - else: - self.conn.sendall(str.encode(message)) - - def visualize(self, max_len=50, legend=None): + def visualize(self, + ip: str="127.0.0.1", + port: int=12346, + max_len: int=50, + legend: Optional[List[str]]=None): """Produces a live plot of classifier decisions -- Note this consumes the decisions. Do not use this alongside the actual control operation of libemg. Online classifier has to be running in "probabilties" output mode for this plot. Parameters ---------- + ip: (str) (optional), default=localhost + The ip address the classifier outputs decisions to. + port: (int) (optional), default=12346 + The port the classifier outputs decisions to. max_len: (int) (optional) number of decisions to visualize legend: (list) (optional) - The legend to display on the plot + Labels used to populate legend. Must be passed in order of output classes. """ - #### NOT CURRENTLY WORKING - assert 1==0, "Method not ready" plt.style.use("ggplot") figure, ax = plt.subplots() figure.suptitle("Live Classifier Output", fontsize=16) plot_handle = ax.scatter([],[],c=[]) - - - # make a new socket that subscribes to the libemg events - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(('127.0.0.1', 12346)) - num_classes = len(self.predictor.model.classes_) + num_classes = len(self.predictor.model.classes_) # assumes that user is using an sklearn model cmap = cm.get_cmap('turbo', num_classes) + controller = ClassifierController(output_format="probabilities", num_classes=num_classes, ip=ip, port=port) + if legend is not None: for i in range(num_classes): plt.plot(i, label=legend[i], color=cmap.colors[i]) @@ -999,14 +1200,15 @@ def visualize(self, max_len=50, legend=None): timestamps = [] start_time = time.time() while True: - data, _ = sock.recvfrom(1024) - data = str(data.decode("utf-8")) - probabilities = np.array([float(i) for i in data.split(" ")[:num_classes]]) + data = controller.get_data(['probabilities', 'timestamp']) + if data is None: + continue + probabilities, timestamp = data max_prob = np.max(probabilities) prediction = np.argmax(probabilities) decision_horizon_classes.append(prediction) decision_horizon_probabilities.append(max_prob) - timestamps.append(float(data.split(" ")[-1]) - start_time) + timestamps.append(timestamp - start_time) decision_horizon_classes = decision_horizon_classes[-max_len:] decision_horizon_probabilities = decision_horizon_probabilities[-max_len:] @@ -1026,13 +1228,11 @@ def visualize(self, max_len=50, legend=None): else: return - def _get_data_helper(self): + def _get_data_helper(self) -> Tuple[Dict[str, Any], Dict[str, Any]]: data, counts = self.odh.get_data(N=self.window_size) for key in data.keys(): data[key] = data[key][::-1] return data, counts - - class OnlineEMGRegressor(OnlineStreamer): """OnlineEMGRegressor. @@ -1067,139 +1267,124 @@ class OnlineEMGRegressor(OnlineStreamer): ['model_output', (100, 3), np.double], # timestamp, prediction 1, prediction 2... (assumes 2 DOFs) ['model_input', (100, 1 + 32), np.double], # timestamp <- features -> ] - port: int (optional), default = 12346 - The port used for streaming predictions over UDP. - ip: string (optional), default = '127.0.0.1' - The ip used for streaming predictions over UDP. std_out: bool (optional), default = False If True, prints predictions to std_out. - tcp: bool (optional), default = False - If True, will stream predictions over TCP instead of UDP. + output_writers: OutputWriter, default = None + A list of OutputWriters. This defines what is typically done with the output of the OnlineStreamer. """ - def __init__(self, offline_regressor, window_size, window_increment, online_data_handler, features, - file_path = '.', file = False, smm = False, smm_items = None, - port=12346, ip='127.0.0.1', std_out=False, tcp=False): - if smm_items is None: - # I think probably just have smm_items default to None and remove the smm flag. Then if the user wants to track stuff, they can pass in smm_items and a function to handle them? - smm_items = [ - ['model_input', (100, 1 + 32), np.double], # timestamp <- features -> - ['model_output', (100, 3), np.double] # timestamp, prediction 1, prediction 2... (assumes 2 DOFs) - ] - assert 'model_input' in [item[0] for item in smm_items], f"'model_input' tag not found in smm_items. Got: {smm_items}." - assert 'model_output' in [item[0] for item in smm_items], f"'model_output' tag not found in smm_items. Got: {smm_items}." + def __init__(self, + offline_regressor: EMGRegressor, + window_size: int, + window_increment: int, + online_data_handler: Any, + features: Optional[List[Any]], + file_path: str = '.', + file: bool = False, + smm: bool = False, + smm_items: Optional[List[Any]] = None, + std_out: bool = False, + output_writers: Optional[List[Any]]=None) -> None: super(OnlineEMGRegressor, self).__init__(offline_regressor, window_size, window_increment, online_data_handler, file_path, - file, smm, smm_items, features, port, ip, std_out, tcp) + file, smm, smm_items, features, std_out, output_writers) self.smi = smm_items - - def run(self, block=True): - """Runs the regressor - continuously streams predictions over UDP or TCP. - Parameters - ---------- - block: bool (optional), default = True - If True, the run function blocks the main thread. Otherwise it runs in a - seperate process. - """ - self.start_stream(block) + # Set the streaming pipeline function handles in the classifier subclass. + self.on_startup_function_handle = self.default_startup + self.window_trigger_function_handle = self.default_window_trigger + self.model_flag_handle = self.default_model_flag_handler + self.on_window_function_handle = self.default_on_window + self.prediction_function_handle = self.default_prediction_function + self.postprocessing_function_handle = self.default_postprocessing_function + + def default_prediction_function(self, model_input: np.ndarray) -> Tuple[Any, Any]: + return self.predictor.run(model_input).squeeze() - def stop_running(self): - """Kills the process streaming classification decisions. + def default_postprocessing_function(self, raw: Any, model_input: np.ndarray, window: Dict[str, Any]): """ - self.process.terminate() + Postprocessing: apply additional processing if needed (e.g., deadband). + In this simple example, we return the predictions unmodified (currently a pass-through). + """ + return raw + + def format_output_info(self, + processed: Any, + model_input: Any, + model_input_raw: Any, + window: Dict[str, Any]) -> Dict[str, Any]: + predictions = processed + info = { + "timestamp": time.time(), + "model_output": predictions, + "model_input": model_input, + "model_input_raw": model_input_raw, + "window": window + } + return info - def write_output(self, model_input, window): - # Make prediction - predictions = self.predictor.run(model_input).squeeze() - - time_stamp = time.time() - if self.options['std_out']: - print(f"{predictions} {time.time()}") - - # Write model output: - if self.options['file']: - if not 'file_handle' in self.files.keys(): - self.files['file_handle'] = open(self.options['file_path'] + 'model_output.txt', "a", newline="") - writer = csv.writer(self.files['file_handle']) - feat_str = str(model_input[0]).replace('\n','')[1:-1] - row = [f"{time_stamp} {predictions} {feat_str}"] - writer.writerow(row) - self.files['file_handle'].flush() - - if "smm" in self.options.keys(): - #assumed to have "model_input" and "model_output" keys - # these are (1+) - # This could maybe be moved to OnlineStreamer instead - def insert_model_input(data): - input_size = self.options['smm'].variables['model_input']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, model_input[0]]), data))[:input_size,:] - return data - def insert_model_output(data): - output_size = self.options['smm'].variables['model_output']["shape"][0] - data[:] = np.vstack((np.hstack([time_stamp, predictions]), data))[:output_size,:] - return data - self.options['smm'].modify_variable("model_input", - insert_model_input) - self.options['smm'].modify_variable("model_output", - insert_model_output) - self.options['model_smm_writes'] += 1 - - message = f"{str(predictions)} {str(time_stamp)}\n" - if not self.tcp: - self.sock.sendto(bytes(message, 'utf-8'), (self.ip, self.port)) - else: - self.conn.sendall(str.encode(message)) + def visualize(self, + ip: str="127.0.0.1", + port: int=12346, + max_len: int = 50, + legend: bool = False): + """Plot a live visualization of the online regressor's predictions. Please note that the animation updates every 5 milliseconds, + so keep this in mind when choosing window size and increment. For example, a window increment that's too small may cause delay in the plotting + if the regressor is making predictions faster than the plot can be updated. + + Parameters + ---------- + ip: str (optional), default="localhost" + The ip to monitor for regressor outputs. + port: int (optional), default=12346 + The port to monitor for regressor outputs. + max_len: int (optional), default = 50 + Maximum number of predictions to plot at a time. Defaults to 50. + legend: bool (optional), default = False + True if a legend should be shown, otherwise False. Defaults to False. + """ - def visualize(self, max_len = 50, legend = False): - # TODO: Maybe add an extra option for 2 DOF problems where a single point is plotted on a 2D plane plt.style.use('ggplot') fig, ax = plt.subplots(layout='constrained') - fig.suptitle('Live Regressor Output', fontsize=20) + fig.suptitle('Live Regressor Output', fontsize=16) + ax.set_xlabel('Time (s)') + ax.set_ylabel('Prediction') - # Make local UDP socket whose purpose is to read from regressor output - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind((self.ip, self.port)) - - # Grab sample data to determine # of DOFs - data, _ = sock.recvfrom(1024) - data = str(data.decode('utf-8')) - predictions = self.parse_output(data)[0] + controller = RegressorController(ip=ip, port=port) + + # Wait for controller to start receiving data + predictions = None + while predictions is None: + predictions = controller.get_data('predictions') cmap = cm.get_cmap('turbo', len(predictions)) + plots = [ax.plot([], [], '.', color=cmap.colors[dof_idx], alpha=0.8)[0] for dof_idx in range(len(predictions))] + if legend: handles = [mpatches.Patch(color=cmap.colors[dof_idx], label=f"DOF {dof_idx}") for dof_idx in range(len(predictions))] - decision_horizon_predictions = [] - timestamps = [] start_time = time.time() - while True: - data, _ = sock.recvfrom(1024) - data = str(data.decode('utf-8')) - predictions, timestamp = self.parse_output(data) + + def update(frame, decision_horizon_predictions, timestamps): + data = controller.get_data(['predictions', 'timestamp']) + if data is None: + return + predictions, timestamp = data + timestamps.append(timestamp - start_time) decision_horizon_predictions.append(predictions) timestamps = timestamps[-max_len:] decision_horizon_predictions = decision_horizon_predictions[-max_len:] - if plt.fignum_exists(fig.number): - ax.clear() - ax.set_xlabel('Time (s)') - ax.set_ylabel('Prediction') - for dof_idx in range(len(predictions)): - ax.scatter(timestamps, np.array(decision_horizon_predictions)[:, dof_idx], color=cmap.colors[dof_idx], s=4, alpha=0.8) + for dof_idx in range(len(predictions)): + plots[dof_idx].set_xdata(timestamps) + plots[dof_idx].set_ydata(np.array(decision_horizon_predictions)[:, dof_idx]) - if legend: - ax.legend(handles=handles, loc='upper right') - plt.draw() - plt.pause(0.01) - else: - # Figure was closed - return + if legend: + ax.legend(handles=handles, loc='upper right') - @staticmethod - def parse_output(message): - outputs = re.findall(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?", message) - outputs = list(map(float, outputs)) - predictions = outputs[:-1] - timestamp = outputs[-1] - return predictions, timestamp + ax.relim() + ax.autoscale_view() + return plots + + _ = FuncAnimation(fig, partial(update, decision_horizon_predictions=[], timestamps=[]), interval=5, blit=False) # must return value or animation won't work + plt.show() diff --git a/libemg/environments/__init__.py b/libemg/environments/__init__.py new file mode 100644 index 00000000..8fb7cbbc --- /dev/null +++ b/libemg/environments/__init__.py @@ -0,0 +1 @@ +from libemg.environments import _base, controllers, emg_hero, fitts,curricular_fitts \ No newline at end of file diff --git a/libemg/environments/_base.py b/libemg/environments/_base.py new file mode 100644 index 00000000..1aba9ac8 --- /dev/null +++ b/libemg/environments/_base.py @@ -0,0 +1,88 @@ +import json +from abc import ABC, abstractmethod +from multiprocessing import Process +from pathlib import Path +import pickle +import os + +os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide" # hide pygame welcome message +import pygame + +from libemg.environments.controllers import Controller + +class Environment(ABC): + def __init__(self, controller: Controller, fps: int, log_dictionary: dict, save_file: str | None = None): + """Abstract environment interface for pygame environments. + + Parameters + ---------- + controller : Controller + Controller instance that defines how control actions are parsed. + fps : int + Frames per second (Hz). + log_dictionary : dict + Dictionary containing metrics to log. + save_file : str | None, optional + Name of save file (e.g., log.pkl). Supports .json and .pkl file formats. If None, no results are saved. Defaults to None. + """ + # Assumes this is a pygame environment + self.controller = controller + self.done = False # flag to determine when loop should be exited + self.clock = pygame.time.Clock() + self.fps = fps + self.log_dictionary = log_dictionary + self.save_file = save_file + self.process = Process(target=self.run, daemon=True) + + @abstractmethod + def game_setup(self): + # setup things like font in here. + ... + + def run(self): + """Run environment in main loop. Blocks all further execution. Results are saved after task is completed.""" + pygame.init() + pygame.font.init() + pygame.mixer.init() + + self.game_setup() + while not self.done: + self._run_loop() + pygame.display.update() + self.clock.tick(self.fps) + + self.save_results() + + pygame.display.quit() + pygame.mixer.quit() + pygame.font.quit() + pygame.quit() + + @abstractmethod + def _run_loop(self): + ... + + def run_helper(self, block=True): + if block: + self.process.start() + self.process.join() + else: + self.process.start() + + + def save_results(self): + if self.save_file is None: + # Don't log anything + return + + file = Path(self.save_file).absolute() + file.parent.mkdir(parents=True, exist_ok=True) # create parent directories if they don't exist + + if file.suffix == '.pkl': + with open(self.save_file, 'wb') as f: + pickle.dump(self.log_dictionary, f) + elif file.suffix == '.json': + with open(self.save_file, 'w') as f: + json.dump(self.log_dictionary, f) + else: + raise ValueError(f"Unexpected file format '{file.suffix}'. Choose from '.pkl' or '.json'.") diff --git a/libemg/environments/controllers.py b/libemg/environments/controllers.py new file mode 100644 index 00000000..8b02af0c --- /dev/null +++ b/libemg/environments/controllers.py @@ -0,0 +1,283 @@ +from abc import ABC, abstractmethod +from typing import overload +import socket +import re +import time + +import numpy as np +import pygame + + +class Controller(ABC): + def __init__(self): + """Abstract controller interface for controlling environments. Runs as a Process in a separate thread and collects control signals continuously. Call start() to start collecting control signals.""" + self.info_function_map = { + 'predictions': self._parse_predictions, + 'pc': self._parse_proportional_control, + 'timestamp': self._parse_timestamp + } + # TODO: Maybe add a flag for continuous vs. not continuous... not sure if that's needed though + + @overload + def get_data(self, info: list[str]) -> tuple | None: + ... + + @overload + def get_data(self, info: str) -> list[float] | None: + ... + + def get_data(self, info: list[str] | str) -> tuple | list[float] | None: + """Get data from current action. This method should be used to access data to ensure that all parsing happens on the same action. Velocity control must be enabled when using proportional control. + + Parameters + ---------- + info: list[str] or str + Type of data requested. Must be a string in info_function_map. + """ + if isinstance(info, str): + # Cast to list + info = [info] + + action = self._get_action() + if action is None: + # Action didn't occur + return None + + + data = [] + for info_type in info: + try: + parse_function = self.info_function_map[info_type] + result = parse_function(action) + except KeyError as e: + raise ValueError(f"Unexpected value for info type. Accepted parameters are: {list(self.info_function_map.keys())}. Got: {info_type}.") from e + + data.append(result) + + data = tuple(data) # convert to tuple so unpacking can be used if desired + if len(data) == 1: + data = data[0] + return data + + @abstractmethod + def _parse_predictions(self, action: str) -> list[float]: + """Parse the latest prediction from a message. + + Parameters + ---------- + action: str + Message to parse. + + Returns + ---------- + list[float] + List of predictions. + """ + ... + + @abstractmethod + def _parse_proportional_control(self, action: str) -> list[float]: + """Parse the latest proportional control info from a message. + + Parameters + ---------- + action: str + Message to parse. + + Returns + ---------- + list[float] + List of proportional control values. + """ + ... + + + @abstractmethod + def _parse_timestamp(self, action: str) -> float: + """Parse the latest timestamp from a message. + + Parameters + ---------- + action : str + Message to parse. + + Returns + ------- + float + Timestamp. + """ + ... + + @abstractmethod + def _get_action(self) -> str | None: + """Grab the latest action. + + Returns + ---------- + str or None + Latest action or None if no action has occurred. + """ + ... + + +class SocketController(Controller): + def __init__(self, ip: str = '127.0.0.1', port: int = 12346) -> None: + """Controller interface for controlling environments using a UDP socket. + Runs as a Process in a separate thread and collects control signals continuously. Call start() to start collecting control signals. + + Parameters + ---------- + ip: str + IP address for UDP socket used to read messages. + port: int + Port for UDP socket used to read messages. + """ + super().__init__() + self.ip = ip + self.port = port + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.bind((self.ip, self.port)) + self.sock.setblocking(False) + + @abstractmethod + def _parse_predictions(self, action: str) -> list[float]: + ... + + @abstractmethod + def _parse_proportional_control(self, action: str) -> list[float]: + ... + + @abstractmethod + def _parse_timestamp(self, action: str) -> float: + ... + + def _get_action(self): + try: + data, _ = self.sock.recvfrom(1024) + action = str(data.decode('utf-8')) + except BlockingIOError: + action = None + return action + + +class ClassifierController(SocketController): + def __init__(self, output_format: str, num_classes: int, ip: str = '127.0.0.1', port: int = 12346) -> None: + """Controller interface for controlling environments using a classifier. + Runs as a Process in a separate thread and collects control signals continuously. Call start() to start collecting control signals. + + Parameters + ---------- + output_format: str + Output format of classifier. Accepted values are 'probabliities' and 'predictions. + num_classes: int + Number of classes in classification problem. + ip: str + IP address for UDP socket used to read messages. + port: int + Port for UDP socket used to read messages. + """ + super().__init__(ip, port) + self.info_function_map['probabilities'] = self._parse_probabilities # add option for classifier to parse probabilities + self.output_format = output_format + self.num_classes = num_classes # could remove this parameter if we always sent a velocity value (e.g., set it to -1 if velocity control is not enabled) + self.error_message = f"Unexpected value for output_format. Accepted values are 'predictions' or 'probabilities'. Got: {output_format}." + if output_format not in ['predictions', 'probabilities']: + raise ValueError(self.error_message) + + def _parse_predictions(self, action: str) -> list[float]: + if self.output_format == 'predictions': + return [float(action.split(' ')[0])] + elif self.output_format == 'probabilities': + probabilities = self._parse_probabilities(action) + return [float(np.argmax(probabilities))] + + raise ValueError(self.error_message) + + def _parse_timestamp(self, action: str) -> float: + if self.output_format == 'predictions': + raise ValueError("Output format is set to 'predictions', so timestamp cannot be parsed because timestamp is not sent when output_format='predictions'.") + return float(action.split(' ')[-1]) + + def _parse_proportional_control(self, action: str) -> list[float]: + components = action.split(' ') + if self.output_format == 'predictions': + try: + return [float(components[1])] + except IndexError as e: + raise IndexError('Attempted to parse proportional control, but no velocity value was found. Please enable velocity control in the EMGClassifier.') from e + elif self.output_format == 'probabilities': + # Assume that user has enabled velocity control and take the value before the timestamp + if len(components) < (self.num_classes + 2): + raise ValueError('Did not find velocity value in message. Please enable velocity control in the EMGClassifier.') + return [float(components[-2])] + + raise ValueError(self.error_message) + + def _parse_probabilities(self, action: str) -> list[float]: + if self.output_format == 'predictions': + raise ValueError("Output format is set to 'predictions', so probabilities cannot be parsed. Set output_format='probabilities' if this functionality is needed.") + + return [float(prob) for prob in action.split(' ')[:self.num_classes]] + + +class RegressorController(SocketController): + """Controller interface for controlling environments using a regressor. + Runs as a Process in a separate thread and collects control signals continuously. Call start() to start collecting control signals. + + Parameters + ---------- + ip: str + IP address for UDP socket used to read messages. + port: int + Port for UDP socket used to read messages. + """ + def _parse_predictions(self, action: str) -> list[float]: + outputs = re.findall(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?", action) + outputs = list(map(float, outputs)) + return outputs[:-1] + + def _parse_timestamp(self, action: str) -> float: + outputs = re.findall(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?", action) + outputs = list(map(float, outputs)) + return outputs[-1] + + def _parse_proportional_control(self, action: str) -> list[float]: + predictions = self._parse_predictions(action) + return [1. for _ in predictions] # proportional control is built into prediction, so return 1 for each DOF + + +class KeyboardController(Controller): + def __init__(self) -> None: + """Controller interface for controlling environments using a keyboard. start() method is not required because this doesn't run in a separate thread.""" + # No run method b/c using pygame events in another thread doesn't work (pygame.init is required but isn't thread-safe) + super().__init__() + self.keys = [ + pygame.K_LEFT, + pygame.K_RIGHT, + pygame.K_UP, + pygame.K_DOWN, + pygame.K_1, + pygame.K_2, + pygame.K_3, + pygame.K_4 + ] + + def _parse_predictions(self, action: str) -> list[float]: + return [float(action)] + + def _parse_proportional_control(self, action: str) -> list[float]: + predictions = self._parse_predictions(action) + return [1. for _ in predictions] # proportional control is built into prediction, so return 1 for each key pressed + + def _parse_timestamp(self, action: str) -> float: + return time.time() + + def _get_action(self): + keys = pygame.key.get_pressed() + keys_pressed = [key for key in self.keys if keys[key]] + if len(keys_pressed) == 0: + # No data received + keys_pressed = [-1] + + key_pressed = keys_pressed[0] # take the first value, maybe change later to support combined keys + return str(key_pressed) diff --git a/libemg/environments/curricular_fitts.py b/libemg/environments/curricular_fitts.py new file mode 100644 index 00000000..04cc97a5 --- /dev/null +++ b/libemg/environments/curricular_fitts.py @@ -0,0 +1,557 @@ +from libemg.output_writer import OutputWriter +from libemg.environments.controllers import Controller +from libemg.environments._base import Environment +from libemg.adaptation._base import produce_tciil_feedback +from dataclasses import dataclass +from multiprocessing import Process +from typing import Sequence +import time +import pickle +import pygame +import random +import numpy as np + +@dataclass +class CurricularFittsConfig: + """Configuration for the Curricular Fitts environment. + + Parameters + ---------- + num_trials : int + Number of fitts law target to spawn before the game is done. + width : int + Width of the main panel in pixels. + height : int + Height of the main panel in pixels. + fps : int + Frames per second (Hz). + block_height : int + Height of the info-block in pixels. + block_width : int + Width of the info-block in pixels. + default_target_radius : int + Default radius of the target in pixels. + default_timeout : float + Default timeout for the target in seconds. + default_speed : float + Default speed of the target in pixels per second. + cursor_radius : int + Radius of the cursor in pixels. + brownian_motion : bool + Whether the target should move (randomly). + cursor_speed_multiplier : int + The number multiplied by the controller output to determine the cursor speed. + color_bg : tuple[int, int, int] + Background color of the main panel in RGB format. + color_block_border : tuple[int, int, int] + Border color of the info-block in RGB format. + color_block_fill : tuple[int, int, int] + Fill color of the info-block in RGB format. + color_cursor : tuple[int, int, int] + Color of the cursor in RGB format. + color_target : tuple[int, int, int] + Color of the target in RGB format. + """ + # game parameters + num_trials: int = 100 + + # main panel parameters + width: int = 1000 + height: int = 1080 + fps: int = 60 + + # info-block + block_height: int = 80 + block_width: int = 1000 + + # cursor / target defaults + default_target_radius: int = 25 + default_timeout: float = 10.0 + default_speed: float = 1.0 + cursor_radius: int = 5 + brownian_motion: bool = False + cursor_speed_multiplier: int = 20 + target_countdown: float = 0.5 # seconds + + # color scheme - darkula by default + color_bg: tuple[int, int, int] = (0,0,0) + color_block_border: tuple[int, int, int] = (10, 0, 71) + color_block_fill: tuple[int, int, int] = (0, 70, 135) + color_cursor: tuple[int, int, int] = (0, 255, 210) + color_target: tuple[int, int, int] = (255, 68, 153) + color_target_good: tuple[int, int, int] = (255, 0, 0) # Color for successful target acquisition + + # controller related parameters + controller_fields : tuple[str, str] = ('predictions', 'timestamp') # for regression; add 'pc' for classification + controller_map : tuple[int, int] = (1,1) + + # feedback configuration + feedback_handle: callable = produce_tciil_feedback + +class Target: + def __init__(self, + screen, + config: CurricularFittsConfig, + radius : int = 10, + speed: float = 1): + self.screen = screen + self.config = config + self.color = [config.color_target, config.color_target_good] + self.contact = 0 # 0: not in contact, 1: in contact + self.radius = radius + self.randomize_location() + self.speed = speed + self.direction = [random.uniform(-1, 1), random.uniform(-1, 1)] + + def randomize_location(self): + x = random.randint(self.radius, self.config.width - self.radius) + y = random.randint(self.config.block_height + self.radius, self.config.height - self.radius) # Spawn below the information block + self.position = [x, y] + + def update(self): + if self.config.brownian_motion: + # Update target position with Brownian motion + self.position[0] += self.direction[0] * self.speed + self.position[1] += self.direction[1] * self.speed + + # Randomly change direction occasionally + if random.random() < 0.01: # Adjust frequency of direction change + self.direction = [random.uniform(-1, 1), random.uniform(-1, 1)] + + # Keep the target within the bounds of the play area + if self.position[0] < self.radius or self.position[0] > self.config.width - self.radius: + self.direction[0] *= -1 + if self.position[1] < self.config.block_height + self.radius or self.position[1] > self.config.height - self.radius: + self.direction[1] *= -1 + + def draw(self): + if self.contact: + pygame.draw.circle(self.screen, self.color[1], (int(self.position[0]), int(self.position[1])), self.radius) + else: + pygame.draw.circle(self.screen, self.color[0], (int(self.position[0]), int(self.position[1])), self.radius) + + def reset(self): + self.randomize_location() # Spawn in a new location + +class BaseTargetGenerator: + def __init__(self, + N, + F, + P): + self.N = N # Number of trials or reversals + self.F = F # Factor for increase on miss + self.P = P # Factor for decrease on hit + self.yields = [] # Store yields for logging + + def save_yields(self, filename="target_yields.pkl"): + with open(filename, 'wb') as f: + pickle.dump(self.yields, f) + +class ConstantTargetGenerator(BaseTargetGenerator): + def __init__(self, + config: CurricularFittsConfig): + super().__init__(N=config.num_trials, + F=0, # nothing changes + P=0 # nothing changes + ) + self.config = config + self.current_radius = config.default_target_radius + self.current_timeout = config.default_timeout + self.counter = 0 + + def generate(self, + result: bool): + # result = 1 : Passed + # result = 0 : Failed (via timeout or miss) + if self.counter < self.N: + self.counter += 1 + self.yields.append((self.current_radius, 0, self.current_timeout)) + return self.current_radius, 0, self.current_timeout + +class RadiusTargetGenerator(BaseTargetGenerator): + def __init__(self, + config: CurricularFittsConfig, + F, + P): + super().__init__(N=config.num_trials, + F=F, + P=P) + self.config = config + self.current_radius = config.default_target_radius + self.last_result = -1 + self.counter = 0 + self.reversal_counter = 0 + + def generate(self, result): + # a success + if result == 1: + factor = 1 - (self.P / 100) + if int(self.current_radius * factor) == self.current_radius and self.P != 0: + # if its just marginally smaller, reduce by a whole pixel. + self.current_radius = self.current_radius - 1 + else: + self.current_radius *= factor + # a fail + elif result == 0: + factor = 1 + (self.F / 100) + if int(self.current_radius * factor) == self.current_radius and self.F != 0: + self.current_radius = self.current_radius + 1 + else: + self.current_radius *= factor + # first spawn + else: + pass + self.current_radius = int(max([self.config.cursor_radius, self.current_radius]))# don't allow it smaller than cursor radius + + # check reversals + if self.last_result != -1: + if result != self.last_result: + self.reversal_counter += 1 + + self.counter += 1 + self.last_result = result + + self.yields.append((self.current_radius, 0, self.config.default_timeout)) # Assume constant radius and speed of zero + return self.current_radius, 0, self.config.default_timeout + +class SpeedTargetGenerator(BaseTargetGenerator): + def __init__(self, + config : CurricularFittsConfig, + F, + P): + super().__init__(N=config.num_trials, F=F, P=P) + self.config = config + self.current_speed = config.default_speed + self.last_result = -1 + self.counter = 0 + self.reversal_counter = 0 + + def generate(self, result): + # a success + if result == 1: + self.current_speed *= (1 + self.P / 100) + # a fail + elif result == 0: + self.current_speed *= (1 - self.F / 100) + # first spawn + else: + pass + + # check reversals + if self.last_result != -1: + if result != self.last_result: + self.reversal_counter += 1 + + self.counter += 1 + self.last_result = result + + self.yields.append((self.config.default_target_radius, self.current_speed, self.config.default_timeout)) # Assume constant radius and speed + return self.config.default_target_radius, self.current_speed, self.config.default_timeout + +class TimeoutTargetGenerator(BaseTargetGenerator): + def __init__(self, + config : CurricularFittsConfig, + F, + P): + super().__init__(N=config.num_trials, + F=F, + P=P) + self.config = config + self.current_timeout = config.default_timeout + self.last_result = -1 + self.counter = 0 + self.reversal_counter = 0 + + def generate(self, result): + # a success + if result == 1: + self.current_timeout *= (1 - self.P / 100) + # a fail + elif result == 0: + self.current_timeout *= (1 + self.F / 100) + # first spawn + else: + pass + + # check reversals + if self.last_result != -1: + if result != self.last_result: + self.reversal_counter += 1 + + self.counter += 1 + self.last_result = result + + self.yields.append((self.config.default_target_radius, 0, self.current_timeout)) # Assume constant radius and speed of 0 + return self.config.default_target_radius, 0, self.current_timeout + +class Log: + def __init__(self): + self.entries = { + "trial_number": [], + "target_position": [], + "cursor_position": [], + "target_size": [], + "feedback": [], + "timestamp": [] + } + + def record(self, trial_number, target_position: list, cursor_position: list, target_size, feedback:list, timestamp): + self.entries['trial_number'].append(trial_number) + self.entries['target_position'].append(target_position.copy()) + self.entries['cursor_position'].append(cursor_position.copy()) + self.entries['target_size'].append(target_size) + self.entries['feedback'].append(feedback.copy()) + self.entries['timestamp'].append(timestamp) + + def save(self, dir, trial_number, result): + filename = f'trial_log_{trial_number}_{result}.pkl' + with open(dir + "/" + filename, 'wb') as f: + pickle.dump(self, f) + + def __add__(self, obj): + log = Log() + log.entries['trial_number'].extend(self.entries['trial_number']) + log.entries['trial_number'].extend(obj.entries['trial_number']) + log.entries['target_position'].extend(self.entries['target_position']) + log.entries['target_position'].extend(obj.entries['target_position']) + log.entries['cursor_position'].extend(self.entries['cursor_position']) + log.entries['cursor_position'].extend(obj.entries['cursor_position']) + log.entries['target_size'].extend(self.entries['target_size']) + log.entries['target_size'].extend(obj.entries['target_size']) + log.entries['feedback'].extend(self.entries['feedback']) + log.entries['feedback'].extend(obj.entries['feedback']) + log.entries['timestamp'].extend(self.entries['timestamp']) + log.entries['timestamp'].extend(obj.entries['timestamp']) + return log + +class InformationBlock: + def __init__(self, + screen, + config: CurricularFittsConfig, + font_size: int = 50, + header_font_size: int = 30, + countdown_seconds: int = 60): + self.screen = screen + self.config = config + self.rect = pygame.Rect(0, 0, config.block_width, config.block_height) + self.font = pygame.font.Font(None, font_size) + self.header_font = pygame.font.Font(None, header_font_size) + self.update_countdown_seconds(countdown_seconds) + self.update_trial_number(0) + self.update_metadata("") + self.color = config.color_cursor # Default text color + + def update_trial_number(self, trial_number): + self.trial_number = trial_number + + def update_metadata(self, metadata): + self.metadata = metadata + + def update_countdown_seconds(self, countdown_seconds): + self.countdown_seconds = countdown_seconds + self.start_time = time.time() + + def get_remaining_time(self): + elapsed_time = time.time() - self.start_time + remaining_time = self.countdown_seconds - elapsed_time + return max(remaining_time, 0) + + def draw(self): + pygame.draw.rect(self.screen, self.config.color_block_fill, self.rect) + pygame.draw.rect(self.screen, self.config.color_block_border, self.rect, 2) # Draw border + + headers = ["Trial:", "Time:", "Metadata:"] + values = [f"{self.trial_number}", f"{self.get_remaining_time():.1f}s", self.metadata] + + header_surfaces = [self.header_font.render(header, True, self.color) for header in headers] + value_surfaces = [self.font.render(value, True, self.color) for value in values] + + column_width = self.rect.width // 3 + + for i, header_surface in enumerate(header_surfaces): + header_x = self.rect.x + (i * column_width) + (column_width - header_surface.get_width()) // 2 + value_x = self.rect.x + (i * column_width) + (column_width - value_surfaces[i].get_width()) // 2 + + header_y = self.rect.y + 5 + value_y = header_y + header_surface.get_height() + 5 + + self.screen.blit(header_surface, (header_x, header_y)) + self.screen.blit(value_surfaces[i], (value_x, value_y)) + + +class Cursor: + def __init__(self, + screen, + config: CurricularFittsConfig): + self.screen = screen + self.config = config + self.radius = config.cursor_radius + self.color = config.color_cursor # Cursor color + self.position = [0, 0] # Initial position + + def update(self, update_direction): + if update_direction is not None: + self.position[0] += update_direction[0] * self.config.cursor_speed_multiplier * self.config.controller_map[0] + self.position[1] += update_direction[1] * self.config.cursor_speed_multiplier * self.config.controller_map[1] + + # Ensure cursor stays in the bounds of the screen + self.position[0] = max(0, self.position[0]) + self.position[0] = min(self.config.width - self.radius, self.position[0]) + self.position[1] = max(0, self.position[1]) + self.position[1] = min(self.config.height - self.radius, self.position[1]) + + def draw(self): + pygame.draw.circle(self.screen, self.color, (int(self.position[0]), int(self.position[1])), self.radius) + + +class CurricularFitts(Environment): + """ + """ + + def __init__(self, + controller: Controller, + config: CurricularFittsConfig, + target_generator: BaseTargetGenerator = None, + environment_ow: list[OutputWriter] = [], + save_file: str | None = None): + + super().__init__(controller, + fps=config.fps, + log_dictionary=None, + save_file=save_file) + + self.config = config + + self.target_generator = target_generator or ConstantTargetGenerator( + radius = self.config.default_target_radius, + timeout = self.config.default_timeout + ) + + self.environment_ow = environment_ow + + self.predictions = None + self.timestamp = None + + self.game_feedback = None + self.model_feedback = None + + def game_setup(self): + self.screen = pygame.display.set_mode((self.config.width, self.config.height)) + pygame.display.set_caption("LibEMG -> Curricular Fitts Law") + + self.cursor = Cursor(self.screen, self.config) + self.target = Target(self.screen, self.config, radius=self.config.default_target_radius, speed=self.config.default_speed) + self.log = Log() + + self.trial_number = 1 + self._start_new_trial(initial=True) + + def _start_new_trial(self, + initial: bool = False, + result: bool = True): + """ + Get new target parameters (size, timeout, speed) from the generator, and use this to setup the next target. + """ + # if we've finished a trial, save the log, increment the trial counter + if not initial: + self.log.save(self.save_file, self.trial_number, result) + self.log = Log() + self.trial_number += 1 + + self.cursor_timer = time.time() + + # get the radius, speed, and timeout from the generator + radius, speed, timeout = self.target_generator.generate(result) + + self.timeout = timeout + self.target.radius = radius + self.target.speed = speed + self.target.randomize_location() + + # update the information block + self.info_block = InformationBlock( + self.screen, + self.config, + countdown_seconds = timeout + ) + self.info_block.update_trial_number(self.trial_number) + + self.trial_distance = np.linalg.norm([x - y for x,y in zip(self.cursor.position, self.target.position)]) + + def _run_loop(self): + self.input() + self.update() + self.draw() + if self.trial_number > self.config.num_trials: + self.done = True + + def input(self): + self.pygame_inputs() + self.controller_inputs() + self.check_collisions() + + def pygame_inputs(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + self.done = True + + def controller_inputs(self): + # get the controller message + + # get the feedback ready for this message + data = self.controller.get_data(self.config.controller_fields) + if data is not None: + self.predictions = data[0] + self.direction = [i*j for i,j in zip(self.predictions, self.config.controller_map)] + self.timestamp = data[1] + # game feedback is in the game space, i.e, down is positive y, right is positive x. + self.game_feedback = self.config.feedback_handle(self.cursor.position, self.direction, self.target.position, self.target.radius, self.trial_distance) + # model feedback is in the classifier space, so we need to transform it BACK via multiplying by controller_map again + self.model_feedback = [i*j for i,j in zip(self.game_feedback, self.config.controller_map)] + self.info = {'timestamp': self.timestamp, + 'environment_feedback': self.model_feedback, + 'game_feedback': self.game_feedback, + 'trial': self.trial_number, + 'prediction': self.predictions, + 'direction': self.direction} + self.log.record(self.trial_number, self.target.position, self.cursor.position, self.target.radius, self.game_feedback, self.timestamp) + + if self.environment_ow is not None: + + self.environment_ow[0].write(self.info) + # make self._info, + # save last timestamp, last controller output, etc. + + + def check_collisions(self): + target_rect = pygame.Rect(self.target.position[0] - self.target.radius, + self.target.position[1] - self.target.radius, + self.target.radius * 2, self.target.radius * 2) + + # TODO: Make a countdown for this to acquire the target + if target_rect.collidepoint(self.cursor.position): + self.target.contact = 1 + if time.time() - self.cursor_timer > self.config.target_countdown: + self.info_block.update_metadata("Hit!") + self._start_new_trial(initial=False, result=1) + else: + self.cursor_timer = time.time() + self.target.contact = 0 + + def update(self): + self.target.update() + self.cursor.update(self.predictions) + + if self.info_block.get_remaining_time() <= 0: + self.info_block.update_metadata("Timeout!") + self._start_new_trial(initial=0, result=0) + + def draw(self): + self.screen.fill(self.config.color_bg) + self.info_block.draw() + self.target.draw() + self.cursor.draw() + pygame.display.flip() + + def save_results(self): + self.target_generator.save_yields() \ No newline at end of file diff --git a/libemg/environments/emg_hero.py b/libemg/environments/emg_hero.py new file mode 100644 index 00000000..46d3f0d1 --- /dev/null +++ b/libemg/environments/emg_hero.py @@ -0,0 +1,176 @@ +import time +from typing import Sequence + +import pygame +import numpy as np + +from libemg.environments.controllers import Controller +from libemg.environments._base import Environment + + +class _Note: + def __init__(self, type): + self.type = type + assert self.type in [0,1,2,3] + y_poses = [75, 200, 325, 450] + colors = [(255, 0, 0),(0, 255, 0),(0, 0, 255),(255, 165, 0)] + # Based on the type, set up the note + self.x_pos = y_poses[self.type] + self.y_pos = 0 + self.color = colors[self.type] + self.length = 35 * (5 * np.random.random()) # Random integer between 1 and 5 + + def move_note(self, speed=5): + self.y_pos += speed + if self.y_pos > 1000: + return -1 + return 0 + + +class EMGHero(Environment): + def __init__(self, controller: Controller, prediction_map: dict | None = None, test_time: int = 120, min_speed: float = 2.5, max_speed: float = 7.5, min_time: float = 0.6, max_time: float = 2.2, + img_files: Sequence | None = None, save_file: str | None = None, fps: int = 60): + """Guitar Hero style game that tests user's ability to elicit contractions at specific times. Game speed progressively gets quicker over the course of the task. + Simultaneous contractions, such as with regression, are not currently supported. + + Parameters + ---------- + controller : Controller + Interface to parse predictions which determine the notes being played. + prediction_map : dict | None, optional + Maps received control commands to notes being played. If None, a standard map for classifiers is created where 0, 1, 2, 3, 4 are mapped to 0, 1, -1, 2, and 3, respectively. + For custom mappings, pass in a dictionary where keys represent received control signals (from the Controller) and values map to actions in the environment. + Accepted actions are: -1 (play nothing), 0 (first note), 1 (second note), 2 (third note), 3 (fourth note). All of these actions must be represented by a single key in the dictionary. + Defaults to None. + test_time : int, optional + Amount of time test will take (in seconds). Defaults to 120. + min_speed : float, optional + Minimum game speed. Defaults to 2.5. + max_speed : float, optional + Maximum game speed. Defaults to 7.5. + min_time : float, optional + Minimum time between notes (in seconds). + max_time : float, optional + Maximum time between notes (in seconds). + img_files : Sequence | None, optional + List of image filenames to put at the bottom of the display to show users which notes are represented by which gestures. If None, no images are shown. Defaults to None. + save_file : str | None, optional + Path to save file for logging metrics. If None, no results are logged. Defaults to None. + fps : int, optional + Frames per second (in Hz). Defaults to 60. + """ + if prediction_map is None: + prediction_map = { + 0: 0, + 1: 1, + 2: -1, + 3: 2, + 4: 3 + } + + assert set(np.unique(list(prediction_map.values()))) == set([0, 1, -1, 2, 3]), f"Did not find all commands (0, 1, 2, 3, -1) represented as values in prediction_map. Got: {prediction_map}." + + if img_files is None: + img_files = [] + + log_dictionary = { + "times": [], + "notes": [], + "button_pressed": [] + } + super().__init__(controller, fps=fps, log_dictionary=log_dictionary, save_file=save_file) + self.prediction_map = prediction_map + self.test_time = test_time + self.min_speed = min_speed + self.max_speed = max_speed + self.min_time = min_time + self.max_time = max_time + + self.imgs = [] + if len(img_files) > 0: + assert len(img_files) == 4, f"Expected 4 image files, but got {len(img_files)}." + for i in img_files: + self.imgs.append(pygame.transform.smoothscale(pygame.image.load(i), (100,100))) + + + + + def game_setup(self): + pygame.display.set_caption('Testing Environment') + self.font = pygame.font.SysFont('Comic Sans MS', 30) + self.screen = pygame.display.set_mode([525, 700]) + + + self.last_note = time.time() + self.start_time = time.time() + self.test_time + + self.notes = [] + self.key_pressed = -1 + + + def _run_loop(self): + # Run until the user asks to quit + gen_time = ((self.start_time - time.time())/self.test_time) * (self.max_time - self.min_time) + self.min_time + if time.time() - self.last_note > gen_time: # Generation + new_note = np.random.randint(0,4) + self.notes.append(_Note(new_note)) + self.last_note = time.time() + + if self.start_time - time.time() <= 0: + # Time's up + self.done = True + + # Fill the background with white + self.screen.fill((255, 255, 255)) + + # Update time remaining + text = self.font.render('{0:.1f}'.format(self.start_time - time.time()), True, (0,0,0), (255,255,255)) + textRect = text.get_rect() + textRect.center = (470, 25) + self.screen.blit(text, textRect) + + # Did the user click the window close button? + for event in pygame.event.get(): + if event.type == pygame.QUIT: + self.done = True + + predictions = self.controller.get_data('predictions') + + if predictions is not None: + # Received data + assert len(predictions) == 1, f"Expected a single prediction, but got {len(predictions)}. Controllers that produce multiple predictions, like RegressionController, are not currently supported." + self.key_pressed = self.prediction_map[predictions[0]] + + # Draw notes on bottom of screen + pygame.draw.circle(self.screen, (255, 0, 0), (75, 500), 35, width=8 - (self.key_pressed==0) * 8) + pygame.draw.circle(self.screen, (0, 255, 0), (200, 500), 35, width=8 - (self.key_pressed==1) * 8) + pygame.draw.circle(self.screen, (0, 0, 255), (325, 500), 35, width=8 - (self.key_pressed==2) * 8) + pygame.draw.circle(self.screen, (255, 165, 0), (450, 500), 35, width=8 - (self.key_pressed==3) * 8) + + # Move and deal with notes coming down + for n in self.notes: + speed = (1 - (self.start_time - time.time())/self.test_time) * (self.max_speed - self.min_speed) + self.min_speed + if n.move_note(speed=speed) == -1: + self.notes.remove(n) + # Check to see if the shape is over top of the note + w = 0 + if n.type == self.key_pressed and n.y_pos >= 500 and n.y_pos - 60 - n.length <= 500: + w = 5 + pygame.draw.circle(self.screen, n.color, (n.x_pos, n.y_pos), 35, width=w) + pygame.draw.rect(self.screen, n.color, (n.x_pos - 20, n.y_pos - 30 - n.length, 40, n.length), width=w) + pygame.draw.circle(self.screen, n.color, (n.x_pos, n.y_pos - 60 - n.length), 35, width=w) + + pygame.draw.rect(self.screen, (255,255,255), (0, 550, 1000, 300)) + + # Draw images on screen + if len(self.imgs) > 0: + self.screen.blit(self.imgs[0], (25,550)) + self.screen.blit(self.imgs[1], (150,550)) + self.screen.blit(self.imgs[2], (275,550)) + self.screen.blit(self.imgs[3], (400,550)) + + # Log everything + self.log_dictionary['times'].append(time.time()) + self.log_dictionary['notes'].append([[n.type, n.y_pos, n.length] for n in self.notes]) + self.log_dictionary['button_pressed'].append(self.key_pressed) + \ No newline at end of file diff --git a/libemg/environments/fitts.py b/libemg/environments/fitts.py new file mode 100644 index 00000000..80daa346 --- /dev/null +++ b/libemg/environments/fitts.py @@ -0,0 +1,445 @@ +import math +import time +from dataclasses import dataclass + +import pygame +import numpy as np + +from libemg.environments.controllers import Controller +from libemg.environments._base import Environment + + +OUTSIDE_TARGET = pygame.USEREVENT + 1 +INSIDE_TARGET = pygame.USEREVENT + 2 + + +@dataclass(frozen=True) +class FittsConfig: + """Dataclass to customize parameters (e.g., target size and color) for a real-time Fitts' Law style task. + + Parameters + ---------- + num_trials : int + Number of trials user must complete. + dwell_time : float, optional + Time (in seconds) user must dwell in target to complete trial. Defaults to 1.0. + timeout : float | None, optional + Time limit (in seconds) that signifies a failed trial. If None, no timeout is used. Defaults to None. + velocity : float, optional + Velocity scalar that controls the max speed of the cursor. Defaults to 25. + save_file : str | None, optional + Name of save file (e.g., log.pkl). Supports .json and .pkl file formats. If None, no results are saved. Defaults to None. + width : int, optional + Width of display (in pixels). Defaults to 1250. + height : int, optional + Height of display (in pixels). Defaults to 750. + fps : int, optional + Frames per second (in Hz). Defaults to 60. + proportional_control : bool, optional + True if proportional control should be used, otherwise False. This value is ignored for Controllers that have proportional control built in, like regressors. Defaults to False. + target_radius : int, optional + Radius (in pixels) of each individual target. Defaults to 40. + cursor_radius : int, optional + Radius (in pixels) of cursor. Defaults to 14. + game_time : float, optional + Time (in seconds) that the task should run. If None, no time limit is set and the task ends when the number of targets are acquired. + If a value is passed, the task is stopped when either the time limit has been reached or the number of trials has been acquired. Defaults to None. + mapping : str, optional + Space to map predictions to. Setting this to 'cartesian' uses the standard Fitts' style input space, where predictions map to the x and y position of the cursor. + Setting this mapping to polar will instead map horizontal and vertical predictions to the radius and angle of a semi-circle, respectively (similar to spinning a wheel). + Pass in 'polar+' or 'polar-' to map up or down to counter-clockwise changes in angle, respectively. Defaults to 'cartesian'. + cursor_color : tuple, optional + Color of cursor. Pass in a tuple of the format (red, green, blue), where each color value is in the range 0-255. Defaults to orange (255, 95, 31). + cursor_in_target_color : tuple, optional + Color of cursor when in target. Pass in a tuple of the format (red, green, blue), where each color value is in the range 0-255. Defaults to blue (0, 102, 204). + target_color : tuple, optional + Color of targets. Pass in a tuple of the format (red, green, blue), where each color value is in the range 0-255. Defaults to white (255, 255, 255). + background_color : tuple, optional + Color of background. Pass in a tuple of the format (red, green, blue), where each color value is in the range 0-255. Defaults to black (0, 0, 0). + timer_color : tuple, optional + Color of dwell timer. Pass in a tuple of the format (red, green, blue), where each color value is in the range 0-255. Defaults to blue (0, 102, 204). + """ + num_trials: int + dwell_time: float = 1.0 + timeout: float | None = None + velocity: float = 25.0 + save_file: str | None = None + width: int = 1250 + height: int = 750 + fps: int = 60 + proportional_control: bool = True + target_radius: int = 40 + cursor_radius: int = 7 + game_time: float | None = None + mapping: str = 'cartesian' + cursor_color: tuple[int, int, int] = (255, 95, 31) + cursor_in_target_color: tuple[int, int, int] = (0, 102, 204) + target_color: tuple[int, int, int] = (255, 255, 255) + background_color: tuple[int, int, int] = (0, 0, 0) + timer_color: tuple[int, int, int] = (0, 102, 204) + + +class Fitts(Environment): + def __init__(self, controller: Controller, config: FittsConfig, prediction_map: dict | None = None): + """Fitts style task. Targets are generated at random and the user is asked to acquire targets as quickly as possible. + + Parameters + ---------- + controller : Controller + Interface to parse predictions which determine the direction of the cursor. + config : FittsConfig + Configuration class that determines environment parameters (e.g., target size). + prediction_map : dict | None, optional + Maps received control commands to cursor movement - only used if a non-continuous controller is used (e.g., classifier). If a continuous controller is used (e.g., regressor), + then 2 DoFs are expected when parsing predictions and this parameter is not used. If None, a standard map for classifiers is created where 0, 1, 2, 3, 4 are mapped to + down, up, no motion, right, and left, respectively. For custom mappings, pass in a dictionary where keys represent received control signals (from the Controller) and + values map to actions in the environment. Accepted actions are: 'S' (down), 'N' (up), 'NM' (no motion), 'E' (right), and 'W' (left). All of these actions must be + represented by a single key in the dictionary. Defaults to None. + """ + # logging information + log_dictionary = { + 'time_stamp': [], + 'trial_number': [], + 'goal_target' : [], + 'global_clock' : [], + 'cursor_position': [], + 'class_label': [], + 'current_direction': [] + } + default_prediction_map = { + 0: 'S', + 1: 'N', + 2: 'NM', + 3: 'E', + 4: 'W' + } + self.config = config + super().__init__(controller, fps=self.config.fps, log_dictionary=log_dictionary, save_file=self.config.save_file) + + if self.config.mapping == 'cartesian': + self.render_as_polar = False + elif self.config.mapping in ['polar+', 'polar-']: + self.render_as_polar = True + else: + raise ValueError(f"Unexpected value for mapping. Got: {self.config.mapping}.") + + if prediction_map is None: + prediction_map = default_prediction_map + assert set(np.unique(list(prediction_map.values()))) == set(list(default_prediction_map.values())), f"Did not find all commands {list(default_prediction_map.values())} represented as values in prediction_map. Got: {prediction_map}." + + self.prediction_map = prediction_map + self.current_direction = [0., 0.] + + def game_setup(self): + self.font = pygame.font.SysFont('helvetica', 40) + self.screen = pygame.display.set_mode([self.config.width, self.config.height]) + + # gameplay parameters + self.trial = -1 + + if '+' in self.config.mapping: + theta_bounds = (math.pi, 0) + else: + theta_bounds = (0, math.pi) + + self.theta_bounds = theta_bounds + self.polar_origin = (self.config.width // 2, self.config.height // 2) + self.polar_origin = (self.config.width // 2, self.config.height) + self.polygon_angles = np.linspace(0, 2 * math.pi, num=100) # could change how many points are calculated based on desired FPS (having 1000 caused frame rate issues) + + # interface objects + self.cursor = pygame.Rect(self.config.width // 2 - self.config.cursor_radius, self.config.height // 2 - self.config.cursor_radius, + self.config.cursor_radius * 2, self.config.cursor_radius * 2) + self._get_new_goal_target() + self.current_direction = [0,0] + + self.timeout_timer = None + self.trial_duration = 0 + self._info = ['predictions', 'timestamp'] + if self.config.proportional_control: + self._info.append('pc') + self.start_time = time.time() + self.dwell_timer = None + + def _draw(self): + self.screen.fill(self.config.background_color) + self._draw_targets() + self._draw_cursor() + self._draw_timer() + + def _draw_targets(self): + self._draw_circle(self.goal_target, self.config.target_color) + + def _draw_cursor(self): + color = self.config.cursor_in_target_color if self.dwell_timer is not None else self.config.cursor_color + self._draw_circle(self.cursor, color) + + def _draw_timer(self): + if self.dwell_timer is not None: + toc = time.perf_counter() + duration = round((toc-self.dwell_timer),2) + time_str = str(duration) + draw_text = self.font.render(time_str, 1, self.config.timer_color) + self.screen.blit(draw_text, (10, 10)) + + def _update_game(self): + self._draw() + self._run_game_process() + self._move() + + def _run_game_process(self): + self._check_collisions() + self._check_events() + + def _check_collisions(self): + if math.sqrt((self.goal_target.centerx - self.cursor.centerx)**2 + (self.goal_target.centery - self.cursor.centery)**2) < (self.goal_target[2]/2 + self.cursor[2]/2): + pygame.event.post(pygame.event.Event(INSIDE_TARGET)) + self.Event_Flag = True + else: + pygame.event.post(pygame.event.Event(OUTSIDE_TARGET)) + self.Event_Flag = False + + def _check_events(self): + # closing window + for event in pygame.event.get(): + if event.type == pygame.QUIT: + self.done = True + return + + if self.config.game_time is not None and (time.time() - self.start_time) >= self.config.game_time: + self.done = True + return + + data = self.controller.get_data(self._info) + + #self.current_direction = [0., 0.] + if data is not None: + # Move cursor + predictions = data[0] + timestamp = data[1] + if len(data) == 3: + pc = data[2] + else: + pc = [1. for _ in predictions] + + if len(predictions) == 1 and len(pc) == 1: + # Output is a class/action, not a set of DOFs + prediction = predictions[0] + pc = pc[0] + direction = self.prediction_map[prediction] + + if direction == 'N': + predictions = [0, 1] + elif direction == 'E': + predictions = [1, 0] + elif direction == 'S': + predictions = [0, -1] + elif direction == 'W': + predictions = [-1, 0] + elif direction == 'NM': + predictions = [0, 0] + else: + raise ValueError(f"Expected prediction map to have keys 'N', 'E', 'S', 'W', and 'NM', but found key: {direction}.") + + pc = [pc, pc] + + self.current_direction[0] = self.config.velocity * float(predictions[0]) * pc[0] + self.current_direction[1] = -1* self.config.velocity * float(predictions[1]) * pc[1] # -ve b/c pygame origin pixel is at top left of screen + + self._log(str(predictions), timestamp) + + if len(self.log_dictionary['time_stamp']) == 0: + # No data has been received, so don't start counting + print('Waiting for Fitts to receive data...') + return + + ## CHECKING FOR COLLISION BETWEEN CURSOR AND RECTANGLES + if event.type == OUTSIDE_TARGET: + if self.Event_Flag == False: + self.dwell_timer = None + self.duration = 0 + elif event.type == INSIDE_TARGET: + if self.dwell_timer is None: + self.dwell_timer = time.perf_counter() + else: + toc = time.perf_counter() + self.duration = round((toc - self.dwell_timer), 2) + if self.duration >= self.config.dwell_time: + self._get_new_goal_target() + + if self.timeout_timer is None: + self.timeout_timer = time.perf_counter() + else: + toc = time.perf_counter() + self.trial_duration = round((toc - self.timeout_timer), 2) + + if self.config.timeout is not None and self.trial_duration >= self.config.timeout: + # Timeout + self._get_new_goal_target() + + def _move(self): + self.cursor.left += self.current_direction[0] + self.cursor.top += self.current_direction[1] + + # Ensure cursor stays in the bounds of the screen + self.cursor.left = max(0, self.cursor.left) + self.cursor.left = min(self.config.width - self.cursor.width, self.cursor.left) + self.cursor.top = max(0, self.cursor.top) + self.cursor.top = min(self.config.height - self.cursor.height, self.cursor.top) + + def _get_new_goal_target(self): + self.dwell_timer = None + self.timeout_timer = None + self.trial_duration = 0 + + max_radius = int(min(self.config.width, self.config.height) * 0.5) # only create targets in a centered circle (based on size of screen) + while True: + target_radius = np.random.randint(self.cursor[2], self.config.target_radius) + target_position_radius = np.random.randint(0, max_radius - target_radius) + target_angle = np.random.uniform(0, 2 * math.pi) + # Convert to cartesian (relative to pygame origin, not center of screen) + x = self.config.width // 2 + target_position_radius * math.cos(target_angle) + y = self.config.height // 2 - target_position_radius * math.sin(target_angle) # subtract b/c y is inverted in pygame + # Continue until we create a target that isn't on the cursor + if math.dist((x, y), self.cursor.center) > (target_radius + self.cursor[2] // 2): + break + + left = x - target_radius + top = y - target_radius + self.goal_target = pygame.Rect(left, top, target_radius * 2, target_radius * 2) + + self.trial += 1 + if self.trial == self.config.num_trials: + self.done = True + + def _log(self, label, timestamp): + self.log_dictionary['time_stamp'].append(timestamp) + self.log_dictionary['trial_number'].append(self.trial) + self.log_dictionary['goal_target'].append((self.goal_target.centerx, self.goal_target.centery, self.goal_target[2])) + self.log_dictionary['global_clock'].append(time.perf_counter()) + self.log_dictionary['cursor_position'].append((self.cursor.centerx, self.cursor.centery, self.cursor[2])) + self.log_dictionary['class_label'].append(label) + self.log_dictionary['current_direction'].append(self.current_direction) + + def _map_to_polar_space(self, x, y): + radius = np.interp(x, (0, self.config.width), (0, min(self.config.width // 2, self.config.height))) # limit radius based on screen dimensions so circles stay on screen + theta = np.interp(y, (0, self.config.height), self.theta_bounds) + + # theta is the angle from the right side of the screen and goes counter-clockwise (same as unit circle) + polar_x = radius * np.cos(theta) + self.polar_origin[0] + polar_y = self.polar_origin[1] - radius * np.sin(theta) # subtract b/c y is inverted in pygame + + return polar_x, polar_y + + def _draw_circle(self, rect, color, fill = True, draw_radius = False): + # Keep the underlying circle (e.g., target or cursor) coordinates the same, but render as polar to keep downstream calculations the same + polygon_width = 0 if fill else 2 + target_radius = rect.width // 2 + + if not self.render_as_polar: + pygame.draw.circle(self.screen, color, rect.center, target_radius, width=polygon_width) + return + + points = [] + for circle_theta in self.polygon_angles: + # Create points to make a circle in Cartesian space + x = rect.centerx + target_radius * np.cos(circle_theta) + y = rect.centery + target_radius * np.sin(circle_theta) + + # Remap to polar equivalents + polar_x, polar_y = self._map_to_polar_space(x, y) + points.append((polar_x, polar_y)) + + pygame.draw.polygon(self.screen, color, points, width=polygon_width) + + if draw_radius: + # NOTE: This option is there, but I'm not sure that it should be used. You don't have runways in real life (or other Fitts tasks), so it might not be fair to add. + # If we are going to add it, we'd need to have them for the angle (not just the radius) + # Also the calculation of the radius should probably be a field because recalculating and rounding every time will cause instability when just changing the angle. + semi_circle_x, semi_circle_y = self._map_to_polar_space(rect.centerx, rect.centery) + semi_circle_radius = int(np.linalg.norm(np.array([semi_circle_x - self.polar_origin[0], semi_circle_y - self.polar_origin[1]]))) + pygame.draw.circle(self.screen, color, self.polar_origin, semi_circle_radius, width=2, draw_top_right=True, draw_top_left=True) + pygame.draw.line(self.screen, color, (self.polar_origin[0] - semi_circle_radius, self.polar_origin[1]), (self.polar_origin[0] + semi_circle_radius, self.polar_origin[1])) + + def _run_loop(self): + # updated frequently for graphics & gameplay + self._update_game() + pygame.display.set_caption(str(self.clock.get_fps())) + + +class ISOFitts(Fitts): + def __init__(self, controller: Controller, config: FittsConfig, prediction_map: dict | None = None, num_targets: int = 8, target_distance_radius: int = 275): + """ISO Fitts style task. Targets are generated in a circle and the user is asked to acquire targets as quickly as possible. + + Parameters + ---------- + controller : Controller + Interface to parse predictions which determine the direction of the cursor. + config : FittsConfig + Configuration class that determines environment parameters (e.g., target size). + prediction_map : dict | None, optional + Maps received control commands to cursor movement - only used if a non-continuous controller is used (e.g., classifier). If a continuous controller is used (e.g., regressor), + then 2 DoFs are expected when parsing predictions and this parameter is not used. If None, a standard map for classifiers is created where 0, 1, 2, 3, 4 are mapped to + down, up, no motion, right, and left, respectively. For custom mappings, pass in a dictionary where keys represent received control signals (from the Controller) and + values map to actions in the environment. Accepted actions are: 'S' (down), 'N' (up), 'NM' (no motion), 'E' (right), and 'W' (left). All of these actions must be + represented by a single key in the dictionary. Defaults to None. + num_targets : int, optional + Number of targets in task. Defaults to 8. + target_distance_radius : int, optional + Radius (in pixels) of target of targets in Iso Fitts' environment. Defaults to 275. + """ + width_is_too_small = target_distance_radius > config.width // 2 + height_is_too_small = target_distance_radius > config.height // 2 + if width_is_too_small and height_is_too_small: + error_info = f"width and height" + elif width_is_too_small: + error_info = f"width" + elif height_is_too_small: + error_info = f"height" + else: + error_info = None + + if error_info is not None: + raise ValueError(f"Radius between ISO Fitts targets is larger than screen size will allow. " + f"Target distance radius must be less than half the screen dimensions. " + f"Please increase screen {error_info} or reduce target distance radius.") + assert target_distance_radius < config.width // 2 and target_distance_radius < config.height // 2, f"Radius between ISO Fitts targets is larger than screen size will allow. Please increase screen size or reduce target distance radius." + self.goal_target_idx = -1 + self.num_of_targets = num_targets + self.big_rad = target_distance_radius + + # interface objects + self.targets = [] + angle = 0 + angle_increment = 360 // self.num_of_targets + while angle < 360: + self.targets.append(pygame.Rect( + (config.width // 2 - config.target_radius) + math.cos(math.radians(angle)) * self.big_rad, + (config.height // 2 - config.target_radius) + math.sin(math.radians(angle)) * self.big_rad, + config.target_radius * 2, config.target_radius * 2 + )) + angle += angle_increment + + super().__init__(controller, config, prediction_map=prediction_map) + + def _draw_targets(self): + for target in self.targets: + self._draw_circle(target, self.config.target_color, fill=False) # draw target outlines + + self._draw_circle(self.goal_target, self.config.target_color, fill=True) # fill in goal target + + def _get_new_goal_target(self): + super()._get_new_goal_target() + if self.goal_target_idx == -1: + self.goal_target_idx = 0 + self.next_target_in = self.num_of_targets//2 + self.target_jump = 0 + else: + self.goal_target_idx = (self.goal_target_idx + self.next_target_in )% self.num_of_targets + if self.target_jump == 0: + self.next_target_in = self.num_of_targets//2 + 1 + self.target_jump = 1 + else: + self.next_target_in = self.num_of_targets // 2 + self.target_jump = 0 + self.goal_target = self.targets[self.goal_target_idx] diff --git a/libemg/feature_extractor.py b/libemg/feature_extractor.py index 1e513d8e..2ebf6b7a 100644 --- a/libemg/feature_extractor.py +++ b/libemg/feature_extractor.py @@ -8,6 +8,7 @@ from scipy.stats import skew, kurtosis from librosa import lpc from pywt import wavedec, upcoef +from sklearn.preprocessing import StandardScaler class FeatureExtractor: """ @@ -133,7 +134,7 @@ def extract_feature_group(self, feature_group, windows, feature_dic={}, array=Fa return self._format_data(feats) return feats - def extract_features(self, feature_list, windows, feature_dic={}, array=False): + def extract_features(self, feature_list, windows, feature_dic={}, array=False, normalize=False, normalizer=None, fix_feature_errors=False): """Extracts a list of features. Parameters @@ -147,22 +148,44 @@ def extract_features(self, feature_list, windows, feature_dic={}, array=False): A dictionary containing the parameters you'd like passed to each feature. ex. {"MDF_sf":1000} array: bool (optional), default=False If True, the dictionary will get converted to a list. + normalize: bool (optional), default=False + If True, the features will be normalized between using sklearn StandardScaler. The returned object will be a list. + normalizer: StandardScaler, default=None + This should be set to the output from feature extraction on the training data. Do not normalize testing features without this as this could be considered information leakage. + fix_feature_errors: bool (optional), default=False + If true, fixes all feature errors (NaN=0, INF=0, -INF=0). Returns ---------- dictionary or list A dictionary where each key is a specific feature and its value is a list of the computed features for each window. + StandardScaler + If normalize is true it will return the normalizer object. This should be passed into the feature extractor for test data. """ features = {} + scaler = None for feature in feature_list: if feature in self.get_feature_list(): method_to_call = getattr(self, 'get' + feature + 'feat') valid_keys = [i for i in list(feature_dic.keys()) if feature+"_" in i] smaller_dictionary = dict((k, feature_dic[k]) for k in valid_keys if k in feature_dic) - features[feature] = method_to_call(windows, **smaller_dictionary) + feats = method_to_call(windows, **smaller_dictionary) + if fix_feature_errors: + if self.check_features(feats, False): + feats = np.nan_to_num(feats, neginf=0, nan=0, posinf=0) + features[feature] = feats if array: - return self._format_data(features) - return features + features = self._format_data(features) + if normalize: + if isinstance(features, dict): + features = self._format_data(features) + if not normalizer: + scaler = StandardScaler() + features = scaler.fit_transform(features) + else: + features = normalizer.transform(features) + return features, scaler + return features def check_features(self, features, silent=False): """Assesses a features object for np.nan, np.inf, and -np.inf. Can be used to check for clean data. @@ -1397,15 +1420,18 @@ def getWENGfeat(self, windows, WENG_fs = 1000): list The computed features associated with each window. Size: Wx((order+1)*Nchannels) """ - # get the highest power of 2 the nyquist rate is divisible by - order = math.floor(np.log(WENG_fs/2)/np.log(2) - 1) - # Khushaba et al suggests using sym8 - # note, this will often throw a WARNING saying the user specified order is too high -- but this is what the - # original paper suggests using as the order. - wavelets = wavedec(windows, wavelet='sym8', level=order,axis=2) - # for every order, compute the energy (sum of DWT) - total of the squared signal - features = np.hstack([np.log(np.sum(i**2, axis=2)+1e-10) for i in wavelets]) - return features + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + # get the highest power of 2 the nyquist rate is divisible by + order = math.floor(np.log(WENG_fs/2)/np.log(2) - 1) + # Khushaba et al suggests using sym8 + # note, this will often throw a WARNING saying the user specified order is too high -- but this is what the + # original paper suggests using as the order. + wavelets = wavedec(windows, wavelet='sym8', level=order,axis=2) + # for every order, compute the energy (sum of DWT) - total of the squared signal + features = np.hstack([np.log(np.sum(i**2, axis=2)+1e-10) for i in wavelets]) + return features def getWVfeat(self, windows, WV_fs=1000): @@ -1800,6 +1826,9 @@ def __project_data(self, projection, projection_engine, feature_matrix, classes, return train_data, test_data def _format_data(self, feature_dictionary): + if not isinstance(feature_dictionary, dict): + return feature_dictionary + arr = None for feat in feature_dictionary: if arr is None: diff --git a/libemg/gui.py b/libemg/gui.py index 7f43ae72..d445936f 100644 --- a/libemg/gui.py +++ b/libemg/gui.py @@ -22,7 +22,10 @@ class GUI: Online data handler used for acquiring raw EMG data. args: dic, default={'media_folder': 'images/', 'data_folder':'data/', 'num_reps': 3, 'rep_time': 5, 'rest_time': 3, 'auto_advance': True} The dictionary that defines the SGT window. Keys are: 'media_folder', - 'data_folder', 'num_reps', 'rep_time', 'rest_time', and 'auto_advance'. + 'data_folder', 'num_reps', 'rep_time', 'rest_time', and 'auto_advance'. All media (i.e., images and videos) in 'media_folder' will be played in alphabetical order. + For video files, a matching labels file of the same name will be searched for and added to the 'data_folder' if found. + 'rep_time' is only used for images since the duration of videos is automatically calculated based on + the number of frames (assumed to be 24 FPS). width: int, default=1920 The width of the SGT window. height: int, default=1080 @@ -112,7 +115,7 @@ def _file_menu_init(self): def _data_collection_callback(self): panel_arguments = list(inspect.signature(DataCollectionPanel.__init__).parameters) passed_arguments = {i: self.args[i] for i in self.args.keys() if i in panel_arguments} - self.dcp = DataCollectionPanel(**passed_arguments, gui=self, video_player_width=self.video_player_width, video_player_height=self.video_player_height) + self.dcp = DataCollectionPanel(self.online_data_handler, **passed_arguments, video_player_width=self.video_player_width, video_player_height=self.video_player_height) self.dcp.spawn_configuration_window() def _import_data_callback(self): diff --git a/libemg/offline_metrics.py b/libemg/offline_metrics.py index e7675136..fda605bd 100644 --- a/libemg/offline_metrics.py +++ b/libemg/offline_metrics.py @@ -3,8 +3,17 @@ import matplotlib.pyplot as plt class OfflineMetrics: - """Offline Metrics class is used for extracting offline performance metrics. """ + Offline Metrics class is used for extracting offline performance metrics. + """ + + def _ignore_rejected(self, y_predictions, y_true): + # ignore rejections + valid_samples = y_predictions != -1 + y_predictions = y_predictions[valid_samples] + y_true = y_true[valid_samples] + return y_predictions, y_true + def get_common_metrics(self): """Gets a list of the common metrics used for assessing EMG performance. @@ -50,9 +59,9 @@ def extract_common_metrics(self, y_true, y_predictions, null_label=None): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of the true labels associated with each prediction. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted outputs from a classifier. null_label: int (optional) A null label used for the AER metric - this should correspond to the label associated @@ -74,9 +83,9 @@ def extract_offline_metrics(self, metrics, y_true, y_predictions, null_label=Non metrics: list A list of the metrics to extract. A list of metrics can be found running the get_available_metrics function. - y_true: list + y_true: numpy.ndarray A list of the true labels associated with each prediction. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted outputs from a classifier. null_label: int (optional) A null label used for the AER metric - this should correspond to the label associated @@ -126,9 +135,9 @@ def get_CA(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -136,10 +145,7 @@ def get_CA(self, y_true, y_predictions): float Returns the classification accuracy. """ - # ignore rejections - valid_samples = y_predictions != -1 - y_predictions = y_predictions[valid_samples] - y_true = y_true[valid_samples] + y_predictions, y_true = self._ignore_rejected(y_predictions, y_true) if len(y_true) == 0: print("No test samples - check the rejection rate.") return 1.0 @@ -148,13 +154,13 @@ def get_CA(self, y_true, y_predictions): def get_AER(self, y_true, y_predictions, null_class): """Active Error. - Classification accuracy on active classes (i.e., all classes but no movement/rest). + Classification accuracy on active classes (i.e., all classes but no movement/rest). Rejected samples are ignored. Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. null_class: int The null class that shouldn't be considered. @@ -164,6 +170,7 @@ def get_AER(self, y_true, y_predictions, null_class): float Returns the active error. """ + y_predictions, y_true = self._ignore_rejected(y_predictions, y_true) nm_predictions = [i for i, x in enumerate(y_predictions) if x == null_class] return 1 - self.get_CA(np.delete(y_true, nm_predictions), np.delete(y_predictions, nm_predictions)) @@ -174,9 +181,9 @@ def get_INS(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -196,7 +203,7 @@ def get_REJ_RATE(self, y_predictions): Parameters ---------- - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. -1 in the list correspond to rejected predictions. Returns @@ -214,9 +221,9 @@ def get_CONF_MAT(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -240,9 +247,9 @@ def get_RECALL(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -250,6 +257,7 @@ def get_RECALL(self, y_true, y_predictions): list Returns a list consisting of the recall for each class. """ + y_predictions, y_true = self._ignore_rejected(y_predictions, y_true) recall, weights = self._get_RECALL_helper(y_true, y_predictions) return np.average(recall, weights=weights) @@ -273,9 +281,9 @@ def get_PREC(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -283,6 +291,7 @@ def get_PREC(self, y_true, y_predictions): list Returns a list consisting of the precision for each class. """ + y_predictions, y_true = self._ignore_rejected(y_predictions, y_true) precision, weights = self._get_PREC_helper(y_true, y_predictions) return np.average(precision, weights=weights) @@ -307,9 +316,9 @@ def get_F1(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -317,6 +326,7 @@ def get_F1(self, y_true, y_predictions): list Returns a list consisting of the f1 score for each class. """ + y_predictions, y_true = self._ignore_rejected(y_predictions, y_true) prec, weights = self._get_PREC_helper(y_true, y_predictions) recall, _ = self._get_RECALL_helper(y_true, y_predictions) f1 = 2 * (prec * recall) / (prec + recall) @@ -329,9 +339,9 @@ def get_R2(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -351,9 +361,9 @@ def get_MSE(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -372,9 +382,9 @@ def get_MAPE(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -393,9 +403,9 @@ def get_RMSE(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -415,9 +425,9 @@ def get_NRMSE(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns @@ -437,9 +447,9 @@ def get_MAE(self, y_true, y_predictions): Parameters ---------- - y_true: list + y_true: numpy.ndarray A list of ground truth labels. - y_predictions: list + y_predictions: numpy.ndarray A list of predicted labels. Returns diff --git a/libemg/output_writer.py b/libemg/output_writer.py new file mode 100644 index 00000000..93ee93a0 --- /dev/null +++ b/libemg/output_writer.py @@ -0,0 +1,148 @@ +from abc import ABC, abstractmethod +import socket +from libemg.shared_memory_manager import SharedMemoryManager +import numpy as np +import types + +class OutputWriter(ABC): + @abstractmethod + def write(self, info: dict) -> None: + """ + Write the output information. + + Parameters + ---------- + info : dict + A dictionary containing output information such as timestamp, + prediction, probability, velocity, etc. + """ + pass +class ConsoleOutputWriter(OutputWriter): + def __init__(self, tag): + self.tag = tag + + def write(self, info: dict) -> None: + print(str(info['timestamp'])," ", info[self.tag]) + +class FileOutputWriter(OutputWriter): + def __init__(self, tag, file_path: str, file_name: str): + self.file_path = file_path + self.file_name = file_name + self.handle = open(self.file_path + self.file_name, "a", newline="") + + def write(self, info: dict) -> None: + # Format the info as a line. + line = f"{info.get('timestamp', '')} {info.get('prediction', '')} {info.get('probability', '')} {info.get('velocity', '')}\n" + self.handle.write(line) + self.handle.flush() + +class SocketOutputWriter(OutputWriter): + def __init__(self, tag, ip: str = '127.0.0.1', port: int = 12346, protocol: str = "UDP"): + self.ip = ip + self.port = port + self.protocol = protocol.upper() + self.sock = None + self.tag = tag + self._create_socket() + + def _create_socket(self): + if self.protocol == "UDP": + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + elif self.protocol == "TCP": + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((self.ip, self.port)) + else: + raise ValueError("Protocol must be UDP or TCP.") + + def write(self, info: dict) -> None: + message = str(info[self.tag]) + " " + str(info['timestamp']) + if self.sock is None: + self._create_socket() + if self.protocol == "UDP": + self.sock.sendto(message.encode('utf-8'), (self.ip, self.port)) + else: + self.sock.sendall(message.encode('utf-8')) + + def __getstate__(self): + # Remove the socket from the state so it's not pickled. + state = self.__dict__.copy() + if "sock" in state: + state['sock'].close() + del state["sock"] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + # Reinitialize the socket in the child process. + self.sock = None + self._create_socket() + +class SharedMemoryOutputWriter(OutputWriter): + def __init__(self, tag: str, shape, dtype, lock, mod_fn=None, mod_fn_count=None): + """ + Parameters: + tag (str): + The shared memory variable tag. + shape (tuple): + The shape of the shared memory variable. + dtype: + The data type of the shared memory variable. + lock (Lock): + A multiprocessing lock for synchronization. + mod_fn (callable, optional): + A function that takes (current_data, message) and returns new data. + If not provided, defaults to a function that simply returns the message. + """ + self.tag = tag + self.shape = shape + self.dtype = dtype + self.lock = lock + self.mod_fn = types.MethodType(mod_fn, self) if mod_fn is not None else self.default_mod_fn + self.mod_fn_count = types.MethodType(mod_fn_count, self) if mod_fn_count is not None else self.default_mod_fn_count + # Create a new shared memory manager and create the variable. + self.smm = SharedMemoryManager() + self.smm.create_variable(tag, shape, dtype, lock) + # Share the buffer's lock so (data, count) can be snapshotted atomically. + self.smm.create_variable(tag+"_count", (1,1), np.int32, lock) + + def write(self, info: dict) -> None: + if self.smm is None: + raise RuntimeError("SharedMemoryOutputWriter not attached to a manager.") + # Use the provided mod_fn to modify the shared memory variable. + self.smm.modify_variable(self.tag, lambda data: self.mod_fn(data, info)) + self.smm.modify_variable(self.tag + "_count", lambda data: self.mod_fn_count(data, info)) + + def reset(self) -> None: + if self.smm is None: + raise RuntimeError("SharedMemoryOutputWriter not attached to a manager.") + self.smm.modify_variable(self.tag, lambda data: np.zeros(self.shape, dtype=self.dtype)) + self.smm.modify_variable(self.tag + "_count", lambda data: 0) + + def default_mod_fn(self, data, info): + input_size = self.smm.variables[self.tag]["shape"][0] + data[:] = np.vstack((info[self.tag], data))[:input_size, :] + return data + + def default_mod_fn_count(self, data, info): + data[:] = data[:] + info[self.tag].shape[0] + return data + + def __getstate__(self): + self._smm_item = self.smm.get_shared_memory_items() + state = self.__dict__.copy() + # Remove the non-serializable shared memory manager. + if "smm_manager" in state: + del state["smm_manager"] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + # Reconstruct the shared memory manager using the stored _smm_item. + if self._smm_item is not None: + new_mgr = SharedMemoryManager() + tag, shape, dtype, lock = self._smm_item + new_mgr.create_variable(tag, shape, dtype, lock) + self.smm_manager = new_mgr + else: + self.smm_manager = None + diff --git a/libemg/shared_memory_manager.py b/libemg/shared_memory_manager.py index 75b0cd6a..f51edf1b 100644 --- a/libemg/shared_memory_manager.py +++ b/libemg/shared_memory_manager.py @@ -1,6 +1,38 @@ import numpy as np +from multiprocessing import Lock from multiprocessing.shared_memory import SharedMemory + +def assign_shared_memory_locks(shared_memory_items): + """Append a synchronization lock to each shared-memory item in place. + + A modality buffer ```` and its sample counter ``_count`` are + given the *same* lock object so that the (data, count) pair can be read as + a single atomic snapshot (see :meth:`SharedMemoryManager.get_variables`). + Without this, a reader can copy the buffer and its count in two separate + critical sections and observe a count that runs ahead of the data it just + copied, which splices dropped/duplicated samples into logged signals. + + Parameters + ---------- + shared_memory_items : list + A list of ``[tag, shape, dtype]`` items. A lock is appended to each, + shared between a tag and its matching ``_count`` entry. + + Returns + ------- + list + The same list, with a lock appended to every item. + """ + locks = {} + for item in shared_memory_items: + tag = item[0] + base = tag[:-len("_count")] if tag.endswith("_count") else tag + lock = locks.setdefault(base, Lock()) + item.append(lock) + return shared_memory_items + + class SharedMemoryManager: def __init__(self): self.variables = {} @@ -9,50 +41,106 @@ def create_variable(self, tag, shape, type, lock): if tag in self.variables.keys(): print(f"Already have access to this variable: {tag}") return True + + # if tag exists already + if self.find_variable(tag, shape, type, lock): + print(f'{tag} already exists in shared memory, found variable.') + return True + try: sm = SharedMemory(tag, create=False) sm.unlink() except: pass - smh = SharedMemory(tag, create=True, size=int(type().itemsize * np.prod(shape))) + try: + type_size = type().itemsize + except TypeError: + # Passed in non-callable dtype + type_size = type.itemsize + + smh = SharedMemory(tag, create=True, size=int(type_size * np.prod(shape))) data = np.ndarray((shape),dtype=type,buffer=smh.buf) data.fill(0) - self.variables[tag] = {} - self.variables[tag]["data"] = data - self.variables[tag]["shape"] = shape - self.variables[tag]["type"] = type - self.variables[tag]["smh"] = smh - self.variables[tag]["lock"] = lock + self.variables[tag] = { + "data" : data, + "shape": shape, + "type" : type, + "smh" : smh, + "lock" : lock + } return True def find_variable(self, tag, shape, type, lock): try: - smh = SharedMemory(tag, size=int(type().itemsize * np.prod(shape))) + type_size = type().itemsize + except TypeError: + # Passed in non-callable dtype + type_size = type.itemsize + try: + smh = SharedMemory(tag, size=int(type_size * np.prod(shape))) # create a new numpy array that uses the shared memory data = np.ndarray((shape), dtype=type, buffer=smh.buf) - self.variables[tag] = {} - self.variables[tag]["data"] = data - self.variables[tag]["shape"] = shape - self.variables[tag]["type"] = type - self.variables[tag]["smh"] = smh - self.variables[tag]["lock"] = lock + self.variables[tag] = { + "data" : data, + "shape": shape, + "type" : type, + "smh" : smh, + "lock" : lock + } return True except FileNotFoundError: return False - def get_variable(self, tag): assert tag in self.variables.keys() with self.variables[tag]["lock"]: return self.variables[tag]["data"].copy() + def get_variables(self, tags): + """Atomically copy several variables as one consistent snapshot. + + Every distinct lock guarding the requested tags is held for the whole + copy, so the returned values reflect the same instant in time. When the + tags share a single lock (the convention established by + :func:`assign_shared_memory_locks` for a ````/``_count`` + pair), that lock is acquired exactly once. + + Parameters + ---------- + tags : list + The shared-memory tags to copy together. + + Returns + ------- + dict + A mapping from each requested tag to a copy of its current data. + """ + for tag in tags: + assert tag in self.variables.keys() + # Collapse duplicate lock objects, then acquire each distinct lock once + # in a deterministic (id-sorted) order to avoid deadlock if a caller + # ever groups tags that don't share a lock. + distinct_locks = {} + for tag in tags: + lock = self.variables[tag]["lock"] + distinct_locks[id(lock)] = lock + ordered_locks = [distinct_locks[key] for key in sorted(distinct_locks.keys())] + acquired = [] + try: + for lock in ordered_locks: + lock.acquire() + acquired.append(lock) + return {tag: self.variables[tag]["data"].copy() for tag in tags} + finally: + for lock in reversed(acquired): + lock.release() + def modify_variable(self, tag, fn): assert tag in self.variables.keys() with self.variables[tag]["lock"]: self.variables[tag]["data"][:] = fn(self.variables[tag]["data"]) - def cleanup(self, parent = True): for k in self.variables.keys(): self.variables[k]["smh"].close() @@ -61,7 +149,7 @@ def cleanup(self, parent = True): self.variables = {} def get_variable_list(self): - result = [] - for k in self.variables.keys(): - result.append([k, self.variables[k]["shape"], self.variables[k]["type"]]) - return result \ No newline at end of file + return [[k, self.variables[k]["shape"], self.variables[k]["type"]] for k in self.variables.keys()] + + def get_shared_memory_items(self): + return [[i, self.variables[i]["shape"], self.variables[i]["type"], self.variables[i]["lock"]] for i in self.variables] \ No newline at end of file diff --git a/libemg/streamers.py b/libemg/streamers.py index 2543d065..b91ca509 100644 --- a/libemg/streamers.py +++ b/libemg/streamers.py @@ -3,76 +3,79 @@ import pickle import platform import numpy as np -from multiprocessing import Process, Event, Lock +import sifi_bridge_py + +from multiprocessing import Process, Event +from libemg.shared_memory_manager import assign_shared_memory_locks from libemg._streamers._myo_streamer import MyoStreamer from libemg._streamers._delsys_streamer import DelsysEMGStreamer from libemg._streamers._delsys_API_streamer import DelsysAPIStreamer -if platform.system() != 'Linux': - from libemg._streamers._oymotion_windows_streamer import Gforce -else: - from libemg._streamers._oymotion_streamer import OyMotionStreamer +from libemg._streamers._oymotion_streamer import Gforce from libemg._streamers._emager_streamer import EmagerStreamer from libemg._streamers._sifi_bridge_streamer import SiFiBridgeStreamer from libemg._streamers._leap_streamer import LeapStreamer -def sifibridge_streamer(version="1_1", - shared_memory_items = None, - ecg=False, - emg=True, - eda=False, - imu=False, - ppg=False, - notch_on=True, notch_freq=60, - emg_fir_on = True, - emg_fir=[20,450], - eda_cfg = True, - fc_lp = 0, # low pass eda - fc_hp = 5, # high pass eda - freq = 250,# eda sampling frequency - streaming=False, - mac= None): - """The streamer for the sifi armband. - This function connects to the sifi bridge and streams its data to the SharedMemory. This is used - for the SiFi biopoint and bioarmband. - Note that the IMU is acc_x, acc_y, acc_z, quat_w, quat_x, quat_y, quat_z. +def sifi_biopoint_streamer( + name = "BioPoint_v1_3", + shared_memory_items = None, + ecg = False, + emg = True, + eda = False, + imu = False, + ppg = False, + filtering = True, + emg_notch_freq = 60, + emg_bandpass = (20,450), + eda_bandpass = (0,5), + eda_freq = 0, + streaming=False, + mac= None +): + """ + The streamer for the SiFi BioPoint. + + This function connects to SiFi Bridge and streams its data to the SharedMemory. + + **Note**: The IMU keys are: + + - Acceleration: ax, ay, az + - Quaternions: qw, qx, qy, qz + Parameters ---------- - version: string (option), default = '1_1' - The version for the sifi streamer. + + device: string, default = BioPoint_v1_3 + The name or MAC of the device. shared_memory_items, default = [] The key, size, datatype, and multiprocessing Lock for all data to be shared between processes. ecg, default = False - The flag to enable electrocardiography recording from the main sensor unit. + Enable electrocardiography recording from the main sensor unit. emg, default = True - The flag to enable electromyography recording. + Enable electromyography recording. eda, default = False - The flag to enable electrodermal recording. + Enable electrodermal recording. imu, default = False - The flag to enable inertial measurement unit recording + Enable inertial measurement unit recording ppg, default = False The flag to enable photoplethysmography recording - notch_on, default = True - The flag to enable a fc Hz notch filter on device (firmware). - notch_freq, default = 60 - The cutoff frequency of the notch filter specified by notch_on. - emg_fir_on, default = True - The flag to enable a bandpass filter on device (firmware). - emg_fir, default = [20, 450] - The low and high cutoff frequency of the bandpass filter specified by emg_fir_on. - eda_cfg, default = True - The flag to specify if using high or low frequency current for EDA or bioimpedance. - fc_lp, default = 0 - The low cutoff frequency for the bioimpedance. - fc_hp, default = 5 - The high cutoff frequency for the bioimpedance. - freq, default = 250 - The sampling frequency for bioimpedance. + filtering, default = True + Enable on-device filtering, including bandpass filters and notch filters. + emg_notch_freq, default = 60 + EMG notch filter frequency, useful for eliminating Mains power interference. Can be {None, 50, 60} Hz. + emg_bandpass, default = (20, 450) + The low and high cutoff frequency of the EMG bandpass filter. + eda_bandpass, default = (0, 5) + The low and high cutoff frequency of the EDA bandpass filter. + eda_freq, default = 0 + The excitation signal frequency for EDA/BIOZ. Setting an AC value may inject a lot of noise into the EMG sensor. streaming, default = False - Whether to package the modalities together within packets for lower latency. + Whether to package the modalities together within packets for lower latency, only supported for BioPoint v1.3 and up. mac, default = None: - mac address of the device to be connected to + Optional MAC address the device to connect to, useful when multiple devices are in the vicinity and you want to connect to a specific one. + Returns ---------- + Object: streamer The sifi streamer process object. Object: shared memory @@ -80,6 +83,119 @@ def sifibridge_streamer(version="1_1", Examples --------- + + >>> streamer, shared_memory = sifibridge_streamer() + """ + + if shared_memory_items is None: + shared_memory_items = [] + if emg: + shared_memory_items.append(["emg", (4000,1), np.double]) + shared_memory_items.append(["emg_count", (1,1), np.int32]) + if imu: + shared_memory_items.append(["imu", (200,7), np.double]) + shared_memory_items.append(["imu_count", (1,1), np.int32]) + if ecg: + shared_memory_items.append(["ecg", (1000,1), np.double]) + shared_memory_items.append(["ecg_count", (1,1), np.int32]) + if eda: + shared_memory_items.append(["eda", (200,1), np.double]) + shared_memory_items.append(["eda_count", (1,1), np.int32]) + if ppg: + shared_memory_items.append(["ppg", (200,4), np.double]) + shared_memory_items.append(["ppg_count", (1,1), np.int32]) + + assign_shared_memory_locks(shared_memory_items) + + sb = SiFiBridgeStreamer( + name, + shared_memory_items, + ecg, + emg, + eda, + imu, + ppg, + filtering, + emg_notch_freq, + emg_bandpass, + eda_bandpass, + eda_freq, + streaming, + mac + ) + sb.start() + return sb, shared_memory_items + + +def sifi_bioarmband_streamer( + name = None, + shared_memory_items = None, + ecg = False, + emg = True, + eda = False, + imu = False, + ppg = False, + filtering = True, + emg_notch_freq = 60, + emg_bandpass = (20,450), + eda_bandpass = (0,5), + eda_freq = 0, + streaming = False, + mac = None +): + """ + The streamer for the SiFi BioArmband. + + This function connects to SiFi Bridge and streams its data to the SharedMemory. + + **Note**: The IMU keys are: + + - Acceleration: ax, ay, az + - Quaternions: qw, qx, qy, qz + + Parameters + ---------- + + name: string, default = BioArmband + The name of the Sifi Device. For example: BioArmband, BioPoint_v1_3, etc. + shared_memory_items, default = [] + The key, size, datatype, and multiprocessing Lock for all data to be shared between processes. + ecg, default = False + Enable electrocardiography recording from the main sensor unit. + emg, default = True + Enable electromyography recording. + eda, default = False + Enable electrodermal recording. + imu, default = False + Enable inertial measurement unit recording + ppg, default = False + The flag to enable photoplethysmography recording + filtering, default = True + Enable on-device filtering, including bandpass filters and notch filters. + emg_notch_freq, default = 60 + EMG notch filter frequency, useful for eliminating Mains power interference. Can be {None, 50, 60} Hz. + emg_bandpass, default = (20, 450) + The low and high cutoff frequency of the EMG bandpass filter. + eda_bandpass, default = (0, 5) + The low and high cutoff frequency of the EDA bandpass filter. + eda_freq, default = 0 + The excitation signal frequency for EDA/BIOZ. Setting an AC value may inject a lot of noise into the EMG sensor. + streaming, default = False + Whether to package the modalities together within packets for lower latency, only supported for BioPoint v1.3 and up. + mac, default = None: + Optional MAC address the device to connect to, useful when multiple devices are in the vicinity and you want to connect to a specific one. + + Returns + ---------- + + Object: streamer + The sifi streamer process object. + Object: shared memory + The shared memory items list to be passed to the OnlineDataHandler. + + Examples + --------- + >>> streamer, shared_memory = sifibridge_streamer() """ @@ -89,37 +205,38 @@ def sifibridge_streamer(version="1_1", shared_memory_items.append(["emg", (3000,8), np.double]) shared_memory_items.append(["emg_count", (1,1), np.int32]) if imu: - shared_memory_items.append(["imu", (100,10), np.double]) + shared_memory_items.append(["imu", (200,7), np.double]) shared_memory_items.append(["imu_count", (1,1), np.int32]) if ecg: - shared_memory_items.append(["ecg", (100,10), np.double]) + shared_memory_items.append(["ecg", (1000,1), np.double]) shared_memory_items.append(["ecg_count", (1,1), np.int32]) if eda: - shared_memory_items.append(["eda", (100,10), np.double]) + shared_memory_items.append(["eda", (200,1), np.double]) shared_memory_items.append(["eda_count", (1,1), np.int32]) if ppg: - shared_memory_items.append(["ppg", (100,10), np.double]) + shared_memory_items.append(["ppg", (200,4), np.double]) shared_memory_items.append(["ppg_count", (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) - sb = SiFiBridgeStreamer(version=version, - shared_memory_items=shared_memory_items, - notch_on=notch_on, - ecg=ecg, - emg=emg, - eda=eda, - imu=imu, - ppg=ppg, - notch_freq=notch_freq, - emgfir_on=emg_fir_on, - emg_fir = emg_fir, - eda_cfg = eda_cfg, - fc_lp = fc_lp, # low pass eda - fc_hp = fc_hp, # high pass eda - freq = freq,# eda sampling frequency - streaming=streaming, - mac = mac) + assign_shared_memory_locks(shared_memory_items) + + + sb = SiFiBridgeStreamer( + name, + shared_memory_items, + ecg, + emg, + eda, + imu, + ppg, + filtering, + emg_notch_freq, + emg_bandpass, + eda_bandpass, + eda_freq, + streaming, + mac + ) + sb.start() return sb, shared_memory_items @@ -163,8 +280,7 @@ def myo_streamer( shared_memory_items.append(["imu", (250,10), np.double]) shared_memory_items.append(["imu_count", (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) myo = MyoStreamer(filtered, emg, imu, shared_memory_items) myo.start() return myo, shared_memory_items @@ -203,7 +319,7 @@ def delsys_streamer(shared_memory_items : list | None = None, Returns ---------- Object: streamer - The sifi streamer object. + The delsys streamer object. Object: shared memory The shared memory object. Examples @@ -218,8 +334,7 @@ def delsys_streamer(shared_memory_items : list | None = None, if imu: shared_memory_items.append(["imu", (500,6), np.double]) shared_memory_items.append(["imu_count", (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) delsys = DelsysEMGStreamer(shared_memory_items=shared_memory_items, emg=emg, @@ -234,9 +349,9 @@ def delsys_streamer(shared_memory_items : list | None = None, return delsys, shared_memory_items -def delsys_api_streamer(license : str = None, - key : str = None, - num_channels : int = None, +def delsys_api_streamer(license : str, + key : str, + num_channels : int | None = None, dll_folder : str = 'resources/', shared_memory_items : list | None = None, emg : bool = True): @@ -252,7 +367,7 @@ def delsys_api_streamer(license : str = None, key : str Delsys key num_channels: int - The number of delsys sensors you are using. + The number of delsys sensors you are using. Not used if shared_memory_items is passed, otherwise is a required parameter. dll_folder: string : optional (default='resources/') The location of the DLL files installed from the Delsys Github. shared_memory_items : list (optional) @@ -263,22 +378,20 @@ def delsys_api_streamer(license : str = None, Returns ---------- Object: streamer - The sifi streamer object. + The delsys streamer object. Object: shared memory The shared memory object. Examples --------- - >>> streamer, shared_memory = delsys_streamer() + >>> streamer, shared_memory = delsys_api_streamer(LICENSE, KEY, num_channels=4) """ - assert license is not None - assert key is not None if shared_memory_items is None: + assert num_channels is not None, f"No shared memory items were passed, so num_channels must be set. Please set num_channels to the number of Delsys sensors. Got: {num_channels}." shared_memory_items = [] if emg: shared_memory_items.append(["emg", (5300,num_channels), np.double]) shared_memory_items.append(["emg_count", (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) delsys = DelsysAPIStreamer(key, license, dll_folder, shared_memory_items=shared_memory_items, emg=emg) delsys.start() @@ -308,7 +421,7 @@ def oymotion_streamer(shared_memory_items : list | None = None, Returns ---------- Object: streamer - The sifi streamer object + The oymotion streamer object Object: shared memory The shared memory object Examples @@ -331,20 +444,11 @@ def oymotion_streamer(shared_memory_items : list | None = None, if imu: shared_memory_items.append(["imu", (100,10), np.double]) shared_memory_items.append(["imu_count", (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) - operating_system = platform.system().lower() - - # I'm only addressing this atm. - if operating_system == "windows" or operating_system == 'mac': - oym = Gforce(sampling_rate, res, emg, imu, shared_memory_items) - oym.start() - else: - # This has not been updated to the new memory manager methods. - # oym = OyMotionStreamer(ip, port, sampRate=sampling, resolution=res) - # oym.start_stream() - raise Exception("Oymotion Streamer is not implemented for Linux.") + oym = Gforce(sampling_rate, res, emg, imu, shared_memory_items) + oym.start() + return oym, shared_memory_items @@ -362,7 +466,7 @@ def emager_streamer(shared_memory_items = None): Returns ---------- Object: streamer - The sifi streamer object. + The emager streamer object. Object: shared memory The shared memory object. Examples @@ -375,8 +479,7 @@ def emager_streamer(shared_memory_items = None): shared_memory_items.append(['emg', (2000, 64), np.double]) # buffer size doesn't have a huge effect - pretty much as long as it's bigger than window size shared_memory_items.append(['emg_count', (1, 1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) ema = EmagerStreamer(shared_memory_items) ema.start() return ema, shared_memory_items @@ -523,8 +626,7 @@ def leap_streamer(shared_memory_items : list | None =None, shared_memory_items.append(['finger_width', (230,3), np.double]) shared_memory_items.append(['finger_width_count', (1,1), np.int32]) - for item in shared_memory_items: - item.append(Lock()) + assign_shared_memory_locks(shared_memory_items) ls = LeapStreamer(shared_memory_items) ls.start() diff --git a/libemg/utils.py b/libemg/utils.py index 765beb29..8422dd5a 100644 --- a/libemg/utils.py +++ b/libemg/utils.py @@ -62,7 +62,7 @@ def _get_fn_windows(data, window_size, window_increment, fn): fn_of_windows = np.apply_along_axis(lambda x: fn(x), axis=2, arr=windows) return fn_of_windows.squeeze() -def make_regex(left_bound, right_bound, values=[]): +def make_regex(left_bound, right_bound, values = None): """Regex creation helper for the data handler. The OfflineDataHandler relies on regexes to parse the file/folder structures and extract data. @@ -74,8 +74,8 @@ def make_regex(left_bound, right_bound, values=[]): The left bound of the regex. right_bound: string The right bound of the regex. - values: list - The values between the two regexes. + values: list or None (optional), default = None + The values between the two regexes. If None, will try to find the values using a wildcard. Defaults to None. Returns ---------- @@ -87,10 +87,16 @@ def make_regex(left_bound, right_bound, values=[]): >>> make_regex(left_bound = "_C_", right_bound="_EMG.csv", values = [0,1,2,3,4,5]) """ left_bound_str = "(?<="+ left_bound +")" - mid_str = "(?:" - for i in values: - mid_str += i + "|" - mid_str = mid_str[:-1] - mid_str += ")" + + if values is None: + # Apply wildcard + mid_str = '(.*?)' + else: + mid_str = "(?:" + for i in values: + mid_str += i + "|" + mid_str = mid_str[:-1] + mid_str += ")" + right_bound_str = "(?=" + right_bound +")" return left_bound_str + mid_str + right_bound_str diff --git a/requirements.txt b/requirements.txt index 619df973..939f0951 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,30 @@ -numpy<2 -scipy==1.7.2 +scipy pytest==7.1.3 pytest-cov==4.0.0 scikit-learn pillow -matplotlib==3.5.0 -librosa==0.9.2 +matplotlib +librosa pyomyo==0.0.5 pyserial wfdb bleak -semantic-version requests +h5py # For Docs -sphinx==5.0.0 -sphinx_rtd_theme==1.0.0 -myst_parser==0.18.1 +sphinx>=8.1.3 +sphinx_rtd_theme>=3.0.2 +myst_parser>=4.0.0 linkify-it-py==2.0.0 # For testing pytest-skip-slow==0.0.3 dearpygui -# For oymotion - only on linx -bluepy -PyWavelets==1.4.1 +PyWavelets # for new sgt dearpygui opencv-python datetime websockets==8.1 +h5py +onedrivedownloader +sifi-bridge-py==2.0.0b18 \ No newline at end of file diff --git a/setup.py b/setup.py index 436f1b7d..ef145baf 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ # python -m twine upload --repository testpypi dist/* --verbose <------ testpypi # -VERSION = "1.0.0" +VERSION = "2.0.0" DESCRIPTION = "LibEMG - Myoelectric Control Library" LONG_DESCRIPTION = "A library for designing and exploring real-time and offline myoelectric control systems." @@ -25,7 +25,7 @@ long_description_content_type="text/markdown", long_description=long_description, install_requires=[ - "numpy<2.0", + "numpy", "scipy", "scikit-learn", "pillow", @@ -35,12 +35,15 @@ "pyserial", "PyWavelets", "requests", - "semantic-version", "websockets", "opencv-python", "pythonnet", "bleak", - "dearpygui" + "dearpygui", + "h5py", + "onedrivedownloader", + "sifi-bridge-py==2.0.0b18", + "pygame", ], keywords=[ "emg", diff --git a/sifi_bridge_windows.exe b/sifi_bridge_windows.exe deleted file mode 100644 index 9f02a93e..00000000 Binary files a/sifi_bridge_windows.exe and /dev/null differ diff --git a/tests/test_offline_metrics.py b/tests/test_offline_metrics.py index a3d833c2..d1e17c01 100644 --- a/tests/test_offline_metrics.py +++ b/tests/test_offline_metrics.py @@ -56,4 +56,18 @@ def test_PREC(om, y_true, y_predictions): def test_F1(om, y_true, y_predictions): # Assuming there is a rounding error - assert om.get_F1(y_true, y_predictions) - f1_score(y_true, y_predictions, average='weighted') < 0.0000000001 \ No newline at end of file + assert om.get_F1(y_true, y_predictions) - f1_score(y_true, y_predictions, average='weighted') < 0.0000000001 + +def test_REMOVE(om): + preds = np.array([0,1,-1,-1,2,2,0,0,-1]) + labels = np.array([0,1,0,0,2,2,0,0,2]) + preds, labels = om._ignore_rejected(preds, labels) + assert np.alltrue(preds == np.array([0,1,2,2,0,0])) + assert np.alltrue(preds == np.array([0,1,2,2,0,0])) + +def test_REMOVE2(om): + preds = np.array([0,1,2,3,4,5,6,7,8,9,0]) + labels = np.array([0,1,2,3,4,5,6,7,8,9,0]) + preds2, labels2 = om._ignore_rejected(preds, labels) + assert np.alltrue(preds2 == preds) + assert np.alltrue(labels2 == labels) \ No newline at end of file