diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..586a174 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +build/* +bridge*.cpp +core +*.so +*.pyx +*.o +*.swp +__pycache__ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b569fb8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +Copyright © 2019, UChicago Argonne, LLC +All Rights Reserved + Software Name: CDI Reconstruction (reccdi) +By: Argonne National Laboratory +OPEN SOURCE LICENSE + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + + +****************************************************************************************************** +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*************************************************************************************************** \ No newline at end of file diff --git a/README.md b/README.md index 5890a9a..cb31b26 100644 --- a/README.md +++ b/README.md @@ -1 +1,64 @@ -# cdi + +cdi + +Coherent Diffraction Imaging technique provides a reconstruction of image of a nanoscale structures. Refer to Wikipedia article CDI description for the technique details. Project summary: Implement and parallelize genetic algorithms and phase retrieval methods for Bragg CDI techniques. + +The CDI experiments are performed at the beamline 34-ID. The team currently uses tool written in Matlab. This tool provides multiple features, and includes newest scientific discoveries in the field. Recently, scientists in general, are shifting to Python written tools, as Python offers better performance, and is easier to maintain. + +Goal of this project is to deliver fast tool, easy to maintain, that includes all the features currently available. There is a prospect to conduct research in the field, and add the new ideas/features when proved successful. + +A genetic algorithm approach to CDI phase retrieval will improve coherent imaging in two aspects. The first is to enable the recovery of highly reproducible images from a given data set. The second is to render previously impossible to image samples amenable to CDI, opening the door to a greater scientific impact for the method. The basic idea is to do the same phasing process with tens to thousands of random starting points. The diversity of results is then exploited to arrive at a highly reproducible image of the sample. Another aspect of genetic algorithm approaches is in the “fitness” criterion used to evaluate the population of results. This can be tuned to enable phase retrieval of datasets that have previously been impossible to produce images from. It is desired to implement and parallelize software for fast processing by non-expert beamline users. Current processing time of a 100 MB sample using serial MATLAB code takes 60 minutes using limited parameters. Current data acquisition time for a 100 MB data set is 20 minutes, and will decrease after the completion of the APS Upgrade. Attaining a robust image of a sample in a computation time nearer the data acquisition time will allow nearer real-time feedback into the experimental parameters. The experimenter may begin to do guided, carefully executed experiments. Currently, the vast majority of Bragg CDI users will benefit from semi-real-time phase retrieval for their data. It will also open the instrument up to far less sophisticated CDI users. This technique will be critical to one or more APS Upgrade beamlines. +Version + +v1.0 - 06/24/2019 +Pre-requisites + + ArrayFire library version 3.5.0 or higher + Libconfig library version 1.5 or higher + Python packages installation: + pip install tifffile + pip install pylibconfig2 + pip install GPUtil + pip install traits + pip install mayavi + pip install xrayutilities (for parsing spec file if using 34Id prep) + +Author(s) + +Barbara Frosik - Principal Software Engineer at Argonne National Laboratory +License + +Copyright (c) UChicago Argonne, LLC. All rights reserved. See LICENSE file. +C++ Libraries + + ArrayFire open source + Libconfig open source + +How to install + + clone the source from the repository: + git clone https://github.com/advancedPhotonSource/cdi + cd cdi + run interactive script to set enviroment variables and set the project: + source init.sh at the promt enter the following: + enter ArrayFire installation directory > (absolute path to ArrayFire installation dir) + enter LibConfig installation directory > (absolute path to LibConfig installation dir) + +How to run + + refer to how_to_run file + + © 2020 GitHub, Inc. + Terms + Privacy + Security + Status + Help + + Contact GitHub + Pricing + API + Training + Blog + About + diff --git a/bin/cdi_window.py b/bin/cdi_window.py new file mode 100644 index 0000000..e9e97f9 --- /dev/null +++ b/bin/cdi_window.py @@ -0,0 +1,1800 @@ +import sys +import os +import shutil +from PyQt5.QtCore import * +from PyQt5.QtGui import * +from PyQt5.QtWidgets import * +import reccdi.src_py.run_scripts.run_data as run_dt +import reccdi.src_py.run_scripts.run_rec as run_rc +#import reccdi.src_py.run_scripts.run_disp as run_dp +import run_disp_tvtk as run_dp +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.beamlines.aps_34id.spec as spec +import reccdi.src_py.utilities.parse_ver as ver +import importlib + + +def select_file(start_dir): + dialog = QFileDialog(None, 'select dir', start_dir) + dialog.setFileMode(QFileDialog.ExistingFile) + dialog.setSidebarUrls([QUrl.fromLocalFile(start_dir)]) + if dialog.exec_() == QDialog.Accepted: + return str(dialog.selectedFiles()[0]) + else: + return None + + +def select_dir(start_dir): + dialog = QFileDialog(None, 'select dir', start_dir) + dialog.setFileMode(QFileDialog.DirectoryOnly) + dialog.setSidebarUrls([QUrl.fromLocalFile(start_dir)]) + if dialog.exec_() == QDialog.Accepted: + return str(dialog.selectedFiles()[0]) + else: + return None + + +def msg_window(text): + msg = QMessageBox() + msg.setIcon(QMessageBox.Information) + msg.setText(text) + msg.setWindowTitle("Info") + msg.exec_() + + +class cdi_gui(QWidget): + def __init__(self, parent=None): + super(cdi_gui, self).__init__(parent) + self.exp_id = None + self.experiment_dir = None + self.working_dir = None + uplayout = QFormLayout() + + self.set_work_dir_button = QPushButton() + uplayout.addRow("Working Directory", self.set_work_dir_button) + self.Id_widget = QLineEdit() + uplayout.addRow("Experiment ID", self.Id_widget) + self.scan_widget = QLineEdit() + uplayout.addRow("scan(s)", self.scan_widget) + # self.set_conf_from_button = QPushButton("Load conf from") + # self.set_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + # uplayout.addRow(self.set_conf_from_button) + # self.create_exp_button = QPushButton('create experiment') + # self.create_exp_button.setStyleSheet("background-color:rgb(120,180,220)") + # uplayout.addRow(self.create_exp_button) + + vbox = QVBoxLayout() + vbox.addLayout(uplayout) + + self.t = cdi_conf_tab(self) + vbox.addWidget(self.t) + + downlayout = QFormLayout() + self.set_conf_from_button = QPushButton("Load conf from") + self.set_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + downlayout.addWidget(self.set_conf_from_button) + self.create_exp_button = QPushButton('set experiment') + self.create_exp_button.setStyleSheet("background-color:rgb(120,180,220)") + downlayout.addWidget(self.create_exp_button) + self.run_button = QPushButton('run everything', self) + self.run_button.setStyleSheet("background-color:rgb(175,208,156)") + downlayout.addWidget(self.run_button) + vbox.addLayout(downlayout) + + self.setLayout(vbox) + self.setWindowTitle("CDI Reconstruction") +# self.init_work_dir() + + self.set_conf_from_button.clicked.connect(self.load_conf_dir) + self.set_work_dir_button.clicked.connect(self.set_working_dir) + self.run_button.clicked.connect(self.run_everything) + self.create_exp_button.clicked.connect(self.set_experiment) + + + def run_everything(self): + if not self.is_exp_exists(): + msg_window('the experiment has not been created yet') + elif not self.is_exp_set(): + msg_window('the experiment has changed, pres "set experiment" button') + else: + self.t.prepare() + self.t.format_data() + self.t.reconstruction() + self.t.display() + + + def is_exp_exists(self): + if self.exp_id is None: + return False + if self.working_dir is None: + return False + exp_id = str(self.Id_widget.text()).strip() + scan = str(self.scan_widget.text()).strip() + if scan != '': + exp_id = exp_id + '_' + scan + if not os.path.exists(os.path.join(self.working_dir, exp_id)): + return False + return True + + + def is_exp_set(self): + if self.exp_id is None: + return False + if self.working_dir is None: + return False + if self.id != str(self.Id_widget.text()).strip(): + return False + if self.scan != str(self.scan_widget.text()).strip(): + return False + return True + + + def load_conf_dir(self): + load_dir = select_dir(os.getcwd()) + missing = 0 + if load_dir is not None: + # load the experiment info only if no id is entered + if os.path.isfile(os.path.join(load_dir, 'config')) and str(self.Id_widget.text()).strip() == '': + self.load_main(load_dir) + else: + missing += 1 + conf_prep_file = os.path.join(load_dir, 'config_prep') + if os.path.isfile(conf_prep_file): + self.t.load_prep_tab(conf_prep_file) + else: + missing += 1 + conf_data_file = os.path.join(load_dir, 'config_data') + if os.path.isfile(conf_data_file): + self.t.load_data_tab(conf_data_file) + else: + missing += 1 + conf_rec_file = os.path.join(load_dir, 'config_rec') + if os.path.isfile(conf_rec_file): + self.t.load_rec_tab(conf_rec_file) + else: + missing += 1 + conf_disp_file = os.path.join(load_dir, 'config_disp') + if os.path.isfile(conf_disp_file): + self.t.load_disp_tab(conf_disp_file) + else: + missing += 1 + if missing == 5: + msg_window('info: no configuration file found in load directory') + else: + self.set_conf_from_button.setStyleSheet("Text-align:left") + self.set_conf_from_button.setText('config loaded') + self.set_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + else: + msg_window('please select valid conf directory') + + + def set_working_dir(self): + self.working_dir = select_dir(self.working_dir) + if self.working_dir is not None: + self.set_work_dir_button.setStyleSheet("Text-align:left") + self.set_work_dir_button.setText(self.working_dir) + else: + self.set_work_dir_button.setText('') + msg_window('please select valid working directory') + return + + + def load_main(self, load_dir): + conf = os.path.join(load_dir, 'config') + if not ver.ver_config(conf): + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + try: + conf_map = ut.read_config(conf) + except Exception: + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + + try: + self.working_dir = conf_map.working_dir + self.set_work_dir_button.setStyleSheet("Text-align:left") + self.set_work_dir_button.setText(self.working_dir) + except: + pass + try: + self.scan = conf_map.scan + self.scan_widget.setText(self.scan) + except: + self.scan = None + try: + self.id = conf_map.experiment_id + self.Id_widget.setText(self.id) + if self.scan != None: + self.exp_id = self.id + '_' + self.scan + else: + self.exp_id = self.id + self.experiment_dir = os.path.join(self.working_dir, self.exp_id) + except: + pass + + if self.experiment_dir is not None: + # this shows default results directory in display tab + self.t.results_dir = os.path.join(self.experiment_dir, 'results') + self.t.result_dir_button.setStyleSheet("Text-align:left") + self.t.result_dir_button.setText(self.t.results_dir) + self.update_rec_configs_choice() + + + def update_rec_configs_choice(self): + # this will update the configuration choices in reconstruction tab + # fill out the config_id choice bar by reading configuration files names + rec_ids = [] + for file in os.listdir(os.path.join(self.experiment_dir, 'conf')): + if file.endswith('_config_rec'): + rec_ids.append(file[0:len(file)-len('_config_rec')]) + if len(rec_ids) > 0: + self.t.rec_id.addItems(rec_ids) + self.t.rec_id.show() + + + def assure_experiment_dir(self): + if not os.path.exists(self.experiment_dir): + os.makedirs(self.experiment_dir) + experiment_conf_dir = os.path.join(self.experiment_dir, 'conf') + if not os.path.exists(experiment_conf_dir): + os.makedirs(experiment_conf_dir) + else: + self.update_rec_configs_choice() + + + def set_experiment(self): + self.id = str(self.Id_widget.text()).strip() + if self.id == '' or self.working_dir is None: + msg_window('id and working directory must be entered') + return + conf_map = {} + + self.scan = str(self.scan_widget.text()).strip() + if self.scan is not '': + scans = self.scan.split('-') + if len(scans) > 2: + msg_window('if entering scan or scan range, please enter numeric values, separated with "-" if range') + return + for sc in scans: + try: + numeric = int(sc) + except: + msg_window('if entering scan or scan range, please enter numeric values, separated with "-" if range') + return + conf_map['scan'] = '"' + self.scan + '"' + self.exp_id = self.id + '_' + self.scan + else: + self.exp_id = self.id + self.experiment_dir = os.path.join(self.working_dir, self.exp_id) + self.assure_experiment_dir() + + # read the configurations from GUI and write to experiment config files + # save the main config + conf_map['working_dir'] = '"' + str(self.working_dir).strip() + '"' + conf_map['experiment_id'] = '"' + self.id + '"' + self.write_conf(conf_map, os.path.join(self.experiment_dir, 'conf'), 'config') + + # save prep config + conf_map = self.t.get_prep_config() + if bool(conf_map): + self.write_conf(conf_map, os.path.join(self.experiment_dir, 'conf'), 'config_prep') + + # save data config + conf_map = self.t.get_data_config() + if bool(conf_map): + self.write_conf(conf_map, os.path.join(self.experiment_dir, 'conf'), 'config_data') + + # save rec config + conf_map = self.t.get_rec_config() + if bool(conf_map): + self.write_conf(conf_map, os.path.join(self.experiment_dir, 'conf'), 'config_rec') + + # save disp config + conf_map = self.t.get_disp_config() + if bool(conf_map): + self.write_conf(conf_map, os.path.join(self.experiment_dir, 'conf'), 'config_disp') + + # this shows default results directory in display window + self.t.results_dir = os.path.join(self.experiment_dir, 'results') + self.t.result_dir_button.setStyleSheet("Text-align:left") + self.t.result_dir_button.setText(self.t.results_dir) + + + def write_conf(self, conf_map, dir, file): + # create "temp" file first, verify it, and if ok, copy to a configuration file + if not os.path.exists(dir): + os.makedirs(dir) + conf_file = os.path.join(dir, file) + temp_file = os.path.join(dir, 'temp') + with open(temp_file, 'a') as f: + for key in conf_map: + value = conf_map[key] + if len(value) > 0: + f.write(key + ' = ' + conf_map[key] + '\n') + f.close() + + if file == 'config': + if not ver.ver_config(temp_file): + os.remove(temp_file) + msg_window('please check the entries in the main window. Cannot save this format') + return False + elif file == 'config_prep': + if not ver.ver_config_prep(temp_file): + os.remove(temp_file) + msg_window('please check the entries in the Data prep tab. Cannot save this format') + return False + elif file == 'config_data': + if not ver.ver_config_data(temp_file): + os.remove(temp_file) + msg_window('please check the entries in the Data tab. Cannot save this format') + return False + elif file.endswith('config_rec'): + if not ver.ver_config_rec(temp_file): + os.remove(temp_file) + msg_window('please check the entries in the Reconstruction tab. Cannot save this format') + return False + elif file == 'config_disp': + if not ver.ver_config_disp(temp_file): + os.remove(temp_file) + msg_window('please check the entries in the Display tab. Cannot save this format') + return False + # copy if verified + shutil.copy(temp_file, conf_file) + os.remove(temp_file) + return True + +class cdi_conf_tab(QTabWidget): + def __init__(self, main_win, parent=None): + super(cdi_conf_tab, self).__init__(parent) + self.main_win = main_win + self.tab1 = QWidget() + self.tab2 = QWidget() + self.tab3 = QWidget() + self.tab4 = QWidget() + + self.data_dir = None + self.specfile = None + self.darkfile = None + self.whitefile = None + self.binning = None + self.results_dir = None + self.addTab(self.tab1, "Data prep") + self.addTab(self.tab2, "Data") + self.addTab(self.tab3, "Reconstruction") + self.addTab(self.tab4, "Display") + self.tab1UI() + self.tab2UI() + self.tab3UI() + self.tab4UI() + + + def tab1UI(self): + self.script = None + self.imported_script = False + layout = QFormLayout() + self.separate_scans = QCheckBox() + layout.addRow("separate scans", self.separate_scans) + self.separate_scans.setChecked(False) + self.data_dir_button = QPushButton() + layout.addRow("data directory", self.data_dir_button) + self.spec_file_button = QPushButton() + layout.addRow("spec file", self.spec_file_button) + self.det_quad = QLineEdit() + layout.addRow("detector quad", self.det_quad) + self.dark_file_button = QPushButton() + layout.addRow("darkfield file", self.dark_file_button) + self.white_file_button = QPushButton() + layout.addRow("whitefield file", self.white_file_button) + self.min_files = QLineEdit() + layout.addRow("min files in scan", self.min_files) + self.exclude_scans = QLineEdit() + layout.addRow("exclude scans", self.exclude_scans) + self.prep = QComboBox() + self.prep.addItem("34ID prep") + self.prep.addItem("custom") + self.prep.addItem("copy from") + layout.addRow("choose data preparation ", self.prep) + # add sub-layout with rows that apply to the choice form above + sub_layout = QFormLayout() + self.load_prep(sub_layout) + layout.addRow(sub_layout) + self.set_prep_conf_from_button = QPushButton("Load prep conf from") + self.set_prep_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + #layout.addWidget(self.set_prep_conf_from_button) + self.prep_button = QPushButton('prepare', self) + self.prep_button.setStyleSheet("background-color:rgb(175,208,156)") + #layout.addWidget(self.prep_button) + layout.addRow(self.set_prep_conf_from_button, self.prep_button) + self.tab1.setLayout(layout) + + self.prep_button.clicked.connect(self.prepare) + self.prep.currentIndexChanged.connect(lambda: self.load_prep(sub_layout)) + self.data_dir_button.clicked.connect(self.set_data_dir) + self.spec_file_button.clicked.connect(self.set_spec_file) + self.dark_file_button.clicked.connect(self.set_dark_file) + self.white_file_button.clicked.connect(self.set_white_file) + self.det_quad.textChanged.connect(lambda: self.set_overriden(self.det_quad)) + self.set_prep_conf_from_button.clicked.connect(self.load_prep_conf) + + + def load_prep(self, layout): + for i in reversed(range(layout.count())): + layout.itemAt(i).widget().deleteLater() + + if str(self.prep.currentText()) == "custom": + self.script_button = QPushButton() + layout.addRow("select script", self.script_button) + self.prep_exec = QLineEdit() + layout.addRow("prep function", self.prep_exec) + self.args = QLineEdit() + layout.addRow("arguments (str/num)", self.args) + self.script_button.clicked.connect(self.set_prep_script) + + elif str(self.prep.currentText()) == "copy from": + self.ready_prep = QPushButton() + layout.addRow("prep file", self.ready_prep) + self.ready_prep.clicked.connect(self.set_prep_file) + + + def tab2UI(self): + layout = QFormLayout() + self.aliens = QLineEdit() + layout.addRow("aliens", self.aliens) + self.amp_intensity = QLineEdit() + layout.addRow("amp intensity", self.amp_intensity) + self.center_shift = QLineEdit() + layout.addRow("center_shift", self.center_shift) + self.adjust_dimensions = QLineEdit() + layout.addRow("pad, crop", self.adjust_dimensions) + self.binning = QLineEdit() + layout.addRow("binning", self.binning) + self.set_data_conf_from_button = QPushButton("Load data conf from") + self.set_data_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + #layout.addRow(self.set_data_conf_from_button) + self.config_data_button = QPushButton('format data', self) + self.config_data_button.setStyleSheet("background-color:rgb(175,208,156)") + #layout.addRow(self.config_data_button) + layout.addRow(self.set_data_conf_from_button, self.config_data_button) + self.tab2.setLayout(layout) + + # this will create config_data file and run data script + # to generate data ready for reconstruction + self.config_data_button.clicked.connect(self.format_data) + self.set_data_conf_from_button.clicked.connect(self.load_data_conf) + + + def tab3UI(self): + self.mult_rec_conf = False + self.old_conf_id = '' + layout = QVBoxLayout() + ulayout = QFormLayout() + mlayout = QHBoxLayout() + self.add_conf_button = QPushButton('add configuration', self) + ulayout.addWidget(self.add_conf_button) + self.rec_id = QComboBox() + self.rec_id.InsertAtBottom + self.rec_id.addItem("") + ulayout.addWidget(self.rec_id) + self.rec_id.hide() + self.proc = QComboBox() + self.proc.addItem("cuda") + self.proc.addItem("opencl") + self.proc.addItem("cpu") + ulayout.addRow("processor type", self.proc) + self.cont = QCheckBox() + ulayout.addRow("continuation", self.cont) + self.cont.setChecked(False) + self.device = QLineEdit() + ulayout.addRow("device(s)", self.device) + self.reconstructions = QLineEdit() + ulayout.addRow("number of reconstructions", self.reconstructions) + self.gc = QLineEdit() + ulayout.addRow("gc triggers", self.gc) + self.alg_seq = QLineEdit() + ulayout.addRow("algorithm sequence", self.alg_seq) + # TODO add logic to show this only if HIO is in sequence + self.beta = QLineEdit() + ulayout.addRow("beta", self.beta) + self.rec_default_button = QPushButton('set to defaults', self) + ulayout.addWidget(self.rec_default_button) + + llayout = QFormLayout() + self.set_rec_conf_from_button = QPushButton("Load rec conf from") + self.set_rec_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + #layout.addWidget(self.set_rec_conf_from_button) + self.features = Features(self, mlayout) + self.config_rec_button = QPushButton('run reconstruction', self) + self.config_rec_button.setStyleSheet("background-color:rgb(175,208,156)") + #layout.addWidget(self.config_rec_button) + llayout.addRow(self.set_rec_conf_from_button, self.config_rec_button) + spacer = QSpacerItem(0,3) + llayout.addItem(spacer) + + layout.addLayout(ulayout) + layout.addLayout(mlayout) + layout.addLayout(llayout) + + self.tab3.setAutoFillBackground(True) + self.tab3.setLayout(layout) + + self.config_rec_button.clicked.connect(self.reconstruction) + self.cont.stateChanged.connect(lambda: self.toggle_cont(ulayout)) + self.rec_default_button.clicked.connect(self.rec_default) + self.add_conf_button.clicked.connect(self.add_rec_conf) + self.rec_id.currentIndexChanged.connect(self.toggle_conf) + self.set_rec_conf_from_button.clicked.connect(self.load_rec_conf_dir) + + + def toggle_cont(self, layout): + cb_label = layout.labelForField(self.cont) + if self.cont.isChecked(): + self.cont_dir = QLineEdit() + layout.insertRow(2, "continue dir", self.cont_dir) + cb_label.setStyleSheet('color: black') + else: + cb_label.setStyleSheet('color: grey') + + + def add_rec_conf(self): + id, ok = QInputDialog.getText(self, '',"enter configuration id") + if ok and len(id) > 0: + if self.mult_rec_conf: + self.rec_id.addItem(id) + else: + self.mult_rec_conf = True + self.rec_id.show() + self.rec_id.addItem(id) + #self.rec_id.setCurrentIndex(self.rec_id.count()-1) + + # copy the config_rec into _config_rec and show the + conf_file = os.path.join(self.main_win.experiment_dir, 'conf', 'config_rec') + new_conf_file = os.path.join(self.main_win.experiment_dir, 'conf', id + '_config_rec') + shutil.copyfile(conf_file, new_conf_file) + self.rec_id.setCurrentIndex(self.rec_id.count()-1) + + + def toggle_conf(self): + # save the configuration file before updating the incoming config + if self.old_conf_id == '': + conf_file = 'config_rec' + else: + conf_file = self.old_conf_id + '_config_rec' + + conf_map = self.get_rec_config() + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + + if self.main_win.write_conf(conf_map, conf_dir, conf_file): + self.old_conf_id = str(self.rec_id.currentText()) + else: + msg_window('configuration ' + conf_file + ' was not saved') + # if a config file corresponding to the rec id exists, load it + # otherwise read from base configuration and load + if self.old_conf_id == '': + conf_file = os.path.join(conf_dir, 'config_rec') + else: + conf_file = os.path.join(conf_dir, self.old_conf_id + '_config_rec') + + if os.path.isfile(conf_file): + self.load_rec_tab(conf_file) + else: + self.load_rec_tab(os.path.join(conf_dir, 'config_rec')) + + + def tab4UI(self): + layout = QFormLayout() + self.result_dir_button = QPushButton() + layout.addRow("results directory", self.result_dir_button) + self.crop = QLineEdit() + layout.addRow("crop", self.crop) + self.rampups = QLineEdit() + layout.addRow("ramp upscale", self.rampups) + self.spec_file_button1 = QPushButton() + layout.addRow("spec file", self.spec_file_button1) + self.energy = QLineEdit() + layout.addRow("energy", self.energy) + self.delta = QLineEdit() + layout.addRow("delta (deg)", self.delta) + self.gamma = QLineEdit() + layout.addRow("gamma (deg)", self.gamma) + self.arm = QLineEdit() + layout.addRow("arm (mm)", self.arm) + self.dth = QLineEdit() + layout.addRow("dth (deg)", self.dth) + self.pixel = QLineEdit() + layout.addRow("pixel", self.pixel) + self.set_disp_conf_from_button = QPushButton("Load disp conf from") + self.set_disp_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + #layout.addRow(self.set_disp_conf_from_button) + self.config_disp = QPushButton('process display', self) + self.config_disp.setStyleSheet("background-color:rgb(175,208,156)") + #layout.addRow(self.config_disp) + layout.addRow(self.set_disp_conf_from_button, self.config_disp) + self.tab4.setLayout(layout) + + self.result_dir_button.clicked.connect(self.set_results_dir) + self.spec_file_button1.clicked.connect(self.set_spec_file) + self.config_disp.clicked.connect(self.display) + self.energy.textChanged.connect(lambda: self.set_overriden(self.energy)) + self.delta.textChanged.connect(lambda: self.set_overriden(self.delta)) + self.gamma.textChanged.connect(lambda: self.set_overriden(self.gamma)) + self.arm.textChanged.connect(lambda: self.set_overriden(self.arm)) + self.dth.textChanged.connect(lambda: self.set_overriden(self.dth)) + self.pixel.textChanged.connect(lambda: self.set_overriden(self.pixel)) + self.set_disp_conf_from_button.clicked.connect(self.load_disp_conf) + self.layout4 = layout + + + def load_prep_tab(self, conf): + if not os.path.isfile(conf): + msg_window('info: the load directory does not contain config_prep file') + return + if not ver.ver_config_prep(conf): + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + try: + conf_map = ut.read_config(conf) + except Exception as e: + msg_window('please check configuration file ' + conf + '. Cannot parse, ' + str(e)) + return + + try: + separate_scans = conf_map.separate_scans + if separate_scans: + self.separate_scans.setChecked(True) + except: + pass + try: + self.data_dir = conf_map.data_dir + self.data_dir_button.setStyleSheet("Text-align:left") + self.data_dir_button.setText(self.data_dir) + except: + self.data_dir = None + self.data_dir_button.setText('') + try: + self.specfile = conf_map.specfile + self.spec_file_button.setStyleSheet("Text-align:left") + self.spec_file_button.setText(self.specfile) + # set specfile also in display tab + self.spec_file_button1.setStyleSheet("Text-align:left") + self.spec_file_button1.setText(self.specfile) + except: + self.specfile = None + self.spec_file_button.setText('') + self.spec_file_button1.setText('') + try: + self.darkfile = conf_map.darkfile + self.dark_file_button.setStyleSheet("Text-align:left") + self.dark_file_button.setText(self.darkfile) + except: + self.darkfile = None + self.dark_file_button.setText('') + try: + self.whitefile = conf_map.whitefile + self.white_file_button.setStyleSheet("Text-align:left") + self.white_file_button.setText(self.whitefile) + except: + self.whitefile = None + self.white_file_button.setText('') + try: + self.min_files.setText(str(conf_map.min_files).replace(" ", "")) + except: + pass + try: + self.exclude_scans.setText(str(conf_map.exclude_scans).replace(" ", "")) + except: + pass + try: + self.det_quad.setText(str(conf_map.det_quad).replace(" ", "")) + except: + pass + prep_file = None + try: + prep_file = conf_map.prep_file + except: + pass + if prep_file is not None: + self.prep.setCurrentIndex(2) + self.ready_prep.setStyleSheet("Text-align:left") + self.ready_prep.setText(prep_file) + + + def load_data_tab(self, conf): + if not os.path.isfile(conf): + msg_window('info: the load directory does not contain config_data file') + return + if not ver.ver_config_data(conf): + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + try: + conf_map = ut.read_config(conf) + except Exception as e: + msg_window('please check configuration file ' + conf + '. Cannot parse, ' + str(e)) + return + + try: + self.aliens.setText(str(conf_map.aliens).replace(" ", "")) + except AttributeError: + pass + try: + self.amp_intensity.setText(str(conf_map.amp_threshold).replace(" ", "")) + except AttributeError: + pass + try: + self.binning.setText(str(conf_map.binning).replace(" ", "")) + except AttributeError: + pass + try: + self.center_shift.setText(str(conf_map.center_shift).replace(" ", "")) + except AttributeError: + pass + try: + self.adjust_dimensions.setText(str(conf_map.adjust_dimensions).replace(" ", "")) + except AttributeError: + pass + + + def load_rec_tab(self, conf): + if not os.path.isfile(conf): + msg_window('info: the load directory does not contain config_rec file') + return + if not ver.ver_config_rec(conf): + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + try: + conf_map = ut.read_config(conf) + except Exception as e: + msg_window('please check configuration file ' + conf + '. Cannot parse, ' + str(e)) + return + + try: + self.device.setText(str(conf_map.device).replace(" ", "")) + except AttributeError: + pass + try: + self.reconstructions.setText(str(conf_map.reconstructions).replace(" ", "")) + except AttributeError: + pass + try: + self.gc.setText(str(conf_map.garbage_trigger).replace(" ", "")) + except AttributeError: + pass + try: + self.alg_seq.setText(str(conf_map.algorithm_sequence).replace(" ", "")) + except AttributeError: + pass + try: + self.beta.setText(str(conf_map.beta).replace(" ", "")) + except AttributeError: + pass + + for feat_id in self.features.feature_dir: + self.features.feature_dir[feat_id].init_config(conf_map) + + # set the results_dir in display tab + self.init_results_dir() + + + def load_disp_tab(self, conf): + if not os.path.isfile(conf): + msg_window('info: the load directory does not contain config_disp file') + return + if not ver.ver_config_data(conf): + msg_window('please check configuration file ' + conf + '. Cannot parse, ') + return + try: + conf_map = ut.read_config(conf) + except Exception as e: + msg_window('please check configuration file ' + conf + '. Cannot parse, ' + str(e)) + return + + try: + specfile = conf_map.specfile + self.specfile = specfile + self.spec_file_button1.setStyleSheet("Text-align:left") + self.spec_file_button1.setText(self.specfile) + if os.path.isfile(self.specfile): + self.parse_spec() + except AttributeError: + pass + # if parameters are configured, override the readingsf rom spec file + try: + self.crop.setText(str(conf_map.crop).replace(" ", "")) + except AttributeError: + pass + try: + self.rampups.setText(str(conf_map.rampups).replace(" ", "")) + except AttributeError: + pass + try: + self.energy.setText(str(conf_map.energy).replace(" ", "")) + self.energy.setStyleSheet('color: black') + except AttributeError: + pass + try: + self.delta.setText(str(conf_map.delta).replace(" ", "")) + self.delta.setStyleSheet('color: black') + except AttributeError: + pass + try: + self.gamma.setText(str(conf_map.gamma).replace(" ", "")) + self.gamma.setStyleSheet('color: black') + except AttributeError: + pass + try: + self.arm.setText(str(conf_map.arm).replace(" ", "")) + self.arm.setStyleSheet('color: black') + except AttributeError: + pass + try: + self.dth.setText(str(conf_map.dth).replace(" ", "")) + self.dth.setStyleSheet('color: black') + except AttributeError: + pass + try: + self.pixel.setText(str(conf_map.pixel).replace(" ", "")) + self.pixel.setStyleSheet('color: black') + except AttributeError: + pass + + + def get_prep_config(self): + conf_map = {} + if self.data_dir is not None: + conf_map['data_dir'] = '"' + str(self.data_dir).strip() + '"' + if self.specfile is not None: + conf_map['specfile'] = '"' + str(self.specfile).strip() + '"' + if self.darkfile is not None: + conf_map['darkfile'] = '"' + str(self.darkfile).strip() + '"' + if self.whitefile is not None: + conf_map['whitefile'] = '"' + str(self.whitefile).strip() + '"' + if self.separate_scans.isChecked(): + conf_map['separate_scans'] = 'true' + if len(self.min_files.text()) > 0: + min_files = str(self.min_files.text()) + conf_map['min_files'] = min_files + if len(self.exclude_scans.text()) > 0: + conf_map['exclude_scans'] = str(self.exclude_scans.text()).replace('\n','') + if len(self.det_quad.text()) > 0: + det_quad = str(self.det_quad.text()) + conf_map['det_quad'] = det_quad + try: + if str(self.ready_prep.text()) != '': + conf_map['prep_file'] = '"' + str(self.prep_file) + '"' + except: + pass + + return conf_map + + + def get_data_config(self): + conf_map = {} + if len(self.aliens.text()) > 0: + conf_map['aliens'] = str(self.aliens.text()).replace('\n', '') + if len(self.amp_intensity.text()) > 0: + conf_map['amp_threshold'] = str(self.amp_intensity.text()) + else: + # msg_window('amplitude threshold not defined. Quiting operation.') + return + if len(self.binning.text()) > 0: + conf_map['binning'] = str(self.binning.text()).replace('\n', '') + if len(self.center_shift.text()) > 0: + conf_map['center_shift'] = str(self.center_shift.text()).replace('\n', '') + if len(self.adjust_dimensions.text()) > 0: + conf_map['adjust_dimensions'] = str(self.adjust_dimensions.text()).replace('\n', '') + + return conf_map + + + def get_rec_config(self): + conf_map = {} + if len(self.reconstructions.text()) > 0: + conf_map['reconstructions'] = str(self.reconstructions.text()) + if len(self.device.text()) > 0: + conf_map['device'] = str(self.device.text()).replace('\n','') + if len(self.gc.text()) > 0: + conf_map['garbage_trigger'] = str(self.gc.text()).replace('\n','') + if len(self.alg_seq.text()) > 0: + conf_map['algorithm_sequence'] = str(self.alg_seq.text()).replace('\n','') + if len(self.beta.text()) > 0: + conf_map['beta'] = str(self.beta.text()) + if self.cont.isChecked(): + conf_map['continue_dir'] = str(self.cont_dir.text()) + + for feat_id in self.features.feature_dir: + self.features.feature_dir[feat_id].add_config(conf_map) + + return conf_map + + + def get_disp_config(self): + conf_map = {} + if self.specfile is not None: + conf_map['specfile'] = '"' + str(self.specfile) + '"' + if len(self.energy.text()) > 0: + conf_map['energy'] = str(self.energy.text()) + if len(self.delta.text()) > 0: + conf_map['delta'] = str(self.delta.text()) + if len(self.gamma.text()) > 0: + conf_map['gamma'] = str(self.gamma.text()) + if len(self.arm.text()) > 0: + conf_map['arm'] = str(self.arm.text()) + if len(self.dth.text()) > 0: + conf_map['dth'] = str(self.dth.text()) + if len(self.pixel.text()) > 0: + conf_map['pixel'] = str(self.pixel.text()).replace('\n', '') + if len(self.crop.text()) > 0: + conf_map['crop'] = str(self.crop.text()).replace('\n', '') + if len(self.rampups.text()) > 0: + conf_map['rampups'] = str(self.rampups.text()).replace('\n', '') + + return conf_map + + + def load_prep_conf(self): + prep_file = select_file(os.getcwd()) + if prep_file is not None: + self.load_prep_tab(prep_file) + self.set_prep_conf_from_button.setStyleSheet("Text-align:left") + self.set_prep_conf_from_button.setText('config loaded') + self.set_prep_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + else: + msg_window('please select valid prep config file') + + + def load_data_conf(self): + data_file = select_file(os.getcwd()) + if data_file is not None: + self.load_data_tab(data_file) + self.set_data_conf_from_button.setStyleSheet("Text-align:left") + self.set_data_conf_from_button.setText('config loaded') + self.set_data_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + # # save data config + # conf_map = self.get_data_config() + # self.main_win.write_conf(conf_map, os.path.join(self.main_win.experiment_dir, 'conf'), 'config_data') + else: + msg_window('please select valid data config file') + + def load_rec_conf_dir(self): + rec_file = select_file(os.getcwd()) + if rec_file is not None: + self.load_rec_tab(rec_file) + self.set_rec_conf_from_button.setStyleSheet("Text-align:left") + self.set_rec_conf_from_button.setText('config loaded') + self.set_rec_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + # # save rec config + # conf_map = self.get_rec_config() + # self.main_win.write_conf(conf_map, os.path.join(self.main_win.experiment_dir, 'conf'), 'config_rec') + else: + msg_window('please select valid rec config file') + + + def load_disp_conf(self): + disp_file = select_file(os.getcwd()) + if disp_file is not None: + self.load_disp_tab(disp_file) + self.set_disp_conf_from_button.setStyleSheet("Text-align:left") + self.set_disp_conf_from_button.setText('config loaded') + self.set_disp_conf_from_button.setStyleSheet("background-color:rgb(205,178,102)") + self.parse_spec() + # # save disp config + # conf_map = self.get_disp_config() + # self.main_win.write_conf(conf_map, os.path.join(self.main_win.experiment_dir, 'conf'), 'config_disp') + else: + msg_window('please select valid disp config file') + + + def set_overriden(self, item): + item.setStyleSheet('color: black') + + + def set_spec_file(self): + self.specfile = select_file(self.specfile) + if self.specfile is not None: + self.spec_file_button.setStyleSheet("Text-align:left") + self.spec_file_button.setText(self.specfile) + self.spec_file_button1.setStyleSheet("Text-align:left") + self.spec_file_button1.setText(self.specfile) + self.parse_spec() + else: + self.spec_file_button.setText('') + self.spec_file_button1.setText('') + + + def parse_spec(self): + try: + last_scan = int(self.main_win.scan.split('-')[-1]) + energy, delta, gamma, dth, arm, pixel = spec.parse_spec(self.specfile, last_scan) + self.energy.setText(str(energy)) + self.energy.setStyleSheet('color: blue') + self.delta.setText(str(delta)) + self.delta.setStyleSheet('color: blue') + self.gamma.setText(str(gamma)) + self.gamma.setStyleSheet('color: blue') + self.dth.setText(str(dth)) + self.dth.setStyleSheet('color: blue') + self.arm.setText(str(arm)) + self.arm.setStyleSheet('color: blue') + self.pixel.setText(str(pixel)) + self.pixel.setStyleSheet('color: blue') + except: + print ('scan not available, cannot parse spec') + + + def set_dark_file(self): + self.darkfile = select_file(self.darkfile) + if self.darkfile is not None: + self.dark_file_button.setStyleSheet("Text-align:left") + self.dark_file_button.setText(self.darkfile) + else: + self.dark_file_button.setText('') + + + def set_white_file(self): + self.whitefile = select_file(self.whitefile) + if self.whitefile is not None: + self.white_file_button.setStyleSheet("Text-align:left") + self.white_file_button.setText(self.whitefile) + else: + self.white_file_button.setText('') + + + def set_data_dir(self): + self.data_dir = select_dir(self.data_dir) + if self.data_dir is not None: + self.data_dir_button.setStyleSheet("Text-align:left") + self.data_dir_button.setText(self.data_dir) + else: + self.data_dir_button.setText('') + + + def set_prep_file(self): + self.prep_file = select_file(self.main_win.working_dir) + if self.prep_file is not None: + selected = str(self.prep_file) + if not selected.endswith('tif') and not selected.endswith('tiff'): + msg_window("the file extension must be tif or tiff") + return + self.ready_prep.setStyleSheet("Text-align:left") + self.ready_prep.setText(self.prep_file) + else: + self.ready_prep.setText('') + + + def set_prep_script(self): + self.script = select_file(self.main_win.working_dir) + if self.script is not None: + self.script_button.setStyleSheet("Text-align:left") + self.script_button.setText(self.script) + # fill the arguments with experiment_dir, scans, config file + conf_file = os.path.join(self.main_win.experiment_dir, 'conf', 'config_prep') + self.args.setText(str(self.main_win.experiment_dir) + ',' + str(self.main_win.scan) + ',' + conf_file) + else: + self.script_button.setText('') + + + def prepare(self): + if not self.main_win.is_exp_exists(): + msg_window('the experiment has not been created yet') + elif not self.main_win.is_exp_set(): + msg_window('the experiment has changed, pres "set experiment" button') + else: + conf_map = self.get_prep_config() + if str(self.prep.currentText()) == "custom": + self.prepare_custom(conf_map) + elif str(self.prep.currentText()) == "34ID prep": + self.prepare_34id(conf_map) + elif str(self.prep.currentText()) == "copy from": + self.prepare_copy(conf_map) + + + def prepare_custom(self, conf_map): + # determine script directory and script name + if self.script is None: + msg_window("script not defined") + return + full_script = str(self.script) + script_info = full_script.split('/') + script = script_info[len(script_info)-1] + script_dir = full_script[0 : -(len(script)+1)] + script = script[0 : -3] + func = str(self.prep_exec.text()) + if len(func) == 0: + msg_window("function not defined") + return + + current_dir = os.getcwd() + args = str(self.args.text()) + if len(args) == 0: + args = [] + else: + args = args.split(',') + for i in range(len(args)): + try: + if args[i].find('.') == -1: + args[i] = int(args[i]) + else: + args[i] = float(args[i]) + except: + pass + try: + if args[i].find('-') > -1: + l = args[i].split('-') + nl = [] + for n in l: + nl.append(int(n)) + args[i] = nl + except: + pass + + os.chdir(script_dir) + sys.path.append(script_dir) + if not self.imported_script: + self.m = importlib.import_module(script) + self.imported_script = True + else: + self.m = importlib.reload(self.m) + os.chdir(current_dir) + f = getattr(self.m, func) + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + if self.main_win.write_conf(conf_map, conf_dir, 'config_prep'): + try: + prep_data = f(*args) + except Exception as e: + msg_window('custom script failed ' + str(e)) + return + if prep_data is not None: + tif_file = os.path.join(self.main_win.experiment_dir, 'prep', 'prep_data.tif') + ut.save_tif(prep_data, tif_file) + print ('done with prep') + + + def prepare_34id(self, conf_map): + mod = importlib.import_module('reccdi.src_py.run_scripts.run_34id_prepare') + scan = str(self.main_win.scan_widget.text()) + try: + # after checking that scan is entered convert it to list of int + scan_range = scan.split('-') + for i in range(len(scan_range)): + scan_range[i] = int(scan_range[i]) + except: + pass + + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + conf_file = os.path.join(conf_dir, 'config_prep') + if self.main_win.write_conf(conf_map, conf_dir, 'config_prep'): + f = getattr(mod, 'prepare') + f(self.main_win.experiment_dir, scan_range, conf_file) + if self.separate_scans.isChecked(): + self.results_dir = self.main_win.experiment_dir + self.result_dir_button.setStyleSheet("Text-align:left") + self.result_dir_button.setText(self.results_dir) + + + def prepare_copy(self, conf_map): + # save the file as experiment prep file + prep_dir = os.path.join(self.main_win.experiment_dir, 'prep') + if not os.path.exists(prep_dir): + os.makedirs(prep_dir) + exp_prep_file = os.path.join(prep_dir, 'prep_data.tif') + shutil.copyfile(self.prep_file, exp_prep_file) + # save config_prep + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + self.main_win.write_conf(conf_map, conf_dir, 'config_prep') + + + def format_data(self): + if not self.main_win.is_exp_exists(): + msg_window('the experiment has not been created yet') + elif not self.main_win.is_exp_set(): + msg_window('the experiment has changed, pres "set experiment" button') + else: + if os.path.isfile(os.path.join(self.main_win.experiment_dir, 'prep','prep_data.tif'))\ + or self.separate_scans.isChecked(): + conf_map = self.get_data_config() + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + if self.main_win.write_conf(conf_map, conf_dir, 'config_data'): + run_dt.data(self.main_win.experiment_dir) + else: + msg_window('Please, run data preparation in previous tab to activate this function') + + + def reconstruction(self): + if not self.main_win.is_exp_exists(): + msg_window('the experiment has not been created yet') + elif not self.main_win.is_exp_set(): + msg_window('the experiment has changed, pres "set experiment" button') + else: + if os.path.isfile(os.path.join(self.main_win.experiment_dir, 'data', 'data.tif'))\ + or self.separate_scans.isChecked(): + # find out which configuration should be saved + if self.old_conf_id == '': + conf_file = 'config_rec' + conf_id = None + else: + conf_file = self.old_conf_id + '_config_rec' + conf_id = self.old_conf_id + + conf_map = self.get_rec_config() + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + + if self.main_win.write_conf(conf_map, conf_dir, conf_file): + run_rc.manage_reconstruction(str(self.proc.currentText()), self.main_win.experiment_dir, conf_id) + + # set the results_dir in display tab. + self.init_results_dir() + else: + msg_window('Please, run format data in previous tab to activate this function') + + + def init_results_dir(self): + # if alternate configuration was chosen in reconstruction tab, use it in results_dir + if self.old_conf_id == '': + res_file = 'results' + else: + res_file = self.old_conf_id + '_results' + # set the results_dir in display tab. If GA, set it to the best results dir, if separate scans + # set to experiment + ga_feat = self.features.feature_dir['GA'] + if ga_feat.active.isChecked() and int(ga_feat.generations.text()) > 1: + generations = int(ga_feat.generations.text()) + # if only one reconstruction, it will be saved in gen dir, otherwise, + # the directories will be enumerated + if int(self.reconstructions.text()) > 1: + self.results_dir = os.path.join(self.main_win.experiment_dir, res_file, + 'g_' + str(generations-1), '0') + else: + self.results_dir = os.path.join(self.main_win.experiment_dir, res_file, + 'g_' + str(generations-1)) + else: + self.results_dir = os.path.join(self.main_win.experiment_dir, res_file) + if self.separate_scans.isChecked(): + self.results_dir = self.main_win.experiment_dir + self.result_dir_button.setStyleSheet("Text-align:left") + self.result_dir_button.setText(self.results_dir) + + + def set_results_dir(self): + if self.main_win.is_exp_exists(): + + self.results_dir = os.path.join(self.main_win.experiment_dir, 'results') + self.results_dir = select_dir(self.results_dir) + if self.results_dir is not None: + self.result_dir_button.setStyleSheet("Text-align:left") + self.result_dir_button.setText(self.results_dir) + else: + self.result_dir_button.setText('') + else: + msg_window('the experiment has not been created yet') + + + def display(self): + if not self.main_win.is_exp_exists(): + msg_window('the experiment has not been created yet') + return + if not self.main_win.is_exp_set(): + msg_window('the experiment has changed, pres "set experiment" button') + return + # check if the results exist + if self.results_dir is None: + self.results_dir = self.main_win.experiment_dir + is_result = False + for (dirpath, dirnames, filenames) in os.walk(self.results_dir): + for file in filenames: + if file.endswith('image.npy'): + is_result = True + break + if is_result: + break + if not is_result: + msg_window('No image files found in the results directory tree. Please, run reconstruction in previous tab to activate this function') + return + if (self.specfile is None or not os.path.isfile(self.specfile)) and \ + (len(self.energy.text()) == 0 or \ + len(self.delta.text()) == 0 or \ + len(self.gamma.text()) == 0 or \ + len(self.arm.text()) == 0 or \ + len(self.dth.text()) == 0 or \ + len(self.pixel.text()) == 0): + msg_window('Please, enter valid spec file or all detector parameters') + return + + conf_map = self.get_disp_config() + + conf_dir = os.path.join(self.main_win.experiment_dir, 'conf') + if self.main_win.write_conf(conf_map, conf_dir, 'config_disp'): + run_dp.to_vtk(self.main_win.experiment_dir, self.results_dir) + + + def rec_default(self): + if self.main_win.working_dir is None or self.main_win.id is None or \ + len(self.main_win.working_dir) == 0 or len(self.main_win.id) == 0: + msg_window('Working Directory or Reconstruction ID not configured') + else: + self.reconstructions.setText('1') + self.device.setText('(0,1)') + self.gc.setText('(1000)') + self.alg_seq.setText('((3,("ER",20),("HIO",180)),(1,("ER",20)))') + self.beta.setText('.9') + self.cont.setChecked(False) + + +class Feature(object): + def __init__(self): + self.stack = QWidget() + + + def stackUI(self, item, feats): + layout = QFormLayout() + self.active = QCheckBox("active") +# self.active.setChecked(True) + layout.addWidget(self.active) + self.toggle(layout, item, feats) + self.stack.setLayout(layout) + self.active.stateChanged.connect(lambda: self.toggle(layout, item, feats)) + + + def toggle(self, layout, item, feats): + if self.active.isChecked(): + self.fill_active(layout) + + self.default_button = QPushButton('set to defaults', feats) + layout.addWidget(self.default_button) + self.default_button.clicked.connect(self.rec_default) + + item.setForeground(QColor('black')); + else: + for i in reversed(range(1, layout.count())): + layout.itemAt(i).widget().setParent(None) + item.setForeground(QColor('grey')); + + def fill_active(self, layout): + pass + + + def rec_default(self): + pass + + + def add_config(self, conf_map): + if self.active.isChecked(): + self.add_feat_conf(conf_map) + + def add_feat_conf(self, conf_map): + pass + + + def init_config(self, conf_map): + pass + + +class GA(Feature): + def __init__(self): + super(GA, self).__init__() + self.id = 'GA' + + # override setting the active to set it False + def stackUI(self, item, feats): + super(GA, self).stackUI(item, feats) + + + def init_config(self, conf_map): + try: + gens = conf_map.generations + self.active.setChecked(True) + self.generations.setText(str(gens).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.metrics.setText(str(conf_map.ga_metrics).replace(" ", "")) + except AttributeError: + pass + try: + self.breed_modes.setText(str(conf_map.ga_breed_modes).replace(" ", "")) + except AttributeError: + pass + try: + self.removes.setText(str(conf_map.ga_cullings).replace(" ", "")) + except AttributeError: + pass + try: + self.ga_support_thresholds.setText(str(conf_map.ga_support_thresholds).replace(" ", "")) + except AttributeError: + pass + try: + self.ga_support_sigmas.setText(str(conf_map.ga_support_sigmas).replace(" ", "")) + except AttributeError: + pass + try: + self.lr_sigmas.setText(str(conf_map.ga_low_resolution_sigmas).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.generations = QLineEdit() + layout.addRow("generations", self.generations) + self.metrics = QLineEdit() + layout.addRow("fitness metrics", self.metrics) + self.breed_modes = QLineEdit() + layout.addRow("breed modes", self.breed_modes) + self.removes = QLineEdit() + layout.addRow("cullings", self.removes) + self.ga_support_thresholds = QLineEdit() + layout.addRow("after breed support thresholds", self.ga_support_thresholds) + self.ga_support_sigmas = QLineEdit() + layout.addRow("after breed support sigmas", self.ga_support_sigmas) + self.lr_sigmas = QLineEdit() + layout.addRow("low resolution sigmas", self.lr_sigmas) + + + def rec_default(self): + self.generations.setText('5') + self.metrics.setText('("chi","chi","area","chi","sharpness")') + self.breed_modes.setText('("sqrt_ab","sqrt_ab","max_all","Dhalf","sqrt_ab")') + self.removes.setText('(2,2,1)') + self.ga_support_thresholds.setText('(.1,.1,.1,.1,.1)') + self.ga_support_sigmas.setText('(1.0,1.0,1.0,1.0)') + self.lr_sigmas.setText('(2.0,1.5)') + self.active.setChecked(True) + + + def add_feat_conf(self, conf_map): + conf_map['generations'] = str(self.generations.text()) + conf_map['ga_metrics'] = str(self.metrics.text()).replace('\n','') + conf_map['ga_breed_modes'] = str(self.breed_modes.text()).replace('\n','') + conf_map['ga_cullings'] = str(self.removes.text()).replace('\n','') + conf_map['ga_support_thresholds'] = str(self.ga_support_thresholds.text()).replace('\n','') + conf_map['ga_support_sigmas'] = str(self.ga_support_sigmas.text()).replace('\n','') + conf_map['ga_low_resolution_sigmas'] = str(self.lr_sigmas.text()).replace('\n','') + + +class low_resolution(Feature): + def __init__(self): + super(low_resolution, self).__init__() + self.id = 'low resolution' + + + def init_config(self, conf_map): + try: + triggers = conf_map.resolution_trigger + self.active.setChecked(True) + self.res_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.sigma_range.setText(str(conf_map.iter_res_sigma_range).replace(" ", "")) + except AttributeError: + pass + try: + self.det_range.setText(str(conf_map.iter_res_det_range).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.res_triggers = QLineEdit() + layout.addRow("low resolution triggers", self.res_triggers) + self.sigma_range = QLineEdit() + layout.addRow("sigma range", self.sigma_range) + self.det_range = QLineEdit() + layout.addRow("det range", self.det_range) + + + def rec_default(self): + #TODO add to accept fractions in trigger, so the default will be (.5,1) + self.res_triggers.setText('(0, 1, 320)') + self.sigma_range.setText('(2.0)') + self.det_range.setText('(.7)') + + + def add_feat_conf(self, conf_map): + conf_map['resolution_trigger'] = str(self.res_triggers.text()).replace('\n','') + conf_map['iter_res_sigma_range'] = str(self.sigma_range.text()).replace('\n','') + conf_map['iter_res_det_range'] = str(self.det_range.text()).replace('\n','') + + +class amplitude_support(Feature): + def __init__(self): + super(amplitude_support, self).__init__() + self.id = 'amplitude support' + + + def init_config(self, conf_map): + try: + triggers = conf_map.amp_support_trigger + self.active.setChecked(True) + self.support_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.support_type.setText(str(conf_map.support_type).replace(" ", "")) + except AttributeError: + pass + try: + self.support_area.setText(str(conf_map.support_area).replace(" ", "")) + except AttributeError: + pass + try: + self.threshold.setText(str(conf_map.support_threshold).replace(" ", "")) + except AttributeError: + pass + try: + self.sigma.setText(str(conf_map.support_sigma).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.support_triggers = QLineEdit() + layout.addRow("support triggers", self.support_triggers) + self.support_type = QLineEdit() + layout.addRow("support algorithm", self.support_type) + self.support_area = QLineEdit() + layout.addRow("starting support area", self.support_area) + self.threshold = QLineEdit() + layout.addRow("threshold", self.threshold) + self.sigma = QLineEdit() + layout.addRow("sigma", self.sigma) + + + def rec_default(self): + self.support_triggers.setText('(1,1)') + self.support_type.setText('GAUSS') + self.support_area.setText('(.5,.5,.5)') + self.sigma.setText('1.0') + self.threshold.setText('0.1') + + + def add_feat_conf(self, conf_map): + conf_map['amp_support_trigger'] = str(self.support_triggers.text()).replace('\n','') + conf_map['support_type'] = '"' + str(self.support_type.text()) + '"' + conf_map['support_threshold'] = str(self.threshold.text()) + conf_map['support_sigma'] = str(self.sigma.text()) + conf_map['support_area'] = str(self.support_area.text()).replace('\n','') + + +class phase_support(Feature): + def __init__(self): + super(phase_support, self).__init__() + self.id = 'phase support' + + + def init_config(self, conf_map): + try: + triggers = conf_map.phase_support_trigger + self.active.setChecked(True) + self.phase_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.phase_min.setText(str(conf_map.phase_min).replace(" ", "")) + except AttributeError: + pass + try: + self.phase_max.setText(str(conf_map.phase_max).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.phase_triggers = QLineEdit() + layout.addRow("phase support triggers", self.phase_triggers) + self.phase_min = QLineEdit() + layout.addRow("phase minimum", self.phase_min) + self.phase_max = QLineEdit() + layout.addRow("phase maximum", self.phase_max) + + + def rec_default(self): + self.phase_triggers.setText('(0,1,20)') + self.phase_min.setText('-1.57') + self.phase_max.setText('1.57') + + + def add_feat_conf(self, conf_map): + conf_map['phase_support_trigger'] = str(self.phase_triggers.text()).replace('\n','') + conf_map['phase_min'] = str(self.phase_min.text()) + conf_map['phase_max'] = str(self.phase_max.text()) + + +class pcdi(Feature): + def __init__(self): + super(pcdi, self).__init__() + self.id = 'pcdi' + + + def init_config(self, conf_map): + try: + triggers = conf_map.pcdi_trigger + self.active.setChecked(True) + self.pcdi_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.pcdi_type.setText(str(conf_map.partial_coherence_type).replace(" ", "")) + except AttributeError: + pass + try: + self.pcdi_iter.setText(str(conf_map.partial_coherence_iteration_num).replace(" ", "")) + except AttributeError: + pass + try: + self.pcdi_normalize.setText(str(conf_map.partial_coherence_normalize).replace(" ", "")) + except AttributeError: + pass + try: + self.pcdi_roi.setText(str(conf_map.partial_coherence_roi).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.pcdi_triggers = QLineEdit() + layout.addRow("pcdi triggers", self.pcdi_triggers) + self.pcdi_type = QLineEdit() + layout.addRow("partial coherence algorithm", self.pcdi_type) + self.pcdi_iter = QLineEdit() + layout.addRow("pcdi iteration number", self.pcdi_iter) + self.pcdi_normalize = QLineEdit() + layout.addRow("normalize", self.pcdi_normalize) + self.pcdi_roi = QLineEdit() + layout.addRow("pcdi kernel area", self.pcdi_roi) + + + def rec_default(self): + self.pcdi_triggers.setText('(50,50)') + self.pcdi_type.setText('LUCY') + self.pcdi_iter.setText('20') + self.pcdi_normalize.setText('true') + self.pcdi_roi.setText('(32,32,32)') + + + def add_feat_conf(self, conf_map): + conf_map['pcdi_trigger'] = str(self.pcdi_triggers.text()).replace('\n','') + conf_map['partial_coherence_type'] = '"' + str(self.pcdi_type.text()) + '"' + conf_map['partial_coherence_iteration_num'] = str(self.pcdi_iter.text()) + conf_map['partial_coherence_normalize'] = str(self.pcdi_normalize.text()) + conf_map['partial_coherence_roi'] = str(self.pcdi_roi.text()).replace('\n','') + + +class twin(Feature): + def __init__(self): + super(twin, self).__init__() + self.id = 'twin' + + + def init_config(self, conf_map): + try: + triggers = conf_map.twin_trigger + self.active.setChecked(True) + self.twin_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + try: + self.twin_halves.setText(str(conf_map.twin_halves).replace(" ", "")) + except AttributeError: + pass + + + def fill_active(self, layout): + self.twin_triggers = QLineEdit() + layout.addRow("twin triggers", self.twin_triggers) + self.twin_halves = QLineEdit() + layout.addRow("twin halves", self.twin_halves) + + + def rec_default(self): + self.twin_triggers.setText('(2)') + self.twin_halves.setText('(0,0)') + + + def add_feat_conf(self, conf_map): + conf_map['twin_trigger'] = str(self.twin_triggers.text()).replace('\n','') + conf_map['twin_halves'] = str(self.twin_halves.text()).replace('\n','') + + +class average(Feature): + def __init__(self): + super(average, self).__init__() + self.id = 'average' + + + def init_config(self, conf_map): + try: + triggers = conf_map.average_trigger + self.active.setChecked(True) + self.average_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + + + def fill_active(self, layout): + self.average_triggers = QLineEdit() + layout.addRow("average triggers", self.average_triggers) + + + def rec_default(self): + self.average_triggers.setText('(-400,1)') + + + def add_feat_conf(self, conf_map): + conf_map['average_trigger'] = str(self.average_triggers.text()).replace('\n','') + + +class progress(Feature): + def __init__(self): + super(progress, self).__init__() + self.id = 'progress' + + + def init_config(self, conf_map): + try: + triggers = conf_map.progress_trigger + self.active.setChecked(True) + self.progress_triggers.setText(str(triggers).replace(" ", "")) + except AttributeError: + self.active.setChecked(False) + return + + + def fill_active(self, layout): + self.progress_triggers = QLineEdit() + layout.addRow("progress triggers", self.progress_triggers) + + + def rec_default(self): + self.progress_triggers.setText('(0,20)') + + + def add_feat_conf(self, conf_map): + conf_map['progress_trigger'] = str(self.progress_triggers.text()).replace('\n','') + + + +class Features(QWidget): + def __init__(self, tab, layout): + super(Features, self).__init__() + feature_ids = ['GA', 'low resolution', 'amplitude support', 'phase support', 'pcdi', 'twin', 'average', 'progress'] + self.leftlist = QListWidget() + self.feature_dir = {'GA' : GA(), + 'low resolution' : low_resolution(), + 'amplitude support' : amplitude_support(), + 'phase support' : phase_support(), + 'pcdi' : pcdi(), + 'twin' : twin(), + 'average' : average(), + 'progress' : progress()} + self.Stack = QStackedWidget(self) + for i in range(len(feature_ids)): + id = feature_ids[i] + self.leftlist.insertItem(i, id) + feature = self.feature_dir[id] + feature.stackUI(self.leftlist.item(i), self) + self.Stack.addWidget(feature.stack) + + layout.addWidget(self.leftlist) + layout.addWidget(self.Stack) + + self.leftlist.currentRowChanged.connect(self.display) + + + def display(self, i): + self.Stack.setCurrentIndex(i) + + +def main(): + app = QApplication(sys.argv) + ex = cdi_gui() + ex.show() + sys.exit(app.exec_()) + + +if __name__ == '__main__': + main() diff --git a/bin/cdi_window.sh b/bin/cdi_window.sh new file mode 100644 index 0000000..21fa991 --- /dev/null +++ b/bin/cdi_window.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +#specific for xfm1 +p=$PATH +p=${p//"anaconda2"/"anaconda3"} +p=${p//"CXDUSER/anaconda"/"CXDUSER/CDI/anaconda"} +export PATH=$p + +export LD_LIBRARY_PATH=LIB_PATH +python bin/cdi_window.py diff --git a/bin/everything.py b/bin/everything.py new file mode 100644 index 0000000..d97514c --- /dev/null +++ b/bin/everything.py @@ -0,0 +1,7 @@ +import reccdi.src_py.run_scripts.everything as ev +import sys + +if __name__ == "__main__": + ev.main(sys.argv[1:]) + +#python everything.py device prefix scans conf_dir diff --git a/bin/everything.sh b/bin/everything.sh new file mode 100644 index 0000000..0e54d97 --- /dev/null +++ b/bin/everything.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +#specific for xfm1 +p=$PATH +p=${p//"anaconda2"/"anaconda3"} +p=${p//"CXDUSER/anaconda"/"CXDUSER/CDI/anaconda"} +export PATH=$p + +export LD_LIBRARY_PATH=LIB_PATH +dev=$1 +prefix=$2 +scans=$3 +conf_dir=$4 + +python bin/everything.py $dev $prefix $scans $conf_dir + diff --git a/bin/prepare_34id.py b/bin/prepare_34id.py new file mode 100755 index 0000000..8242c04 --- /dev/null +++ b/bin/prepare_34id.py @@ -0,0 +1,7 @@ +import sys +import reccdi.src_py.run_scripts.run_34id_prepare as prep + +if __name__ == "__main__": + exit(prep.main(sys.argv[1:])) + +# python run_prepare.py prefix scans conf_dir diff --git a/bin/run_data.py b/bin/run_data.py new file mode 100755 index 0000000..c1fb31b --- /dev/null +++ b/bin/run_data.py @@ -0,0 +1,7 @@ +import reccdi.src_py.run_scripts.run_data as dt +import sys + +if __name__ == "__main__": + dt.main(sys.argv[1:]) + +#python run_data.py experiment_dir \ No newline at end of file diff --git a/bin/run_disp_pyevtk.py b/bin/run_disp_pyevtk.py new file mode 100755 index 0000000..8f0b029 --- /dev/null +++ b/bin/run_disp_pyevtk.py @@ -0,0 +1,177 @@ +import reccdi.src_py.utilities.viz_util_xu_pyevtk as vu +import reccdi.src_py.utilities.utils as ut +from reccdi.src_py.utilities.utils import measure +import reccdi.src_py.utilities.parse_ver as ver +import argparse +import sys +import os +import numpy as np +from multiprocessing import Pool + +def save_CX(conf_dict, image, support, coh, save_dir): + params = vu.DispalyParams(conf_dict) +# image = np.swapaxes(image, 1,2) +# image = np.swapaxes(image, 0,1) +# support = np.swapaxes(support, 1,2) +# support = np.swapaxes(support, 0,1) + print("center image and support") + image, support = vu.center(image, support) + print("remove phase ramp on image") + image = vu.remove_ramp(image, ups=conf_dict['rampups']) + print("set viz") + viz = vu.CXDViz() + viz.set_geometry(params, image.shape) +# crop = get_crop(params, image.shape) +# viz.set_crop(crop[0], crop[1], crop[2]) # save image + + print("set im amps") + viz.add_array(abs(image), "imAmp", space='direct') + print("set im phase") + viz.add_array(np.angle(image), "imPh", space='direct') + image_file = os.path.join(save_dir, 'image') + print("write im") + viz.write_directspace(image_file) + viz.clear_direct_arrays() + + + print("set support") + viz.add_array(support, "support", space='direct') + support_file = os.path.join(save_dir, 'support') + print("write support") + viz.write_directspace(support_file) + viz.clear_direct_arrays() + + if coh is not None: + coh = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(coh))) + coh = ut.get_zero_padded_centered(coh, image.shape) + coh_file = os.path.join(save_dir, 'coherence') + viz.add_array(np.abs(coh), 'cohAmp', space='direct') + viz.add_array(np.angle(coh), 'cohPh', space='direct') + viz.write_directspace(coh_file) + viz.clear_direct_arrays() + + +def save_vtk(res_dir_conf): + (res_dir, conf_dict) = res_dir_conf + try: + imagefile = os.path.join(res_dir, 'image.npy') + image = np.load(imagefile) + except: + print ('cannot load "image.npy" file') + return + + try: + supportfile = os.path.join(res_dir, 'support.npy') + support = np.load(supportfile) + except: + print ('support file is missing in ' + res_dir + ' directory') + return + + try: + reciprocalfile = os.path.join(res_dir, 'reciprocal.npy') + reciprocal = np.load(reciprocalfile) + # reciprocal is saved as tif file, so no need to pass it to cx module + # saving amp, phase, and square of modulus in tif format + reciprocal_amp = np.absolute(reciprocal) + reciprocal_phase = np.angle(reciprocal) + reciprocal_sq_mod = np.power(reciprocal_amp, 2) + + ut.save_tif(reciprocal_amp, os.path.join(res_dir, 'reciprocal_amp.tif')) + ut.save_tif(reciprocal_phase, os.path.join(res_dir, 'reciprocal_phase.tif')) + ut.save_tif(reciprocal_sq_mod, os.path.join(res_dir, 'reciprocal_sq_mod.tif')) + except: + print ('info: cannot save reciprocal space') + + cohfile = os.path.join(res_dir, 'coherence.npy') + if os.path.isfile(cohfile): + coh = np.load(cohfile) + save_CX(conf_dict, image, support, coh, res_dir) + else: + save_CX(conf_dict, image, support, None, res_dir) + + +def to_vtk(experiment_dir, results_dir=None): + if not os.path.isdir(experiment_dir): + print("Please provide a valid experiment directory") + return + conf_dir = os.path.join(experiment_dir, 'conf') + conf = os.path.join(conf_dir, 'config_disp') + # verify configuration file + if not ver.ver_config_disp(conf): + print ('incorrect configuration file ' + conf +', cannot parse') + return + + # parse the conf once here and save it in dictionary, it will apply to all images in the directory tree + conf_dict = {} + try: + conf_map = ut.read_config(conf) + items = conf_map.items() + for item in items: + conf_dict[item[0]] = item[1] + except: + print('cannot parse configuration file ' + conf) + return + + # get last scan from the config file and add it to conf_dict + last_scan = None + main_conf = os.path.join(conf_dir, 'config') + if os.path.isfile(main_conf): + try: + config_map = ut.read_config(main_conf) + scan = config_map.scan + last_scan = scan.split('-')[-1] + conf_dict['last_scan'] = int(last_scan) + except: + print ("info: scan not determined, can't read " + conf + " configuration file") + + # get binning from the config_data file and add it to conf_dict + binning = None + data_conf = os.path.join(conf_dir, 'config_data') + if os.path.isfile(data_conf): + try: + conf_map = ut.read_config(data_conf) + conf_dict['binning'] = conf_map.binning + except: + pass + + no_gpus = 1 + rec_conf = os.path.join(conf_dir, 'config_rec') + if os.path.isfile(rec_conf): + try: + conf_map = ut.read_config(rec_conf) + device = conf_map.device + no_gpus = len(device) + except: + pass + + if results_dir is None: + results_dir = experiment_dir + # find directories with image.npy file + dirs = [] + for (dirpath, dirnames, filenames) in os.walk(results_dir): + for file in filenames: + if file.endswith('image.npy'): + dirs.append((dirpath, conf_dict)) + save_vtk(dirs[0]) +# with Pool(processes = no_gpus) as pool: +# pool.map_async(save_vtk, dirs) +# pool.close() +# pool.join() + + +def main(arg): + print ('preparing display') + parser = argparse.ArgumentParser() + parser.add_argument("experiment_dir", help="experiment directory") + parser.add_argument("--results_dir", help="directory in experiment that has a tree (or leaf) with reconstruction results which will be visualized") + args = parser.parse_args() + experiment_dir = args.experiment_dir + if args.results_dir: + to_vtk(experiment_dir, args.results_dir) + else: + to_vtk(experiment_dir) + +if __name__ == "__main__": + main(sys.argv[1:]) + +#python run_disp.py experiment_dir diff --git a/bin/run_disp_tvtk.py b/bin/run_disp_tvtk.py new file mode 100755 index 0000000..57e8efd --- /dev/null +++ b/bin/run_disp_tvtk.py @@ -0,0 +1,183 @@ +import reccdi.src_py.utilities.viz_util as vu +import reccdi.src_py.beamlines.aps_34id.viz as v +import reccdi.src_py.utilities.utils as ut +from reccdi.src_py.utilities.utils import measure +import reccdi.src_py.utilities.parse_ver as ver +import argparse +import sys +import os +import numpy as np +from multiprocessing import Pool + +def save_CX(conf_dict, image, support, coh, save_dir): + params = v.DispalyParams(conf_dict) + for k in params.__dict__: + print("params",k,params.__dict__[k]) +# image = np.swapaxes(image, 1,2) +# image = np.swapaxes(image, 0,1) +# support = np.swapaxes(support, 1,2) +# support = np.swapaxes(support, 0,1) + print("center image and support") + image, support = vu.center(image, support) + print("remove phase ramp on image") + image = vu.remove_ramp(image, ups=conf_dict['rampups']) + print("set viz") + viz = v.CXDViz() + viz.set_geometry(params, image.shape) + + print("set im amps") + viz.add_ds_array(abs(image), "imAmp") + print("set im phase") + viz.add_ds_array(np.angle(image), "imPh") + image_file = os.path.join(save_dir, 'image') + # print("write im") + viz.write_directspace(image_file) + viz.clear_direct_arrays() + + + print("set support") + viz.add_ds_array(support, "support") + support_file = os.path.join(save_dir, 'support') + # print("write support") + viz.write_directspace(support_file) + viz.clear_direct_arrays() + + if coh is not None: + coh = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(coh))) + coh = ut.get_zero_padded_centered(coh, image.shape) + coh_file = os.path.join(save_dir, 'coherence') + viz.add_ds_array(np.abs(coh), 'cohAmp') + viz.add_ds_array(np.angle(coh), 'cohPh') + viz.write_directspace(coh_file) + viz.clear_direct_arrays() + +#seems all of this could be consolidated with save_CX. +def save_vtk(res_dir_conf): + (res_dir, conf_dict) = res_dir_conf + try: + imagefile = os.path.join(res_dir, 'image.npy') + image = np.load(imagefile) + except: + print ('cannot load "image.npy" file') + return + + try: + supportfile = os.path.join(res_dir, 'support.npy') + support = np.load(supportfile) + except: + print ('support file is missing in ' + res_dir + ' directory') + return + + try: + reciprocalfile = os.path.join(res_dir, 'reciprocal.npy') + reciprocal = np.load(reciprocalfile) + # reciprocal is saved as tif file, so no need to pass it to cx module + # saving amp, phase, and square of modulus in tif format + reciprocal_amp = np.absolute(reciprocal) + reciprocal_phase = np.angle(reciprocal) + reciprocal_sq_mod = np.power(reciprocal_amp, 2) + + ut.save_tif(reciprocal_amp, os.path.join(res_dir, 'reciprocal_amp.tif')) + ut.save_tif(reciprocal_phase, os.path.join(res_dir, 'reciprocal_phase.tif')) + ut.save_tif(reciprocal_sq_mod, os.path.join(res_dir, 'reciprocal_sq_mod.tif')) + except: + print ('info: cannot save reciprocal space') + + cohfile = os.path.join(res_dir, 'coherence.npy') + if os.path.isfile(cohfile): + coh = np.load(cohfile) + save_CX(conf_dict, image, support, coh, res_dir) + else: + save_CX(conf_dict, image, support, None, res_dir) + +#This is the first thing called by main +#reads the config_disp file into a dictionary +#Gets the binning param from config_data +#Gets GPU list from config_rec +#in principle I think all of this could go to DisplayParams? +def to_vtk(experiment_dir, results_dir=None): + if not os.path.isdir(experiment_dir): + print("Please provide a valid experiment directory") + return + conf_dir = os.path.join(experiment_dir, 'conf') + conf = os.path.join(conf_dir, 'config_disp') + # verify configuration file +# if not ver.ver_config_disp(conf): +# print ('incorrect configuration file ' + conf +', cannot parse') +# return + + # parse the conf once here and save it in dictionary, it will apply to all images in the directory tree + conf_dict = {} + try: + conf_map = ut.read_config(conf) + items = conf_map.items() + for item in items: + conf_dict[item[0]] = item[1] + except: + print('cannot parse configuration file ' + conf) + return + + # get last scan from the config file and add it to conf_dict + last_scan = None + main_conf = os.path.join(conf_dir, 'config') + if os.path.isfile(main_conf): + try: + config_map = ut.read_config(main_conf) + scan = config_map.scan + last_scan = scan.split('-')[-1] + conf_dict['last_scan'] = int(last_scan) + except: + print ("info: scan not determined, can't read " + conf + " configuration file") + + # get binning from the config_data file and add it to conf_dict + binning = None + data_conf = os.path.join(conf_dir, 'config_data') + if os.path.isfile(data_conf): + try: + conf_map = ut.read_config(data_conf) + conf_dict['binning'] = conf_map.binning + except: + pass + + no_gpus = 1 + rec_conf = os.path.join(conf_dir, 'config_rec') + if os.path.isfile(rec_conf): + try: + conf_map = ut.read_config(rec_conf) + device = conf_map.device + no_gpus = len(device) + except: + pass + + if results_dir is None: + results_dir = experiment_dir + # find directories with image.npy file + dirs = [] + for (dirpath, dirnames, filenames) in os.walk(results_dir): + for file in filenames: + if file.endswith('image.npy'): + dirs.append((dirpath, conf_dict)) +#this overrides the pooling and will only work for a single reconstruction. Just for testing. + save_vtk(dirs[0]) +# with Pool(processes = no_gpus) as pool: +# pool.map_async(save_vtk, dirs) +# pool.close() +# pool.join() + + +def main(arg): + print ('preparing display') + parser = argparse.ArgumentParser() + parser.add_argument("experiment_dir", help="experiment directory") + parser.add_argument("--results_dir", help="directory in experiment that has a tree (or leaf) with reconstruction results which will be visualized") + args = parser.parse_args() + experiment_dir = args.experiment_dir + if args.results_dir: + to_vtk(experiment_dir, args.results_dir) + else: + to_vtk(experiment_dir) + +if __name__ == "__main__": + main(sys.argv[1:]) + +#python run_disp.py experiment_dir diff --git a/bin/run_rec.py b/bin/run_rec.py new file mode 100755 index 0000000..3d990c9 --- /dev/null +++ b/bin/run_rec.py @@ -0,0 +1,7 @@ +import reccdi.src_py.run_scripts.run_rec as rec +import sys + +if __name__ == "__main__": + rec.main(sys.argv[1:]) + +# python run_rec.py 'opencl' experiment_dir diff --git a/bin/run_rec.sh b/bin/run_rec.sh new file mode 100644 index 0000000..c8ce957 --- /dev/null +++ b/bin/run_rec.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +#specific for xfm1 +p=$PATH +p=${p//"anaconda2"/"anaconda3"} +p=${p//"CXDUSER/anaconda"/"CXDUSER/CDI/anaconda"} +export PATH=$p + +export LD_LIBRARY_PATH=LIB_PATH + +python bin/run_rec.py $1 $2 + +#python run_rec.py "opencl" diff --git a/bin/setenv.sh b/bin/setenv.sh new file mode 100644 index 0000000..cd90992 --- /dev/null +++ b/bin/setenv.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export LD_LIBRARY_PATH=/home/beams/CXDUSER/CDI/libconfig/lib:/usr/local/lib:/home/beams/CXDUSER/CDI/arrayfire/lib64:/usr/lib64:/usr/nvvm/lib64 diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..1a332d7 --- /dev/null +++ b/build.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +echo PREFIX +echo $PREFIX +conda config --remove channels conda-forge + +python setup.py build_ext --inplace +python setup.py install + +#conda build -c conda-forge -c defaults . + +conda config --add channels conda-forge + +echo PREFIX +echo $PREFIX + +# conda install --use-local reccdi diff --git a/build/Makefile b/build/Makefile deleted file mode 100644 index 09034c6..0000000 --- a/build/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -CC=g++ $(COMPILER_OPTIONS) -COMPILER_OPTIONS=-std=c++11 -g - -TOP = .. -INCLUDE = $(TOP)/include -BIN = bin -SRC = $(TOP)/src_cpp - - -all: - - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/algorithm.o $(SRC)/algorithm.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/bridge.o $(SRC)/bridge.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/manager.o $(SRC)/manager.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/parameters.o $(SRC)/parameters.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/state.o $(SRC)/state.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/support.o $(SRC)/support.cpp - $(CC) -I$(INCLUDE) -I$(INCLUDE)/af/include -fPIC -c -o $(BIN)/worker.o $(SRC)/worker.cpp - - $(CC) -shared -Wl,-soname,$(abspath ../lib/libcdi.so) -o $(abspath ../lib/libcdi.so) $(BIN)/algorithm.o $(BIN)/bridge.o $(BIN)/manager.o $(BIN)/parameters.o $(BIN)/state.o $(BIN)/support.o $(BIN)/worker.o -lc - -#$(CC) -o maincdi $(SRC)/main.cpp -L../lib -lcdi -lafopencl -lconfig++ -I$(INCLUDE) - - diff --git a/conf/config b/conf/config new file mode 100644 index 0000000..83516e1 --- /dev/null +++ b/conf/config @@ -0,0 +1,9 @@ +// This configuration contains parameters defining the experiment +working_dir = "/local/bfrosik/cdi/test" + // working directory; the new experiment will + // be created in this directory +experiment_id = ab + // a string id + +scan = 2-7 + // a single number or a range defining scans, optional diff --git a/conf/config_data b/conf/config_data new file mode 100644 index 0000000..86f7563 --- /dev/null +++ b/conf/config_data @@ -0,0 +1,36 @@ +// GENERAL +data_dir = "data" + // directory where prepared data is saved, default "data" + +// PARAMETER FOR DATA PREPARATION +aliens = ((170,220,112,195,245,123), (50,96,10,60,110,20)) + // comment out aliens for no removal, otherwise input them as + // aliens=[[x0,y0,z0,x1,y1,z1],[x2,y2,z2,x3,y3,z3]] + // will remove two instances of aliens given by the pairs + // #0 and #1 and another given by #2,#3. Accepts as + // many as you like. + +amp_threshold = 2.0 + // mandatory + // min data threshold. Values below this are set to 0. The threshold is applied + // after removing. + +adjust_dimensions = (-13, -13, -65, -65, -65, -65) + // optional + // enter adjust dimensions tuple + // If number is positive, the array will be padded. If negative, cropped. + // The parameters correspond to [x left, x right, y left, y right, z left, z right] + // The final dimensions will be adjusted up to the good number for the FFT which + // also is compatible with opencl supported dimensions + // powers of 2 or a*2^n, where a is 3, 5, or 9 + +center_shift = (0,0,0) + // optional + // enter center shift tuple + // the array maximum is centered before binning, and moved according to center_shift, + // (0,0,0) has no effect + +binning = (1,1,1) + // optional + // enter binning tuple + // defines binning values in respective dimensions, (1,1,1) has no effect diff --git a/conf/config_disp b/conf/config_disp new file mode 100644 index 0000000..07f2ee2 --- /dev/null +++ b/conf/config_disp @@ -0,0 +1,24 @@ +// size to crop the image array, can be entered as actual size or fraction +//crop = (120,120,120) +crop = (.75, .75, .75) + +//If specfile is defined the parameters for visualization will be parsed from it +specfile = "/net/s34data/export/34idc-data/2019/Staff19-1/Staff19-1a.spec" + +// PARAMETERS FOR VISUALISATION +// wavelength +energy = .13933; +// delta (degrees) +delta = 30.1; +// gamma (degrees) +gamma = 14.0; +// camera distance (m) +arm=0.6350; +// angular step size +dth=0.000174532925199; +// detector pixel size +pixel = (55.0e-6, 55.0e-6); +// +rampups = 1 + + diff --git a/conf/config_rec b/conf/config_rec new file mode 100644 index 0000000..3252ec6 --- /dev/null +++ b/conf/config_rec @@ -0,0 +1,171 @@ +// This file contains all configurable parameters that are applied during reconstruction. +// There are general parameters that apply to the main thread of reconstruction and features parameters. +// Reconstruction will use the global definitions and may use any of the supported features: +// twin, amp_support, phase_support, pcdi, resolution, average, garbage collection. +// The iterations during which the feature is active is defined by a trigger. In addition each feature may have other +// configurable parameters. +// Trigger can be defined as a single iteration, or multiple iterations. +// examples: +// (3) trigger at iteration 3 +// (20, 5) trigger starts at iteration 20, repeats every 5 iteration for the rest of run +// (20, 5, 40) trigger starts at iteration 20, repeats every 5 iteration until iteration 40 +// Triggers can also be a combination of any of the above, ex: ((4), (3, 7, 24), (6,20)) +// If a trigger is not defined, the feature is turned off. +// Adding a new feature/ trigger is described in common.h header file. + +// GENERAL +//data_dir = "data" + // directory from which data is read + +//save_dir = "results" + // directory where results of reconstruction are saved as npy files + // if samples > 1, result from each thread will be stored in subdirectory 1,2,3 etc. + +cont = false + // only applied if generations equals 1, or not defined + // if true, the reconstruction start with previous results stored in continue_dir + +// continue_dir = "cont" + // directory from which results are read for reconstruction continuation + // if the directory contains subdirectories, a thread will start for each subdirectory + +reconstructions = 1 + // number of reconstructions to start with + // typically used when running genetic algorithm + +device = (0,1) + // IDs of the target devices for each thread (reconstruction). + // If not defined, it will default to -1 for the OS to select device + +garbage_trigger = (10,5) + // ArrayFire memory management is not reliable, the way around is to call garbage + // collection per defined number of iterations. Decrease this value if out of memory + // error occurs + +algorithm_sequence = ((3, ("ER",20), ("HIO", 180)), (1,("ER",20))) + // defines algorithm applied in each iteration during modulus projection by a sequence of lists. + // The first number in a list is a repeat, followed by lists of pairs, each + // pair defining algorithm and number of iterations to run the algorithm. + +beta = .9; + // used in hio algorithm + +// GENERATIc ALGORITHM +generations = 1 // number of generations + +ga_metrics = ("chi", "sharpness") + // defines which metric should be used to rank the reconstruction results + // supported: 'chi', 'sharpness', 'summed_phase', 'area', ('TV') + +ga_breed_modes = ("sqrt_ab", "dsqrt") + // defines which breeding mode to use to populate new generation. If "none" + // there is no breeding + // supported: 'sqrt_ab', 'dsqrt', 'pixel_switch', 'b_pa', '2ab_a_b', + // '2a-b_pa', 'sqrt_ab_pa', 'sqrt_ab_pa_recip', 'sqrt_ab_recip', + // 'max_ab', 'max_ab_pa', 'min_ab_pa', 'avg_ab', 'avg_ab_pa' + +// ga_cullings = (2,1) + // defines how many worst samples to remove in breeding phase for each generation + // defaults to 0 + +ga_support_thresholds = (.15, .1) + // the support is recalculated with this threshold after breeding phase + // defaults to support threshold + +ga_support_sigmas = (1.1, 1.0) + // the support is recalculated with this sigma after breeding phase + // defaults to support sigma + +// ga_low_resolution_sigmas = (2.0, 1.5) + // list of sigmas that will be used in subsequent generations to calculate + // Gauss (assuming algorithm is GAUSS) and apply it to the data + // This determines low resolution generations number + +// TWIN +// twin feature trims the image array at the current state by zeroing half of the array in each dimension. + +twin_trigger = (2) + // twin defines at which iteration to cut half of the array(i.e. multiply by 0s), + // Comment out, if don't want to apply twin. + // when running GA applied only in first generation + +twin_halves = (0, 0) + // defines which half of the array is zeroed out in x and y dimensions. + // If 0, the first half in that dimension is zeroed out, otherwise, the second half. + +// SUPPORT +// Support area is an array that defines region in which the image is meaningful. This area is recalculated at the +// trigger iteration. The calculation employ an algorithm defined here as support_type. + +amp_support_trigger = (10, 1) + // defines when to update support array using the parameters below. + // Comment out, if support feature not used. +support_type = "GAUSS"; +support_threshold = 0.1; +support_sigma = 1.0; +support_area = (.5,.5,.5) + // initial support area. If the values are fractional, the support area will be calculated + // by multiplying by the data array dimensions. The support will be set to 1s to this + // dimensions centered. + +// PHASE CONSTRAIN +// At the begginning iterations the support area is modified in respect to the phase. Support area will exclude points +// that phase is outside of the defined bounds + +phase_support_trigger = (0, 1, 310) + // defines when to update support array using the parameters below by applaying phase constrain. + // Comment out, if phase constrain feature not used. + // when running GA applied only in first generation +phase_min = -1.57; +phase_max = 1.57; + +// PARTIAL COHERENCE +// Partial coherence triggers recalculation of coherence array for the amplitudes in reciprocal space. +// After first coherence array is determined, it is used for convolution in subsequent iteration. + +pcdi_trigger = (50, 50) + // defines when to update coherence using the parameters below. + // Comment out, if pcdi feature not used. +partial_coherence_type = "LUCY"; + // partial coherence algorithm +partial_coherence_iteration_num = 20; +partial_coherence_normalize = true; +partial_coherence_roi = (32,32,32); + // coherence area. If the values are fractional, the coherence area will be calculated + // by multiplying by the data array dimensions. + +// LOW RESOLUTION +// At the beginning iterations the data resolution and sigma used in recalculation of the support area are modified gradually. +// The sigma for each iteration where the low resolution is applied is a linespaced result of iter_res_sigma_range. +// The last sigma is typically set to support_sigma. If the last sigma is not specified, it defaults to support_sigma. +// The sigma is used by in recalculation of the support area, i.e when support trigger is on. +// The iter_res_det_range is similiary linespaced for the duration of low resolution iterations. The values are used +// as sigmas to calculate Gaussian distribution and applied (multiplied) to data. + +resolution_trigger = (0, 1, 320) + // defines when to apply low resolution using the parameters below. + // Comment out, if low resolution feature not used. + // when running GA applied only in first generation + +iter_res_sigma_range = (2.0) + // used when applying low resolution to replace support sigma. + // The sigmas are linespaced for low resolution iterations from first value to last. + // If only one number given, the last sigma will default to support_sigma. + +iter_res_det_range = (.7) + // used when applying low resolution data mask while iterating. + // The det values are linespaced for low resolution iterations from first value to last. + // The mask is gauss with sigma of linespaced det. If only one number given, + // the last det will default to 1. + +// AVERAGING +// The amplitudes of the last several iterations are averaged. This trigger defines at which iteration the averaging +// starts. + +average_trigger = (-65, 1) + // defines when to apply averaging. Negative start means it is offset from the last iteration + // Comment out, if averaging not used. + +progress_trigger = (0, 20) + // defines when to print info on the console + // the info includes current iteration and error diff --git a/config b/config new file mode 100644 index 0000000..45d24be --- /dev/null +++ b/config @@ -0,0 +1,9 @@ +// This configuration contains parameters defining the experiment +working_dir = "/local/bfrosik/cdi/test" + // working directory; the new experiment will + // be created in this directory +experiment_id = ab + // a string id + +scan = 2-7 + // a single number or a range defining scans, optional \ No newline at end of file diff --git a/config.test b/config.test deleted file mode 100644 index d4d68d2..0000000 --- a/config.test +++ /dev/null @@ -1,89 +0,0 @@ -// PARAMETER FOR DATA PREPARATION -aliens = ((170,220,112,195,245,123), (50,96,10,60,110,20)) -binning = [2,2,2] -center_shift = [0,0,0] -zero_pad = [0,4,0] -save_data = true - - -// RECONSTRUCTION PARAMETERS USED BY FAST MODULE -// ArrayFire memory management is not reliable, the way around is to call garbage collection per -// defined number of iterations. Decrease this value if out of memory error occurs -gc = 5; - -//algorithm_sequence = ((3, ("ER",2), ("HIO", 2), ("ER", 2)), (2, ("ER",3), ("HIO",3))) -//algorithm_sequence = ((1, ("ER",6))) -algorithm_sequence = ((1, ("ER",19), ("HIO", 10)), (1,("ER",10))) - -// twin defines at which iteration to cut half of the array(i.e. multiply by 0s), it is done in ER/HIO -// if negative, no action -twin = 2; - -amp_threshold = 2.0; -amp_threshold_fill_zeros = true; - -phase_min = -2.0; -phase_max = 2.0; -beta = .9; - -// support -// trigger list contains starting iteration, step, and ending iteration (if missing, run to the end) -//support_triggers = ((5, 5, 300), (1, 2)); -support_triggers = ((4, 5, 300)); -support_type = "GAUSS"; -support_threshold = 0.1; -support_threshold_adjust = false; -support_sigma = 1; -support_area = [.5,.5,.5]; - -roi = [32,32,32]; - -// partial coherence -// if type is not defined, partial coherence is not applied -partial_coherence_type = "LUCY"; -partial_coherence_triggers = ((14, 15)); -partial_coherence_iteration_num = 20; -partial_coherence_normalize = true; -// this is an area that will determine coherence, coherence array will come with this size -partial_coherence_roi = [32,32,32]; -// this is an area of the coherence array that will be used to apply coherence to amplitudes -partial_coherence_kernel = [32,32,32]; -partial_coherence_clip = false; - -regularized_amp = "GAUSS"; - -avg_iterations = 15; - - -// PARAMETERS FOR VISUALISATION -// wavelength -lamda = .13933; -// delta (degrees) -delta = 30.1; -// gamma (degrees) -gamma = 14.0; -// camera distance (m) -arm=0.6350; -// angular step size -dth=0.000174532925199; - -// detector pixel sixe -pixel = [55.0e-6, 55.0e-6]; - -// save in two vtk files -save_two_files = False; - -// size to crop the image array -crop = [120,120,120] - -// defines whether save image and support in .mat files -save_results = true - -// the dir of results vtk files -res_dir = "vtk" - - - - - - diff --git a/config_data b/config_data new file mode 100644 index 0000000..911f7d4 --- /dev/null +++ b/config_data @@ -0,0 +1,36 @@ +// GENERAL +// data_dir = "data" + // directory where prepared data is saved, default "data" + +// PARAMETER FOR DATA PREPARATION +aliens = ((170,220,112,195,245,123), (50,96,10,60,110,20)) + // comment out aliens for no removal, otherwise input them as + // aliens=((x0,y0,z0,x1,y1,z1),(x2,y2,z2,x3,y3,z3)) + // will remove two instances of aliens given by the pairs + // #0 and #1 and another given by #2,#3. Accepts as + // many as you like. + +amp_threshold = 2.0 + // mandatory + // min data threshold. Values below this are set to 0. The threshold is applied + // after removing. + +adjust_dimensions = (-13, -13, -65, -65, -65, -65) + // optional + // enter adjust dimensions list + // If number is positive, the array will be padded. If negative, cropped. + // The parameters correspond to (x left, x right, y left, y right, z left, z right) + // The final dimensions will be adjusted up to the good number for the FFT which + // also is compatible with opencl supported dimensions + // powers of 2 or a*2^n, where a is 3, 5, or 9 + +center_shift = (0,0,0) + // optional + // enter center shift list + // the array maximum is centered before binning, and moved according to center_shift, + // (0,0,0) has no effect + +binning = (1,1,1) + // optional + // enter binning list + // defines binning values in respective dimensions, (1,1,1) has no effect diff --git a/config_disp b/config_disp new file mode 100644 index 0000000..e74f32b --- /dev/null +++ b/config_disp @@ -0,0 +1,26 @@ +// PARAMETERS FOR VISUALISATION +// typically the parameters will be parsed from spec file +specfile = "/net/s34data/export/34idc-data/2019/Staff19-3/Staff19-3d.spec" + +// the parsed parameters will be overridden if they are configured +// wavelength +energy = .13933 +// delta (degrees) +delta = 30.1 +// gamma (degrees) +gamma = 14.0 +// camera distance (mm) +arm=0.6350 +// angular step size +dth=0.000174532925199 + +// detector pixel sixe +pixel = (55.0e-6, 55.0e-6) + +// size to crop the image array +//crop = (120,120,120) +crop = (.75, .75, .75) + +//upsize when running ramp removal +rampups = 1 + diff --git a/config_prep b/config_prep new file mode 100644 index 0000000..04b4bc9 --- /dev/null +++ b/config_prep @@ -0,0 +1,39 @@ +data_dir = "/net/s34data/export/34idc-data/2019/Staff19-1/ADStaff19-1a" + // The directory containing raw data. This directory will have several + // subdirectories each containing .tif data files. Each subdirectory + // represents separate scan and is numbered with the scan index + +specfile = "/net/s34data/export/34idc-data/2019/Staff19-1/Staff19-1a.spec" + // a specfile captured during experiment + +darkfile = "/net/s34data/export/34idc-work/2019/dark.tif" + // dark file taken at the time of experiment + +whitefile = "/net/s34data/export/34idc-work/2019/CelaWhiteField.tif" + // white file taken at the time of experiment + +exclude_scans = (78,81) + // a list containing scan indexes that will be excluded from preparation + +min_files = 80 + // only scans containing minimum of .tif files will be included + +// det_quad = 1 + // defines which detector quad was used during experiment. + // 0: all four quads + // 1: lower left + // 2: top left + // 3: lower right + // 4: top right + // this information is typically read from spec file, + // is required if not included in spec file or no spec file is given +separate_scans = false + // in typical scenario the data from all scans in experiment are + // combined. + // if specified as separate scans, each scan will be processed separately + // and will have sub-experiment name containing scan index + // ex. "scan_9", where 9 is scan index + +// prep_file = "/net/s34data/export/34idc-work/2019/test/a_9/prep/prep_data.tiff" + // tif file that has been already read (prepared) + // this parameter is relevant when using GUI \ No newline at end of file diff --git a/config_rec b/config_rec new file mode 100644 index 0000000..3252ec6 --- /dev/null +++ b/config_rec @@ -0,0 +1,171 @@ +// This file contains all configurable parameters that are applied during reconstruction. +// There are general parameters that apply to the main thread of reconstruction and features parameters. +// Reconstruction will use the global definitions and may use any of the supported features: +// twin, amp_support, phase_support, pcdi, resolution, average, garbage collection. +// The iterations during which the feature is active is defined by a trigger. In addition each feature may have other +// configurable parameters. +// Trigger can be defined as a single iteration, or multiple iterations. +// examples: +// (3) trigger at iteration 3 +// (20, 5) trigger starts at iteration 20, repeats every 5 iteration for the rest of run +// (20, 5, 40) trigger starts at iteration 20, repeats every 5 iteration until iteration 40 +// Triggers can also be a combination of any of the above, ex: ((4), (3, 7, 24), (6,20)) +// If a trigger is not defined, the feature is turned off. +// Adding a new feature/ trigger is described in common.h header file. + +// GENERAL +//data_dir = "data" + // directory from which data is read + +//save_dir = "results" + // directory where results of reconstruction are saved as npy files + // if samples > 1, result from each thread will be stored in subdirectory 1,2,3 etc. + +cont = false + // only applied if generations equals 1, or not defined + // if true, the reconstruction start with previous results stored in continue_dir + +// continue_dir = "cont" + // directory from which results are read for reconstruction continuation + // if the directory contains subdirectories, a thread will start for each subdirectory + +reconstructions = 1 + // number of reconstructions to start with + // typically used when running genetic algorithm + +device = (0,1) + // IDs of the target devices for each thread (reconstruction). + // If not defined, it will default to -1 for the OS to select device + +garbage_trigger = (10,5) + // ArrayFire memory management is not reliable, the way around is to call garbage + // collection per defined number of iterations. Decrease this value if out of memory + // error occurs + +algorithm_sequence = ((3, ("ER",20), ("HIO", 180)), (1,("ER",20))) + // defines algorithm applied in each iteration during modulus projection by a sequence of lists. + // The first number in a list is a repeat, followed by lists of pairs, each + // pair defining algorithm and number of iterations to run the algorithm. + +beta = .9; + // used in hio algorithm + +// GENERATIc ALGORITHM +generations = 1 // number of generations + +ga_metrics = ("chi", "sharpness") + // defines which metric should be used to rank the reconstruction results + // supported: 'chi', 'sharpness', 'summed_phase', 'area', ('TV') + +ga_breed_modes = ("sqrt_ab", "dsqrt") + // defines which breeding mode to use to populate new generation. If "none" + // there is no breeding + // supported: 'sqrt_ab', 'dsqrt', 'pixel_switch', 'b_pa', '2ab_a_b', + // '2a-b_pa', 'sqrt_ab_pa', 'sqrt_ab_pa_recip', 'sqrt_ab_recip', + // 'max_ab', 'max_ab_pa', 'min_ab_pa', 'avg_ab', 'avg_ab_pa' + +// ga_cullings = (2,1) + // defines how many worst samples to remove in breeding phase for each generation + // defaults to 0 + +ga_support_thresholds = (.15, .1) + // the support is recalculated with this threshold after breeding phase + // defaults to support threshold + +ga_support_sigmas = (1.1, 1.0) + // the support is recalculated with this sigma after breeding phase + // defaults to support sigma + +// ga_low_resolution_sigmas = (2.0, 1.5) + // list of sigmas that will be used in subsequent generations to calculate + // Gauss (assuming algorithm is GAUSS) and apply it to the data + // This determines low resolution generations number + +// TWIN +// twin feature trims the image array at the current state by zeroing half of the array in each dimension. + +twin_trigger = (2) + // twin defines at which iteration to cut half of the array(i.e. multiply by 0s), + // Comment out, if don't want to apply twin. + // when running GA applied only in first generation + +twin_halves = (0, 0) + // defines which half of the array is zeroed out in x and y dimensions. + // If 0, the first half in that dimension is zeroed out, otherwise, the second half. + +// SUPPORT +// Support area is an array that defines region in which the image is meaningful. This area is recalculated at the +// trigger iteration. The calculation employ an algorithm defined here as support_type. + +amp_support_trigger = (10, 1) + // defines when to update support array using the parameters below. + // Comment out, if support feature not used. +support_type = "GAUSS"; +support_threshold = 0.1; +support_sigma = 1.0; +support_area = (.5,.5,.5) + // initial support area. If the values are fractional, the support area will be calculated + // by multiplying by the data array dimensions. The support will be set to 1s to this + // dimensions centered. + +// PHASE CONSTRAIN +// At the begginning iterations the support area is modified in respect to the phase. Support area will exclude points +// that phase is outside of the defined bounds + +phase_support_trigger = (0, 1, 310) + // defines when to update support array using the parameters below by applaying phase constrain. + // Comment out, if phase constrain feature not used. + // when running GA applied only in first generation +phase_min = -1.57; +phase_max = 1.57; + +// PARTIAL COHERENCE +// Partial coherence triggers recalculation of coherence array for the amplitudes in reciprocal space. +// After first coherence array is determined, it is used for convolution in subsequent iteration. + +pcdi_trigger = (50, 50) + // defines when to update coherence using the parameters below. + // Comment out, if pcdi feature not used. +partial_coherence_type = "LUCY"; + // partial coherence algorithm +partial_coherence_iteration_num = 20; +partial_coherence_normalize = true; +partial_coherence_roi = (32,32,32); + // coherence area. If the values are fractional, the coherence area will be calculated + // by multiplying by the data array dimensions. + +// LOW RESOLUTION +// At the beginning iterations the data resolution and sigma used in recalculation of the support area are modified gradually. +// The sigma for each iteration where the low resolution is applied is a linespaced result of iter_res_sigma_range. +// The last sigma is typically set to support_sigma. If the last sigma is not specified, it defaults to support_sigma. +// The sigma is used by in recalculation of the support area, i.e when support trigger is on. +// The iter_res_det_range is similiary linespaced for the duration of low resolution iterations. The values are used +// as sigmas to calculate Gaussian distribution and applied (multiplied) to data. + +resolution_trigger = (0, 1, 320) + // defines when to apply low resolution using the parameters below. + // Comment out, if low resolution feature not used. + // when running GA applied only in first generation + +iter_res_sigma_range = (2.0) + // used when applying low resolution to replace support sigma. + // The sigmas are linespaced for low resolution iterations from first value to last. + // If only one number given, the last sigma will default to support_sigma. + +iter_res_det_range = (.7) + // used when applying low resolution data mask while iterating. + // The det values are linespaced for low resolution iterations from first value to last. + // The mask is gauss with sigma of linespaced det. If only one number given, + // the last det will default to 1. + +// AVERAGING +// The amplitudes of the last several iterations are averaged. This trigger defines at which iteration the averaging +// starts. + +average_trigger = (-65, 1) + // defines when to apply averaging. Negative start means it is offset from the last iteration + // Comment out, if averaging not used. + +progress_trigger = (0, 20) + // defines when to print info on the console + // the info includes current iteration and error diff --git a/flow_b b/flow_b deleted file mode 100644 index 4e79284..0000000 --- a/flow_b +++ /dev/null @@ -1,84 +0,0 @@ -iteration flow --------------- - -1. ModulusProjection - -- save image as previous image - -- amplitudes(reciprocical) = ifft(image) - -- normalize: amplitudes * data array size - -- amplitude_abs = abs(amplitudes) - ---- pcdi (modifies amplitude_abs) - ---- cut out roi region and do it on this subarray - ---- if update coherence(pcdi trigger) - -------- do partial coherence on (2*amplitudes_abs_previous - amplitudes_abs, data_abs) - -------- if normalize amplitudes_abs = sqrt( amplitudes_abs^2 / sum(amplitudes_abs^2) * sum(data_abs^2) ) - -------- kernel set to roi size - -------- coherence = do Lucy(amplitudes_abs^2, data_abs^2) - -------- find max element in coherence and shift the array in all dimensions to place it at [0,0,0] - -------- normalize coherence abs(coh)/norm_coh - ---- amplitudes_abs = sqrt(fftconvolve(abs(amplitudes)^2, coherence) - back to the big dimensions - -- apply amplitude threshold on the amplitudes (not affected by pcdi) - -- apply ratio amplitudes * amplitudes_abs/data_abs - -- record error = sum(abs(amplitudes)-abs(data))^2/sum(data)^2 - -- image = fft(amplitudes) - -- normalize: image / data array size - - -2.1 ER -- image * support - - -2.2 HIO - -- image * support + (1 - support) * (prev_image - image * beta)) - - -2.3 ER_NORM - -- image_norm_prev - -- image * support - -- image_norm - -- image * sqrt(image_norm/image_norm_prev) - - -2.4 HIO_NORM - -- image_norm_prev - -- phases - -- phase condition: (phase < params->GetPhaseMin()) || (phase > params->GetPhaseMax()) || (support->GetSupportArray() == 0) - -- for phase condition and support replace image with prev_image - image * beta - -- image_norm - -- image * sqrt(image_norm/image_norm_prev) - - -3. - -- update support (not implemented) - - diff --git a/flow_j b/flow_j deleted file mode 100644 index 75e92f0..0000000 --- a/flow_j +++ /dev/null @@ -1,69 +0,0 @@ -Jess flow ---------- - -1. ModulusProjection - -- amplitudes(reciprocical) = fft(image) - -- amplitude_abs = abs(amplitudes) - ---- pcdi (modifies amplitude_abs) - ---- if update coherence(pcdi trigger) - -------- do partial coherence on (2*amplitudes_abs_previous - amplitudes_abs, data_abs) - -------- cut out center of the above two arrays of the roi size - -------- if normalize amplitudes_abs = sqrt( amplitudes_abs^2 / sum(amplitudes_abs^2) * sum(data_abs^2) ) - -------- if symmetrize_data both arrays a = center(a) + center(a.flip(flip(flip())))) - -------- if not use previous, coherence = do Lucy(amplitudes_abs^2, data_abs^2) - -------- find max element in coherence and center it - -------- cut a kernel size subarray centered, and return it as coherence (kernel hardcoded to 17 in each dim) - -------- if symmetrize_kernal kernal+flipdim(flipdim(flipdim(kernal,1),2),3)/norm_after*norm_before - -------- normalize coherence abs(coh)/norm_coh - ---- amplitudes_abs = sqrt(fftconvolve(abs(amplitudes), coherence) - -- apply ratio amplitudes * amplitudes_abs/data_abs - -- image = ifft(amplitudes) - -- record error = sum(abs(amplitudes)-abs(data))^2/sum(data)^2 - - -2. - -- if trigger - ---- shrink_wrap - ---- update support - -- if trigger - ---- phase constraint - -3.1.ER - -- image * support - - -2.2 HIO - -- image * support + (1 - support) * (prev_image - image * beta)) - - - - - - - - - diff --git a/flow_r b/flow_r deleted file mode 100644 index 6e7cb34..0000000 --- a/flow_r +++ /dev/null @@ -1,53 +0,0 @@ -Ross's flow ------------ - -1. ER - -- amplitudes(reciprocical) = ifft(image) - -- normalize: amplitudes * data array size - -- record error = sum(abs(amplitudes)-abs(data))^2/sum(data)^2 - -- apply amplitude threshold on the amplitudes - -- image = fft(amplitudes) - -- normalize: image / data array size - -- image_norm_prev - -- image * support - -- image_norm - -- image * sqrt(image_norm/image_norm_prev) - - -2. Phase Constrained HIO - -- amplitudes(reciprocical) = ifft(image) - -- normalize: amplitudes * data array size - -- record error = sum(abs(amplitudes)-abs(data))^2/sum(data)^2 - -- apply amplitude threshold on the amplitudes - -- image = fft(amplitudes) - -- normalize: image / data array size - -- image_norm_prev - -- phases - -- phase condition: (phase < params->GetPhaseMin()) || (phase > params->GetPhaseMax()) || (support->GetSupportArray() == 0) - -- for phase condition and support replace image with prev_image - image * beta - -- image_norm - -- image * sqrt(image_norm/image_norm_prev) - - diff --git a/how_to_run b/how_to_run new file mode 100644 index 0000000..f469cd3 --- /dev/null +++ b/how_to_run @@ -0,0 +1,49 @@ +To visualize reconstructed image file from data the following steps are needed: + +I. Beamline specific. We support solution for the APS beamline 34-ID. + 1. From data collected by detector create 3D tif file. + 2. From experiment spec record create configuration file for processing display. +II. For any beamline. + 3. Create configuration files. + 4. Prepare the 3D tif data file for reconstruction according to configuration. + This will remove aliens, add padding, etc. + 5. Run reconstruction according to configuration. + 6. Run display processing according to configuration. + +How to run using command line scripts: +-------------------------------------- +1. to run everything: +source bin/everything.sh + +2. to run separately: +python bin/prepare_34id.py +python bin/run_data.py +source bin/run_rec.sh +python bin/run_disp.py + + - specifies processor/library to run reconstruction on ('cpu', 'opencl', 'cuda') + - arbitrary string that defines the experiment, it will be combine with the to derive experiment id + - a range of scans to process, defined by first-last + - a directory containing configuration files (config, config_data, config_rec, config_disp) + - a directory where the experiment files and results are located, derived from configured working directory and experiment id + +example: +1. running all pieces +source bin/everything.sh opencl B 290-290 conf/last + +2. running one by one (when envirenment is set) +python bin/prepare_34id.py B 290-290 conf +python bin/run_data.py test/B_290-290 +python bin/run_rec.py opencl test/B_290-290 +python bin/run_disp.py test/B_290-290 + +3. running reconstruction with alternate configuration: +Note: the configuration should be in conf directory and have name _config_rec +python bin/run_rec.py opencl test/B_290-290 --rec_id + +4. running display with alternate results: +python bin/run_disp.py test/B_290-290 --rec_id + +How to run using GUI: +--------------------- +1. source bin/cdi_window.sh diff --git a/include/algorithm.hpp b/include/algorithm.hpp deleted file mode 100644 index f6adcfb..0000000 --- a/include/algorithm.hpp +++ /dev/null @@ -1,36 +0,0 @@ - -#ifndef algorithm_hpp -#define algorithm_hpp - -class Reconstruction; -#include "arrayfire.h" - -using namespace af; - -class Algorithm -{ -public: - // Using strategy pattern. The RunAlgorihm calls sequence of methods - // overridden by concrete classes - void RunAlgorithm(Reconstruction * reconstruction); - - // the following methods are overridden in concrete algorithms - // they are part of strategy pattern - af::array ModulusProjection(); - virtual void ModulusConstrain(af::array); -}; - -class Hio : public Algorithm -{ -public: - void ModulusConstrain(af::array); -}; - -class Er : public Algorithm -{ -public: - void ModulusConstrain(af::array); -}; - - -#endif /*algorithm_hpp */ diff --git a/include/bridge.hpp b/include/bridge.hpp deleted file mode 100644 index 42145bc..0000000 --- a/include/bridge.hpp +++ /dev/null @@ -1,33 +0,0 @@ -// -// bridge.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#ifndef bridge_hpp -#define bridge_hpp - -#include "vector" -#include "string" -#include "common.h" - -class Bridge -{ -public: -void StartCalcWithGuess(std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config); - -void StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config); -void StartCalcMultiple(std::vector data_buffer_r, std::vector dim, std::string const & config, int nu_threads); - -std::vector GetSupportV(); -std::vector GetCoherenceV(); -std::vector GetImageR(); -std::vector GetImageI(); -std::vector GetErrors(); - -}; - - -#endif /* bridge_hpp */ diff --git a/include/common.h b/include/common.h deleted file mode 100644 index 055b918..0000000 --- a/include/common.h +++ /dev/null @@ -1,49 +0,0 @@ -// -// Constants.h -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/8/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#ifndef common_h -#define common_h - -// defines the type of the data; can be float or double. The def_type will get replaced when running initializing script. -typedef def_type d_type; -const int nD = 3; - - -// a pair that defines an algorithm, and an iteration at which the algorithm is replaced by another or the process ends -struct Alg_switch -{ - int algorithm_id; - int iterations; - Alg_switch(int alg, int iter) - { - algorithm_id = alg; - iterations = iter; - } -}; -typedef struct Alg_switch alg_switch; - -const int ALGORITHM_ER = 1; -const int ALGORITHM_HIO = 2; -const int ALGORITHM_ER_NORM = 3; -const int ALGORITHM_HIO_NORM = 4; -const int ALGORITHM_LUCY = 13; -const int ALGORITHM_LUCY_PREV = 14; -const int ALGORITHM_GAUSS = 15; -const int ALGORITHM_BOX = 16; -const int ALGORITHM_PERCENT = 17; -const int ALGORITHM_GAUSS_FILL = 18; -const int ALGORITHM_GAUSS_PERCENT = 19; -const int ALGORITHM_PERCENT_AUTO = 20; -const int ALGORITHM_GAUSS_MINAREA = 21; - -const int REGULARIZED_AMPLITUDE_NONE = 0; -const int REGULARIZED_AMPLITUDE_GAUSS = 1; -const int REGULARIZED_AMPLITUDE_POISSON = 2; -const int REGULARIZED_AMPLITUDE_UNIFORM = 3; - -#endif /* common_h */ diff --git a/include/parameters.hpp b/include/parameters.hpp deleted file mode 100644 index 4237369..0000000 --- a/include/parameters.hpp +++ /dev/null @@ -1,98 +0,0 @@ -// -// parameters.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#ifndef parameters_hpp -#define parameters_hpp - -#include "common.h" -#include "vector" -#include "arrayfire.h" - -using namespace af; - -struct Trigger_setting -{ - int start_iteration; - int step_iteration; - int end_iteration; - Trigger_setting(int start, int step, int end) - { - start_iteration = start; - step_iteration = step; - end_iteration = end; - } -}; -typedef struct Trigger_setting trigger_setting; - -class Reconstruction; -class Support; -class PartialCoherence; - -// This class holds parameters defining the reconstruction process. The parameters are set based on configuration file. -// Methods of this class are getters. -class Params -{ -private: - std::vector ParseTriggers(std::string trigger_name); - void BuildAlgorithmMap(); - -public: - // Constructor. Takes in configuration file, parses the configuration and sets the parameters accordingly. - Params(const char* config_file, const dim4 data_dim); - - // returns data type (float/double). Used by python code - std::string GetDataType(); - - // Returns number of all iterations. It is calculated from the "algorithm_sequence" parameter. - int GetNumberIterations(); - - // Returns info for support update. trigger list contains starting iteration, step, and ending iteration - // (if missing, run to the end) - Support * GetSupport(); - - // Returns info for partial coherence. trigger list contains starting iteration, step, and ending iteration - // (if missing, run to the end) - PartialCoherence * GetPartialCoherence(); - - // Returns amplitude threshold. Used by ER and HIO algorithms. - d_type GetAmpThreshold(); - - // Returns true if the ER/HIO algorithms should fill the image if not met amplitude threshold condition with zeros. - // Returns false, if the values should not be modified. - bool IsAmpThresholdFillZeros(); - - // Returns minimum phase value for the HIO processing. - d_type GetPhaseMin(); - - // Returns maximum phase value for the HIO processing. - d_type GetPhaseMax(); - - // Returns beta parameter for the HIO processing. - float GetBeta(); - - // Returns iteration number at which the amplitudes are averaged. - int GetAvgIterations(); - - // Returns iteration number at which the "twin" gets zeroed out. - int GetTwin(); - - // Returns a vector containing algorithm switch sequence. - // Algorithm switch is defined as a pair of two elements, the first defines an algorithm, and the second defines - // iteration at which the algorithm stops and switches to a next algorithm. - std::vector GetAlgSwitches(); - - // Returns a constant indication a scheme for modifying data when calculation ratio in modulus projection - int GetRegularizedAmp(); - - // Returns number of iterations between calling garbage collection. - int GetGC(); - -}; - - -#endif /* parameters_hpp */ diff --git a/include/pcdi.hpp b/include/pcdi.hpp deleted file mode 100644 index 2cda432..0000000 --- a/include/pcdi.hpp +++ /dev/null @@ -1,42 +0,0 @@ - -#ifndef pcdi_hpp -#define pcdi_hpp - -#include "vector" - -//class Reconstruction; -namespace af { - class array; -} - -class PartialCoherence -{ -private: - std::vector roi; - int * kernel; - int crop[6]; - std::vector triggers; - int trigger_index; - int algorithm; - bool normalize; - int iteration_num; - bool clip; - - void DeconvLucy(af::array image, af::array filter, int iter_num); - void OnTrigger(af::array abs_image); - void TuneLucyCoherence(af::array); - int GetTriggerAlgorithm(); - std::vector GetRoi(); - int * GetKernel(); - af::array fftConvolve(af::array arr, af::array kernel); - -public: - PartialCoherence(std::vector roi, int * kernel, std::vector partial_coherence_trigger, int alg, bool pcdi_normalize, int pcdi_iter, bool pcdi_clip); - void Init(af::array data); - void SetPrevious(af::array abs_amplitudes); - std::vector GetTriggers(); - af::array ApplyPartialCoherence(af::array abs_image, int current_iteration); - af::array GetKernelArray(); -}; - -#endif /* pcdi_hpp */ diff --git a/include/support.hpp b/include/support.hpp deleted file mode 100644 index b996365..0000000 --- a/include/support.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// -// parameters.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#ifndef support_hpp -#define support_hpp - -#include "common.h" -#include "vector" -#include "arrayfire.h" - -using namespace af; - -class Support -{ -private: - af::array support_array; - af::array distribution; - std::vector triggers; - int algorithm; - float threshold; - bool threshold_adjust; - int sigma; - int twin; - af::array GaussConvFft(af::array ds_image); - -public: - Support(const dim4 data_dim, std::vector area, float threshold, bool threshold_adjust, int sigma, std::vector support_triggers, int alg); - void Update(const af::array ds_image); - std::vector GetTriggers(); - int GetTriggerAlgorithm(); - int GetSigma(); - float GetThreshold(); - af::array GetSupportArray(bool twin=false); -}; - -#endif /* support_hpp */ diff --git a/init.sh b/init.sh index 8e11b2f..ed7b164 100644 --- a/init.sh +++ b/init.sh @@ -3,27 +3,31 @@ echo -n "enter ArrayFire installation directory > " read af_dir -#af_dir='/local/af' AF='AF_DIR' -sed -i 's?'$AF'?'$af_dir'?g' src_py/cyth/*.pyx +sed -i 's?'$AF'?'$af_dir'?g' reccdi/src_py/cyth/*.pyx echo -n "enter LibConfig installation directory > " read lc_dir -#lc_dir=/local/bfrosik/libconfig-1.5 LC='LC_DIR' -sed -i 's?'$LC'?'$lc_dir'?g' src_py/cyth/*.pyx +sed -i 's?'$LC'?'$lc_dir'?g' reccdi/src_py/cyth/*.pyx echo -n "enter cuda installation directory > " read cuda_dir -export LD_LIBRARY_PATH=$lc_dir/lib/.libs:/usr/local/lib:$af_dir/lib/:$cuda_dir/lib64:$cuda_dir/nvvm/lib64 +export LD_LIBRARY_PATH=$lc_dir/lib:/usr/local/lib:$af_dir/lib64:$cuda_dir/lib64:$cuda_dir/nvvm/lib64 + +lib_path='LIB_PATH' +sed -i 's?'$lib_path'?'$lc_dir/lib:/usr/local/lib:$af_dir/lib64:$cuda_dir/lib64:$cuda_dir/nvvm/lib64'?g' bin/everything.sh +sed -i 's?'$lib_path'?'$lc_dir/lib:/usr/local/lib:$af_dir/lib64:$cuda_dir/lib64:$cuda_dir/nvvm/lib64'?g' bin/run_rec.sh +sed -i 's?'$lib_path'?'$lc_dir/lib:/usr/local/lib:$af_dir/lib64:$cuda_dir/lib64:$cuda_dir/nvvm/lib64'?g' bin/cdi_window.sh echo -n "enter data type (float/double) > " read data_type def='def_type' -sed -i 's?'$def'?'$data_type'?g' src_py/cyth/*.pyx -sed -i 's?'$def'?'$data_type'?g' include/common.h +sed -i 's?'$def'?'$data_type'?g' reccdi/src_py/cyth/*.pyx +sed -i 's?'$def'?'$data_type'?g' reccdi/include/common.h python setup.py build_ext --inplace +python setup.py install diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..650e1f3 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,36 @@ +package: + name: reccdi + version: "v1.1" + +source: + path: . + +build: + script_env: + - LD_LIBRARY_PATH + number: 0 + noarch: generic + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + + host: + - python + - cython + + run: + - tifffile + - pylibconfig2 + - gputil + - xrayutilities + - traits + - mayavi + + +about: + home: https://github.com/advancedPhotonSource/cdi + license: BSD + license_file: LICENSE + summary: Implement and parallelize genetic algorithms and phase retrieval methods for Bragg CDI techniques. diff --git a/src_py/__init__.py b/reccdi/__init__.py similarity index 100% rename from src_py/__init__.py rename to reccdi/__init__.py diff --git a/reccdi/include/bridge.hpp b/reccdi/include/bridge.hpp new file mode 100644 index 0000000..b045e1a --- /dev/null +++ b/reccdi/include/bridge.hpp @@ -0,0 +1,46 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef bridge_hpp +#define bridge_hpp + +#include "vector" +#include "string" +#include "common.h" + +class Manager; + +class Bridge +{ +private: + Manager *mgr; + +public: + Bridge(); + + void StartCalcWithGuess(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config); + + void StartCalcWithGuessSupport(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_buffer, std::vector dim, const std::string & config); + + void StartCalcWithGuessSupportCoh(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_buffer, std::vector dim, std::vector coh_buffer, std::vector coh_dim, const std::string & config); + + void StartCalc(int device, std::vector data_buffer_r, std::vector dim, std::string const & config); + + std::vector GetReciprocalR(); + std::vector GetReciprocalI(); + std::vector GetSupportV(); + std::vector GetCoherenceV(); + std::vector GetImageR(); + std::vector GetImageI(); + std::vector GetErrors(); + std::vector GetFlowV(); + std::vector GetIterFlowV(); + + void Cleanup(); +}; + + +#endif /* bridge_hpp */ diff --git a/reccdi/include/common.h b/reccdi/include/common.h new file mode 100644 index 0000000..c0fec3d --- /dev/null +++ b/reccdi/include/common.h @@ -0,0 +1,94 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef common_h +#define common_h + +// defines the type of the data; can be float or double. The double will get replaced when running initializing script. +typedef double d_type; + +const int nD = 3; + + +// a pair that defines an algorithm, and an iteration at which the algorithm is replaced by another or the process ends +struct Alg_switch +{ + int algorithm_id; + int iterations; + Alg_switch(int alg, int iter) + { + algorithm_id = alg; + iterations = iter; + } +}; +typedef struct Alg_switch alg_switch; + +const int ALGORITHM_ER = 2; +const int ALGORITHM_HIO = 3; + +const int ALGORITHM_LUCY = 13; +const int ALGORITHM_LUCY_PREV = 14; +const int ALGORITHM_GAUSS = 15; +const int ALGORITHM_BOX = 16; +const int ALGORITHM_PERCENT = 17; +const int ALGORITHM_GAUSS_FILL = 18; +const int ALGORITHM_GAUSS_PERCENT = 19; +const int ALGORITHM_PERCENT_AUTO = 20; +const int ALGORITHM_GAUSS_MINAREA = 21; + +const int REGULARIZED_AMPLITUDE_NONE = 0; +const int REGULARIZED_AMPLITUDE_GAUSS = 1; +const int REGULARIZED_AMPLITUDE_POISSON = 2; +const int REGULARIZED_AMPLITUDE_UNIFORM = 3; + +const int NOT_TRIGGER = 0; +const int FIRST_RUN_ONLY = 1; +const int FOR_ALL_RUNS = 2; +const int MODIFIED_AFTER_FIRST = 3; +const int CUSTOM = 4; + +typedef struct flow_item_def { char* item_name; + int type; + char* func_name; + flow_item_def(char* item, int t, char* func) : item_name(item), type(t), func_name(func){} + } flow_item_def; + +// The table below defines general iteration sequence. Each item defines a function with the reconstruction algorithm, +// and is called flow_item_def. It is defined by a name, type, and function name. The name is an identifier, also for +// trigger it should match the name in configuration file. The type defines what kind of instruction it is. +// The NO_TRIGGER indicates that the corresponding function will be executed for all iterations. +// FIRST_RUN_ONLY means that the configured triggers apply only if it is an initial run. +// MODIFIED_AFTER_FIRST means that after initial run the trigger gets modified. The start trigger will be equal to step. +// FOR_ALL_RUNS means that the same triggers apply for all runs. +// CUSTOM means that parsing of the triggers is different from the generic (as described in config_rec file) and it +// requires extra code. +// The flow_item_def func_name is a name of the function in worker that implements the flow item. +// +// To add a new trigger/function do the following: +// 1. Insert a new definition for the flow_item in the correct order in the flow_def array. +// 2. Update the flow_seq_len below. +// 3. Add the new function to the worker.hh and worker.cc, and add the pair (func_name, fp) to the flow_ptr_map in worker.cpp. +const flow_item_def flow_def[] = { + flow_item_def((char *)"next", NOT_TRIGGER, (char *)"NextIter"), + flow_item_def((char *)"resolution_trigger", FIRST_RUN_ONLY, (char *)"ResolutionTrigger"), + flow_item_def((char *)"amp_support_trigger", MODIFIED_AFTER_FIRST, (char *)"SupportTrigger"), + flow_item_def((char *)"phase_support_trigger", FIRST_RUN_ONLY, (char *)"PhaseTrigger"), + flow_item_def((char *)"to_reciprocal_space", NOT_TRIGGER, (char *)"ToReciprocal"), + flow_item_def((char *)"pcdi_trigger", MODIFIED_AFTER_FIRST, (char *)"PcdiTrigger"), + flow_item_def((char *)"pcdi", CUSTOM, (char *)"Pcdi"), + flow_item_def((char *)"no_pcdi", CUSTOM, (char *)"NoPcdi"), + flow_item_def((char *)"garbage_trigger", FOR_ALL_RUNS, (char *)"Gc"), + flow_item_def((char *)"set_prev_pcdi_trigger", CUSTOM, (char *)"SetPcdiPrevious"), + flow_item_def((char *)"to_direct_space", NOT_TRIGGER, (char *)"ToDirect"), + flow_item_def((char *)"algorithm", CUSTOM, (char *)"RunAlg"), + flow_item_def((char *)"twin_trigger", FIRST_RUN_ONLY, (char *)"Twin"), + flow_item_def((char *)"average_trigger", FOR_ALL_RUNS, (char *)"Average"), + flow_item_def((char *)"progress_trigger", FOR_ALL_RUNS, (char *)"Prog") +}; + +const int flow_seq_len = 15; + +#endif /* common_h */ diff --git a/include/manager.hpp b/reccdi/include/manager.hpp similarity index 59% rename from include/manager.hpp rename to reccdi/include/manager.hpp index eae52b7..83d2515 100644 --- a/include/manager.hpp +++ b/reccdi/include/manager.hpp @@ -1,10 +1,8 @@ -// -// manager.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik #ifndef manager_hpp #define manager_hpp @@ -20,21 +18,28 @@ class Manager private: // A worker instance managed by the Manager Reconstruction *rec; + bool good_reconstruction; public: + Manager(); + ~Manager(); // This method starts calculations. The Manager uses workers to perform the calculations. The parameters define // calculations type. // This method takes data, real and imaginary guess for the reconstruction algorithm. The dim parameter conveys the // data and guess dimensions, since the data and guess are passed in a c-like buffer. // The config parameter defines configuration file. - void StartCalc(std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config); + void StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config); + + void StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support, std::vector dim, const std::string & config); + + void StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support, std::vector dim, std::vector coh, std::vector coh_dim, const std::string & config); // This method starts calculations. The Manager uses workers to perform the calculations. The parameters define // calculations type. // This method takes data, for the reconstruction algorithm. To perform the reconstruction the code will generate // the guess parameter. The dim parameter conveys data dimensions, since the data is passed in a c-like buffer. // The config parameter defines configuration file. - void StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config); + void StartCalc(int device, std::vector data_buffer_r, std::vector dim, std::string const & config); // This method starts calculations. The Manager uses workers to perform the calculations. The parameters define // calculations type. @@ -42,7 +47,7 @@ class Manager // This method takes data, for the reconstruction algorithm. To perform the reconstruction the code will generate // the guess parameter. The dim parameter conveys data dimensions, since the data is passed in a c-like buffer. // The config parameter defines configuration file. - void StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config, int nu_threads); + // void StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config); // This method returns calculation results. The returned buffer contains a real part of reconstructed image. std::vector GetImageR(); @@ -59,6 +64,18 @@ class Manager // This method returns final coherence array. std::vector GetCoherenceV(); + // This method returns last amplitudes in reciprocal space. The returned buffer contains the real part. + std::vector GetReciprocalR(); + + // This method returns last amplitudes in reciprocal space. The returned buffer contains the imaginary part. + std::vector GetReciprocalI(); + + // This method returns flow vector, i.e the actins that were used. + std::vector GetFlowV(); + + // This method returns 2D flow array (flow x iterations). + std::vector GetIterFlowV(); + }; diff --git a/reccdi/include/parameters.hpp b/reccdi/include/parameters.hpp new file mode 100644 index 0000000..5c3ca4f --- /dev/null +++ b/reccdi/include/parameters.hpp @@ -0,0 +1,127 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef parameters_hpp +#define parameters_hpp + +#include "string" +#include "common.h" +#include "vector" +#include "map" + +// This class holds parameters defining the reconstruction process. The parameters are set based on configuration file. +// Methods of this class are getters. +class Params +{ +private: + // maps algorithm name to algorithm number + std::map algorithm_id_map; + // vector holding algorithm run sequence, where algorithm run is a pair of algorithm and number of iterations + std::vector alg_switches; + + float beta; + + // support + std::vector support_area; + float support_threshold; + float support_sigma; + int support_alg; + + d_type phase_min; + d_type phase_max; + + //partial coherence + bool is_pcdi; + int pcdi_alg; + std::vector pcdi_roi; + bool pcdi_normalize; + int pcdi_iter; + + // twin + std::vector twin_halves; + + // calculated number of iterations + int number_iterations; + + bool plot_errors; + + bool is_resolution; + + int low_res_iterations; + + float iter_res_sigma_first; + + float iter_res_sigma_last; + + float iter_res_det_first; + + float iter_res_det_last; + + std::vector used_flow_seq; +// int flow[]; + std::vector flow_vec; + + void BuildAlgorithmMap(); + +public: + // Constructor. Takes in configuration file, parses the configuration and sets the parameters accordingly. + Params(const char* config_file, std::vector data_dim, bool first); + ~Params(); + + // returns data type (float/double). Used by python code + std::string GetDataType(); + + // Returns number of all iterations. It is calculated from the "algorithm_sequence" parameter. + int GetNumberIterations(); + + std::vector GetSupportArea(); + float GetSupportThreshold(); + float GetSupportSigma(); + int GetSupportAlg(); + + d_type GetPhaseMin(); + d_type GetPhaseMax(); + + bool IsPcdi(); + int GetPcdiAlgorithm(); + std::vector GetPcdiRoi(); + bool GetPcdiNormalize(); + int GetPcdiIterations(); + + std::vector GetTwinHalves(); + + bool IsResolution(); + int GetLowResolutionIter(); + float GetIterResSigmaFirst(); + float GetIterResSigmaLast(); + float GetIterResDetFirst(); + float GetIterResDetLast(); + + // Returns amplitude threshold. Used by ER and HIO algorithms. + d_type GetAmpThreshold(); + + // Returns true if the ER/HIO algorithms should fill the image if not met amplitude threshold condition with zeros. + // Returns false, if the values should not be modified. + bool IsAmpThresholdFillZeros(); + + // Returns beta parameter for the HIO processing. + float GetBeta(); + + // Returns a vector containing algorithm switch sequence. + // Algorithm switch is defined as a pair of two elements, the first defines an algorithm, and the second defines + // iteration at which the algorithm stops and switches to a next algorithm. + std::vector GetAlgSwitches(); + + // Returns boolean flag indication whether to plot errors in during calculations + bool IsPlotErrors(); + + std::vector GetUsedFlowSeq(); +// int* GetFlowArray(); + std::vector GetFlowArray(); +}; + + +#endif /* parameters_hpp */ diff --git a/reccdi/include/pcdi.hpp b/reccdi/include/pcdi.hpp new file mode 100644 index 0000000..da4d47b --- /dev/null +++ b/reccdi/include/pcdi.hpp @@ -0,0 +1,50 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef pcdi_hpp +#define pcdi_hpp + +#include "vector" +#include "arrayfire.h" + +using namespace af; + +class Params; + +class PartialCoherence +{ +private: + Params * params; + std::vector roi; + int algorithm; + bool normalize; + int iteration_num; + + af::array kernel_array; + af::array roi_amplitudes_prev; + af::array roi_data_abs; + d_type sum_roi_data; + af::dim4 roi_dims; + af::dim4 dims; + + void DeconvLucy(af::array image, af::array filter, int iter_num); + void OnTrigger(af::array abs_image); + void TuneLucyCoherence(af::array); + int GetAlgorithm(); + std::vector GetRoi(); + af::array fftConvolve(af::array arr, af::array kernel); + +public: + PartialCoherence(Params *params, af::array coherence_array); + ~PartialCoherence(); + void Init(af::array data); + void SetPrevious(af::array abs_amplitudes); + af::array ApplyPartialCoherence(af::array abs_image); + void UpdatePartialCoherence(af::array abs_image); + af::array GetKernelArray(); +}; + +#endif /* pcdi_hpp */ diff --git a/reccdi/include/resolution.hpp b/reccdi/include/resolution.hpp new file mode 100644 index 0000000..773b4cc --- /dev/null +++ b/reccdi/include/resolution.hpp @@ -0,0 +1,37 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef resolution_hpp +#define resolution_hpp + +#include "vector" +#include "common.h" + +class Params; +namespace af { + class array; +} + +// This class encapsulates low resolution data operations. +class Resolution +{ +private: + std::vector dets; + std::vector sigmas; + +public: + Resolution(Params *params); + + // Needs destructor to free allocated memory. + ~Resolution(); + + // Returns resolution based on iteration + af::array GetIterData(int iter, af::array data); + float GetIterSigma(int iter); +}; + + +#endif /* resolution_hpp */ diff --git a/include/state.hpp b/reccdi/include/state.hpp similarity index 69% rename from include/state.hpp rename to reccdi/include/state.hpp index 1f1edf2..92f52ab 100644 --- a/include/state.hpp +++ b/reccdi/include/state.hpp @@ -1,20 +1,19 @@ -// -// state.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik #ifndef state_hpp #define state_hpp #include "vector" +#include "map" #include "common.h" class Params; class Reconstruction; -class Algorithm; + namespace af { class array; } @@ -23,7 +22,21 @@ namespace af { class State { private: - void MapAlgorithmObject(int alg_id); + // a reference to params object + Params *params; + + // current iteration + int current_iter; + // number of configured iterations for reconstruction + int total_iter_num; + + // The vector of errors indexed by iteration + std::vector errors; + + // current algorithm id + int current_alg; + // current index of index switches vector + int alg_switch_index; public: // Constructor. Takes pointer to the Param object. Uses the Param object to set the initial values. @@ -49,20 +62,9 @@ class State int GetCurrentIteration(); // Returns an algorithm that should be run in a current state (i.e. iteration). - Algorithm * GetCurrentAlg(); - - // Returns true if the current state should include support update. - bool IsUpdateSupport(); - - // Returns true if the current state should include partial coherence update. - bool IsUpdatePartialCoherence(); +// Algorithm * GetCurrentAlg(); + int GetCurrentAlg(); - // Returns true if the current state should apply twin. - bool IsApplyTwin(); - - // Returns the difference of current iteration and iteration of averaging start - bool IsAveragingIteration(); - // Stores the error void RecordError(d_type error); diff --git a/reccdi/include/support.hpp b/reccdi/include/support.hpp new file mode 100644 index 0000000..c86a94e --- /dev/null +++ b/reccdi/include/support.hpp @@ -0,0 +1,42 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#ifndef support_hpp +#define support_hpp + +#include "common.h" +#include "arrayfire.h" + +using namespace af; + +class Params; + +class Support +{ +private: + Params * params; + af::array distribution; + float threshold; + float sigma; + int algorithm; + int update_iter; + d_type last_sigma; + af::array support_array; + af::array init_support_array; + af::array GaussConvFft(af::array ds_image); + af::array GetDistribution(const af::dim4 data_dim, d_type sigma); + +public: + Support(const af::dim4 data_dim, Params *params, af::array support_array); + ~Support(); + void UpdateAmp(const af::array ds_image, d_type sig, int iter); + void UpdatePhase(const af::array ds_image, int iter); + int GetTriggerAlgorithm(); + float GetThreshold(); + af::array GetSupportArray(); +}; + +#endif /* support_hpp */ diff --git a/include/util.hpp b/reccdi/include/util.hpp similarity index 88% rename from include/util.hpp rename to reccdi/include/util.hpp index 61debac..83048a5 100644 --- a/include/util.hpp +++ b/reccdi/include/util.hpp @@ -1,11 +1,19 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + #ifndef util_hpp #define util_hpp -#include "arrayfire.h" #include "common.h" +#include "vector" +#include "string" namespace af { class array; + class dim4; } class Utils @@ -55,14 +63,16 @@ class Utils static af::array CenterMax(af::array arr); static void GetMaxIndices(af::array arr, int* indices); - static af::array ReverseGaussDistribution(const af::dim4, d_type *, int); static af::array GaussDistribution(const af::dim4, d_type *, int); - + // pads symmetrically around array arr to the size on new_dims with the constant value pad static af::array PadAround(af::array arr, af::dim4 new_dims, d_type pad); static af::array PadAround(af::array arr, af::dim4 new_dims, int pad); static af::array GetRatio(af::array, af::array ); + static bool IsNullArray(af::array); + static std::string GetFullFilename(const char * dir, const char * filename); + static std::vector Linspace(int iter, float start_val, float end_val); }; #endif /* util_hpp */ diff --git a/include/worker.hpp b/reccdi/include/worker.hpp similarity index 64% rename from include/worker.hpp rename to reccdi/include/worker.hpp index 8bd3da2..5b49e8e 100644 --- a/include/worker.hpp +++ b/reccdi/include/worker.hpp @@ -1,10 +1,8 @@ -// -// worker.hpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/12/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik #ifndef worker_hpp #define worker_hpp @@ -12,14 +10,14 @@ #include "stdio.h" #include "vector" #include "map" +#include "arrayfire.h" #include "common.h" - class Params; class State; class Support; class PartialCoherence; -#include "arrayfire.h" +class Resolution; using namespace af; @@ -32,6 +30,8 @@ using namespace af; class Reconstruction { +typedef void (Reconstruction::*fp)(void); + private: // Params object constructed by the Reconstruction class @@ -42,6 +42,30 @@ class Reconstruction Support *support; // A reference to PartialCoherence PartialCoherence *partialCoherence; + // A reference to Resolution + Resolution *resolution; + + af::array data; // this is abs + af::array iter_data; // if low resolution is used, data will differ in iterations + d_type sig; + d_type current_error; + int num_points; + d_type norm_data; + int current_iteration; + af::array ds_image; + af::array ds_image_raw; + af::array rs_amplitudes; + int aver_iter; + std::vector aver_v; + std::vector support_vector; + std::vector coherence_vector; + std::vector > iter_flow; + + // mapping of algorithm id to an Algorithm method pointer + std::map algorithm_map; + + // Creates map + void CreateAlgorithmMap(); // This method returns sum of squares of all elements in the array double GetNorm(af::array arr); @@ -49,9 +73,6 @@ class Reconstruction // This method calculates ratio of amplitudes and correction arrays replacing zero divider with 1. af::array GetRatio(af::array ar, af::array correction); - // Averages amplitudes - void Average(); - // vectorize support array at the end of iterations void VectorizeSupport(); @@ -59,12 +80,35 @@ class Reconstruction void VectorizeCoherence(); d_type CalculateError(); - + + void Progress(); + void NextIter(); + void ResolutionTrigger(); + void SupportTrigger(); + void PhaseTrigger(); + void ToReciprocal(); + void PcdiTrigger(); + void Pcdi(); + void NoPcdi(); + void Gc(); + void SetPcdiPrevious(); + void ToDirect(); + void Twin(); + void Average(); + + // Runs one iteration of ER algorithm. + void ModulusConstrainEr(); + + // Runs one iteration of HIO algorithm. + void ModulusConstrainHio(); + public: // The class constructor takes data array, an image guess array in reciprocal space, and configuration file. The image guess // is typically generated as an complex random array. This image can be also the best outcome of previous calculations. The - // data is saved and is used for processing. Configuration file is used to construct the Param object. - Reconstruction(af::array data, af::array guess, const char* config); + // data is saved and is used for processing. + Reconstruction(af::array data, af::array guess, Params* params, af::array support_array, af::array coherence_array); + + ~Reconstruction(); // This initializes the object. It must be called after object is created. // 1. it calculates and sets norm of the data @@ -78,26 +122,19 @@ class Reconstruction // First it calls Next() on State, which determines which algorithm should be run in this state. It also determines whether the // algorithms should be modified by applying convolution or updating support. This method returns false if all iterations have // been completed (i.e. the code reached last state), and true otherwise. Typically this method will be run in a while loop. - void Iterate(); - - int GetCurrentIteration(); - - // This code is common for ER and HIO algorithms. - af::array ModulusProjection(); - - // Runs one iteration of ER algorithm. - void ModulusConstrainEr(af::array); - - // Runs one iteration of HIO algorithm. - void ModulusConstrainHio(af::array); + int Iterate(); af::array GetImage(); + af::array GetSupportArray(); + af::array GetCoherenceArray(); std::vector GetErrors(); std::vector GetSupportVector(); std::vector GetCoherenceVector(); std::vector GetCoherenceVectorR(); std::vector GetCoherenceVectorI(); + std::vector GetFlowVector(); + std::vector GetIterFlowVector(); + af::array GetReciprocal(); }; - #endif /* worker_hpp */ diff --git a/reccdi/src_cpp/bridge.cpp b/reccdi/src_cpp/bridge.cpp new file mode 100644 index 0000000..c9bb9ed --- /dev/null +++ b/reccdi/src_cpp/bridge.cpp @@ -0,0 +1,101 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "sstream" +#include "bridge.hpp" +#include "manager.hpp" +#include "cstdio" + + +Bridge::Bridge() +{ + mgr = new Manager(); +} + +void Bridge::StartCalcWithGuess(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config) +{ + std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); + std::vector guess_i(guess_buffer_i.begin(), guess_buffer_i.end()); + std::vector guess_r(guess_buffer_r.begin(), guess_buffer_r.end()); + mgr->StartCalc(device, data_r, guess_r, guess_i, dim, config); +} + +void Bridge::StartCalcWithGuessSupport(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_buffer, std::vector dim, const std::string & config) +{ + std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); + std::vector guess_i(guess_buffer_i.begin(), guess_buffer_i.end()); + std::vector guess_r(guess_buffer_r.begin(), guess_buffer_r.end()); + std::vector support(support_buffer.begin(), support_buffer.end()); + mgr->StartCalc(device, data_r, guess_r, guess_i, support, dim, config); +} + +void Bridge::StartCalcWithGuessSupportCoh(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_buffer, std::vector dim, std::vector coh_buffer, std::vector coh_dim, const std::string & config) +{ + std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); + std::vector guess_i(guess_buffer_i.begin(), guess_buffer_i.end()); + std::vector guess_r(guess_buffer_r.begin(), guess_buffer_r.end()); + std::vector support(support_buffer.begin(), support_buffer.end()); + std::vector coh(coh_buffer.begin(), coh_buffer.end()); + mgr->StartCalc(device, data_r, guess_r, guess_i, support, dim, coh, coh_dim, config); +} + +void Bridge::StartCalc(int device, std::vector data_buffer_r, std::vector dim, std::string const & config) +{ + std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); + mgr->StartCalc(device, data_r, dim, config); +} + +std::vector Bridge::GetImageR() +{ + return mgr->GetImageR(); +} + +std::vector Bridge::GetImageI() +{ + return mgr->GetImageI(); +} + +std::vector Bridge::GetErrors() +{ + return mgr->GetErrors(); +} + +std::vector Bridge::GetSupportV() +{ + return mgr->GetSupportV(); +} + +std::vector Bridge::GetCoherenceV() +{ + return mgr->GetCoherenceV(); +} + +std::vector Bridge::GetReciprocalR() +{ + return mgr->GetReciprocalR(); +} + +std::vector Bridge::GetReciprocalI() +{ + return mgr->GetReciprocalI(); +} + +std::vector Bridge::GetFlowV() +{ + return mgr->GetFlowV(); +} + +std::vector Bridge::GetIterFlowV() +{ + return mgr->GetIterFlowV(); +} + +void Bridge::Cleanup() +{ + delete mgr; +} + + diff --git a/reccdi/src_cpp/manager.cpp b/reccdi/src_cpp/manager.cpp new file mode 100644 index 0000000..990df5e --- /dev/null +++ b/reccdi/src_cpp/manager.cpp @@ -0,0 +1,294 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "stdexcept" +#include "typeinfo" +#include "arrayfire.h" +#include "worker.hpp" +#include "manager.hpp" +#include "util.hpp" +#include "parameters.hpp" +#include "common.h" +#include "unistd.h" + +using namespace af; + +Manager::Manager() +{ + good_reconstruction = true; +} + + +Manager::~Manager() +{ + delete rec; +} + +void Manager::StartCalc(int device, std::vector data_buffer_r, std::vector dim, std::string const & config) +{ + bool first = true; + Params * params = new Params(config.c_str(), dim, first); + + if (device >= 0) + { + try{ + setDevice(device); + } + catch (...) + {// leave to the os to assign device + printf("can't select gpu %d\n", device); + } + info(); + } + + dim4 af_dims = Utils::Int2Dim4(dim); + af::array real_d(af_dims, &data_buffer_r[0]); + //saving abs(data) + af::array data = abs(real_d); + + af::array guess; + af::randomEngine r(AF_RANDOM_ENGINE_MERSENNE, (uint)(getpid() * 100)); + d_type test1 = 0; + double test2 = 0; + if (typeid(test1) == typeid(test2)) + { + guess = randu(data.dims(), c64, r); + } + else + { + guess = randu(data.dims(), c32, r); + } + af::array null_array = array(); + + rec = new Reconstruction(data, guess, params, null_array, null_array); + rec->Init(); + printf("initialized\n"); + + timer::start(); + int ret = rec->Iterate(); + if (ret > 0) + { + good_reconstruction = false; + } + else + { + printf("iterate function took %g seconds\n", timer::stop()); + } +} + +void Manager::StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config) +{ + bool first = false; + Params * params = new Params(config.c_str(), dim, first); + + if (device >= 0) + { + try{ + setDevice(device); + } + catch (...) + {// leave to the os to assign device + printf("can't select gpu %d, can be not updated cache, try restart\n", device); + good_reconstruction = false; + return; + } + info(); + } + + dim4 af_dims = Utils::Int2Dim4(dim); + af::array real_d(af_dims, &data_buffer_r[0]); + //saving abs(data) + af::array data = abs(real_d); + + af::array real_g(af_dims, &guess_buffer_r[0]); + af::array imag_g(af_dims, &guess_buffer_i[0]); + af::array guess = complex(real_g, imag_g); + + af::array null_array = array(); + + rec = new Reconstruction(data, guess, params, null_array, null_array); + rec->Init(); + printf("initialized\n"); + + timer::start(); + int ret = rec->Iterate(); + if (ret > 0) + { + good_reconstruction = false; + } + else + { + printf("iterate function took %g seconds\n", timer::stop()); + } +} + +void Manager::StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_vector, std::vector dim, const std::string & config) +{ + bool first = false; + Params * params = new Params(config.c_str(), dim, first); + + if (device >= 0) + { + try{ + setDevice(device); + } + catch (...) + {// leave to the os to assign device + printf("can't select gpu %d, can be not updated cache, try restart\n", device); + good_reconstruction = false; + return; + } + info(); + } + + dim4 af_dims = Utils::Int2Dim4(dim); + af::array real_d(af_dims, &data_buffer_r[0]); + //saving abs(data) + af::array data = abs(real_d); + + af::array real_g(af_dims, &guess_buffer_r[0]); + af::array imag_g(af_dims, &guess_buffer_i[0]); + af::array guess = complex(real_g, imag_g); + af::array support_a(af_dims, &support_vector[0]); + + af::array null_array = array(); + + rec = new Reconstruction(data, guess, params, support_a, null_array); + rec->Init(); + printf("initialized\n"); + + timer::start(); + int ret = rec->Iterate(); + if (ret > 0) + { + good_reconstruction = false; + } + else + { + printf("iterate function took %g seconds\n", timer::stop()); + } +} + +void Manager::StartCalc(int device, std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector support_vector, std::vector dim, std::vector coh_vector, std::vector coh_dim, const std::string & config) +{ + bool first = false; + Params * params = new Params(config.c_str(), dim, first); + + if (device >= 0) + { + try{ + printf("can't select gpu %d, can be not updated cache, try restart\n", device); + good_reconstruction = false; + return; + } + catch (...) + {// leave to the os to assign device + printf("can't select gpu %d\n", device); + } + info(); + } + + dim4 af_dims = Utils::Int2Dim4(dim); + af::array real_d(af_dims, &data_buffer_r[0]); + //saving abs(data) + af::array data = abs(real_d); + + af::array real_g(af_dims, &guess_buffer_r[0]); + af::array imag_g(af_dims, &guess_buffer_i[0]); + af::array guess = complex(real_g, imag_g).copy(); + af::array support_a(af_dims, &support_vector[0]); + af::array coh_a(Utils::Int2Dim4(coh_dim), &coh_vector[0]); + + rec = new Reconstruction(data, guess, params, support_a, coh_a); + rec->Init(); + printf("initialized\n"); + + timer::start(); + int ret = rec->Iterate(); + if (ret > 0) + { + good_reconstruction = false; + } + else + { + printf("iterate function took %g seconds\n", timer::stop()); + } +} + +std::vector Manager::GetImageR() +{ + af::array image = rec->GetImage(); + + d_type *image_r = real(image).copy().host(); + std::vector v(image_r, image_r + image.elements()); + + delete [] image_r; + return v; +} + +std::vector Manager::GetImageI() +{ + af::array image = rec->GetImage(); + + d_type *image_i = imag(image).copy().host(); + std::vector v(image_i, image_i + image.elements()); + + delete [] image_i; + return v; +} + +std::vector Manager::GetErrors() +{ + if (! good_reconstruction) + { + std::vector errors; + errors.push_back(-1.0); + return errors; + } + return rec->GetErrors(); +} + +std::vector Manager::GetSupportV() +{ + return rec->GetSupportVector(); +} + +std::vector Manager::GetCoherenceV() +{ + return rec->GetCoherenceVector(); +} + +std::vector Manager::GetReciprocalR() +{ + af::array rs_amplitudes = rec->GetReciprocal(); + + d_type *rs_amplitudes_r = real(rs_amplitudes).copy().host(); + std::vector v(rs_amplitudes_r, rs_amplitudes_r + rs_amplitudes.elements()); + + delete [] rs_amplitudes_r; + return v; +} + +std::vector Manager::GetReciprocalI() +{ + af::array rs_amplitudes = rec->GetReciprocal(); + + d_type *rs_amplitudes_i = imag(rs_amplitudes).copy().host(); + std::vector v(rs_amplitudes_i, rs_amplitudes_i + rs_amplitudes.elements()); + + delete [] rs_amplitudes_i; + return v; +} + +std::vector Manager::GetFlowV() +{ + return rec->GetFlowVector(); +} + +std::vector Manager::GetIterFlowV() +{ + return rec->GetIterFlowVector(); +} diff --git a/reccdi/src_cpp/parameters.cpp b/reccdi/src_cpp/parameters.cpp new file mode 100644 index 0000000..4ede813 --- /dev/null +++ b/reccdi/src_cpp/parameters.cpp @@ -0,0 +1,648 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + + +#include "string.h" +#include "iostream" +#include "algorithm" +#include "parameters.hpp" +#include "common.h" +#include "util.hpp" +#include "libconfig.h++" +#include "math.h" + +using namespace libconfig; + + +Params::Params(const char* config_file, std::vector data_dim, bool first) +{ + algorithm_id_map.clear(); + alg_switches.clear(); + beta = 0.9; + support_area.clear(); + support_threshold = 0.1; + support_sigma = 1.0; + support_alg = -1; + phase_min = -atan(1)*2.0; + phase_max = atan(1)*2.0; + is_pcdi = false; + pcdi_alg = 0; + pcdi_roi.clear(); + pcdi_normalize = false; + pcdi_iter = 20; + twin_halves.clear(); + number_iterations = 0; + plot_errors = false; + is_resolution = false; + low_res_iterations = 0; + iter_res_det_first = 1; + + BuildAlgorithmMap(); + + Config cfg; + + // Read the file. If there is an error, report. + try + { + cfg.readFile(config_file); + } + catch(const FileIOException &fioex) + { + printf("config file I/O exception\n"); + } + catch(const ParseException &pex) + { + printf("config file parse exception\n"); + } + + const Setting& root = cfg.getRoot(); + + try { + plot_errors = cfg.lookup("plot_errors"); + } + catch ( const SettingNotFoundException &nfex) + { } + + try { + const Setting &tmp = root["algorithm_sequence"]; + int count = tmp.getLength(); + + int switch_iter = 0; + number_iterations = 0; + int iter = 0; + for (int i = 0; i < count; ++i) + { + int repeat = tmp[i][0]; + for (int k = 0; k < repeat; k++) + { + for (int j = 1; j < tmp[i].getLength(); ++j) + { + iter = tmp[i][j][1]; + switch_iter = switch_iter + iter; + alg_switches.push_back(Alg_switch(algorithm_id_map[tmp[i][j][0]], iter)); + number_iterations += iter; + } + } + } + } + catch ( const SettingNotFoundException &nfex) + { + printf("No 'algorithm_sequence' parameter in configuration file.\n"); + } + // process triggers + // find which triggers are configured, add the index of the flow_seq item to used_flow_seq vwctor if this item + // is used + for (int i = 0; i < flow_seq_len; i++) + { + char *flow_item = flow_def[i].item_name; + int type = flow_def[i].type; + + // no trigger is part of any flow + if (type == NOT_TRIGGER) + { + used_flow_seq.push_back(i); + } + else if (flow_item == (char *)"pcdi_trigger") + { + if (root.exists(flow_item)) + { + const Setting &tmp = root[flow_item]; + int first_pcdi = tmp[0]; + if (first_pcdi < number_iterations) + { + is_pcdi = true; + used_flow_seq.push_back(i); + } + } + } + else + { + if (type == CUSTOM) + { + if (flow_item == (char *)"algorithm") + { + used_flow_seq.push_back(i); + } + else if (flow_item == (char *)"no_pcdi") + { + if (!is_pcdi || first) + { + used_flow_seq.push_back(i); + } + } + else if (is_pcdi) + { + used_flow_seq.push_back(i); + } + } + else if (first) + { + if (type && root.exists(flow_item)) + { + used_flow_seq.push_back(i); + } + } + else + { + if ((type > FIRST_RUN_ONLY) && root.exists(flow_item)) + { + used_flow_seq.push_back(i); + } + } + } + } + + // parse triggers and flow items into flow array; 0 if not executed, 1 if executed + int used_flow_seq_len = used_flow_seq.size(); + int flow[number_iterations * used_flow_seq_len]; + memset(flow, 0, sizeof(flow)); + std::vector pcdi_tr_iter; + + for (int f = 0; f < used_flow_seq_len; f++) + { + int offset = f * number_iterations; + int type = flow_def[used_flow_seq[f]].type; + char *flow_item = flow_def[used_flow_seq[f]].item_name; + + if (type == NOT_TRIGGER) + { + std::fill_n(flow + offset, number_iterations, 1); + } + else if (type == CUSTOM) + { + if (flow_item == (char *)"algorithm") + { + int alg_start = 0; + for (int k=0; k < alg_switches.size(); k++) + { + std::fill_n(flow + offset + alg_start, alg_switches[k].iterations, alg_switches[k].algorithm_id); + alg_start += alg_switches[k].iterations; + } + } + else if (flow_item == (char *)"pcdi") + { + int start_pcdi = first ? pcdi_tr_iter[0] : 0; + for (int i = start_pcdi; i < number_iterations; i ++) + { + flow[offset + i] = 1; + } + } + else if (flow_item == (char *)"no_pcdi") + { + int stop_pcdi = is_pcdi ? pcdi_tr_iter[0] : number_iterations; + for (int i = 0; i < stop_pcdi; i ++) + { + flow[offset + i] = 1; + } + } + else if (flow_item == (char *)"set_prev_pcdi_trigger") + { + for (int i = 0; i < pcdi_tr_iter.size(); i ++) + { + flow[offset + pcdi_tr_iter[i]-1] = 1; + } + } + } + else + { + const Setting &tmp = root[flow_item]; + if (tmp[0].isNumber()) + { + if (tmp.getLength() == 1) + { + int ind = tmp[0]; + if (ind < number_iterations) + { + // the line below handler negative number + ind = (ind + number_iterations) % number_iterations; + flow[offset + ind] = 1; + if (flow_item == (char *)"pcdi_trigger") + { + pcdi_tr_iter.push_back(ind); + } + } + } + else + { + int step = tmp[1]; + int start_iter = tmp[0]; + if (start_iter < number_iterations) + { + start_iter = (start_iter + number_iterations) % number_iterations; + } + + if (!first && (type == MODIFIED_AFTER_FIRST)) + { + start_iter = step; + } + int stop_iter = number_iterations; + if (tmp.getLength() == 3) + { + int conf_stop_iter = tmp[2]; + if (conf_stop_iter < number_iterations) + { + conf_stop_iter = (conf_stop_iter + number_iterations) % number_iterations; + } + stop_iter = std::min(conf_stop_iter, stop_iter); + } + for (int i = start_iter; i < stop_iter; i += step) + { + flow[offset + i] = 1; + if (flow_item == (char *)"pcdi_trigger") + { + pcdi_tr_iter.push_back(i); + } + } + } + } + else + { + for (int j = 0; j < tmp.getLength(); j++) + { + if (tmp[j].getLength() == 1) + { + int ind = tmp[j][0]; + if (ind < number_iterations) + { + // the line below handler negative number + ind = (ind + number_iterations) % number_iterations; + flow[offset + ind] = 1; + if (flow_item == (char *)"pcdi_trigger") + { + pcdi_tr_iter.push_back(ind); + } + } + } + else + { + int step = tmp[j][1]; + int start_iter = tmp[j][0]; + if (start_iter < number_iterations) + { + start_iter = (start_iter + number_iterations) % number_iterations; + } + if (!first && (type = MODIFIED_AFTER_FIRST)) + { + start_iter = step; + } + int stop_iter = number_iterations; + if (tmp[j].getLength() == 3) + { + int conf_stop_iter = tmp[j][2]; + if (conf_stop_iter < number_iterations) + { + conf_stop_iter = (conf_stop_iter + number_iterations) % number_iterations; + } + stop_iter = std::min(conf_stop_iter, stop_iter); + } + for (int i = start_iter; i < stop_iter; i += step) + { + flow[offset + i] = 1; + if (flow_item == (char *)"pcdi_trigger") + { + pcdi_tr_iter.push_back(i); + } + } + } + } + } + + } +// printf(" %s\n",flow_item); +// for (int i=0; i < number_iterations; i++) +// printf(" %i", flow[offset+i]); +// printf("\n"); + } + + std::vector vec(flow, flow + number_iterations * used_flow_seq.size()); + flow_vec = vec; + + if (root.exists("amp_support_trigger")) + { + try { + const Setting &tmp = root["support_area"]; + for (int i = 0; i < tmp.getLength(); ++i) + { + try { + support_area.push_back(tmp[i]); + } + catch ( const SettingTypeException &nfex) + { + float ftmp = tmp[i]; + support_area.push_back(int(ftmp * data_dim[i])); + } + } + } + catch ( const SettingNotFoundException &nfex) + { + printf("No 'support_area' parameter in configuration file.\n"); + } + try { + support_threshold = cfg.lookup("support_threshold"); + } + catch ( const SettingNotFoundException &nfex) + { } + try { + support_sigma = cfg.lookup("support_sigma"); + } + catch ( const SettingNotFoundException &nfex) + { } + try { + support_alg = algorithm_id_map[cfg.lookup("support_type")]; + } + catch ( const SettingNotFoundException &nfex) + { + printf((std::string("No 'support_type' parameter in configuration file.\n")).c_str()); + } + } + + if ((first) && root.exists("phase_support_trigger")) + { + try { + phase_min = cfg.lookup("phase_min"); + } + catch (const SettingNotFoundException &nfex) + { + printf((std::string("No 'phase_min' parameter in configuration file. Set to pi/2.\n")).c_str()); + } + try { + phase_max = cfg.lookup("phase_max"); + } + catch (const SettingNotFoundException &nfex) + { + printf((std::string("No 'phase_max' parameter in configuration file. Set to pi/2.\n")).c_str()); + } + + } + + if (root.exists("pcdi_trigger")) + { + try { + pcdi_alg = algorithm_id_map[cfg.lookup("partial_coherence_type")]; + } + catch ( const SettingNotFoundException &nfex) + { + printf((std::string("No 'partial_coherence_type' parameter in configuration file.\n")).c_str()); + } + try { + const Setting &tmp = root["partial_coherence_roi"]; + for (int i = 0; i < tmp.getLength(); ++i) + { + try { + pcdi_roi.push_back(Utils::GetDimension(tmp[i])); + } + catch ( const SettingTypeException &nfex) + { + float ftmp = tmp[i]; + pcdi_roi.push_back(Utils::GetDimension(int(ftmp * data_dim[i]))); + } + } + } + catch ( const SettingNotFoundException &nfex) + { + printf("No 'partial_coherence_roi' parameter in configuration file.\n"); + } + try { + pcdi_normalize = cfg.lookup("partial_coherence_normalize"); + } + catch ( const SettingNotFoundException &nfex) + { } + try { + pcdi_iter = cfg.lookup("partial_coherence_iteration_num"); + } + catch ( const SettingNotFoundException &nfex) + { + printf((std::string("No 'partial_coherence_iteration_num' parameter in configuration file. Setting to 20.\n")).c_str()); + } + } + + try { + const Setting &tmp = root["twin_halves"]; + for (int i = 0; i < tmp.getLength(); ++i) + { + twin_halves.push_back(tmp[i]); + } + } + catch ( const SettingNotFoundException &nfex) + { + twin_halves.push_back(0); + twin_halves.push_back(0); + //printf((std::string("No 'twin_halves' parameter in configuration file. Setting to (0,0).\n")).c_str()); + } + + if ((first) && root.exists("resolution_trigger")) + { + is_resolution = true; + const Setting &tmp = root["resolution_trigger"]; + try + { + low_res_iterations = tmp[2]; + if (low_res_iterations < 0) + { + low_res_iterations = low_res_iterations + number_iterations; + } + } + catch ( const SettingNotFoundException &nfex) + { + low_res_iterations = number_iterations; + printf((std::string("No 'resolution_trigger' upper bound in configuration file. Setting number to iteration number.\n")).c_str()); + } + try + { + const Setting &tmp = root["iter_res_sigma_range"]; + int size = tmp.getLength(); + if (size > 1) + { + iter_res_sigma_first = tmp[0]; + iter_res_sigma_last = tmp[1]; + } + else + { + iter_res_sigma_first = tmp[0]; + iter_res_sigma_last = support_sigma; + } + } + catch(const SettingNotFoundException &nfex) + { + iter_res_sigma_first = 2.0; + iter_res_sigma_last = support_sigma; + printf("No 'iter_res_sigma_range' parameter in configuration file.Default to 2.0, sigma.\n"); + } + try + { + const Setting &tmp = root["iter_res_det_range"]; + int size = tmp.getLength(); + if (size > 1) + { + iter_res_det_first = tmp[0]; + iter_res_det_last = tmp[1]; + } + else + { + iter_res_det_first = tmp[0]; + iter_res_det_last = 1.0; + } + } + catch(const SettingNotFoundException &nfex) + { + iter_res_det_first = .7; + iter_res_det_last = 1.0; + printf("No 'iter_res_det_range' parameter in configuration file.\Default to 0.7, 1.0.n"); + } + } + + try + { + beta = cfg.lookup("beta"); + } + catch (const SettingNotFoundException &nfex) + { + printf("No 'beta' parameter in configuration file. Setting to .9\n"); + } + +} + + +Params::~Params() +{ + algorithm_id_map.clear(); + alg_switches.clear(); + support_area.clear(); + pcdi_roi.clear(); + used_flow_seq.clear(); + flow_vec.clear(); +} + +void Params::BuildAlgorithmMap() +{ + // hardcoded + algorithm_id_map.insert(std::pair((char *)"ER", ALGORITHM_ER)); + algorithm_id_map.insert(std::pair((char *)"HIO", ALGORITHM_HIO)); + algorithm_id_map.insert(std::pair((char *)"LUCY", ALGORITHM_LUCY)); + algorithm_id_map.insert(std::pair((char *)"LUCY_PREV", ALGORITHM_LUCY_PREV)); + algorithm_id_map.insert(std::pair((char *)"GAUSS", ALGORITHM_GAUSS)); +} + +int Params::GetNumberIterations() +{ + return number_iterations; +} + +float Params::GetBeta() +{ + return beta; +} + +std::vector Params::GetSupportArea() +{ + return support_area; +} + +float Params::GetSupportThreshold() +{ + return support_threshold; +} + +float Params::GetSupportSigma() +{ + return support_sigma; +} + +int Params::GetSupportAlg() +{ + return support_alg; +} + +d_type Params::GetPhaseMin() +{ + return phase_min; +} + +d_type Params::GetPhaseMax() +{ + return phase_max; +} + +bool Params::IsPcdi() +{ + return is_pcdi; +} + +int Params::GetPcdiAlgorithm() +{ + return pcdi_alg; +} + +std::vector Params::GetPcdiRoi() +{ + return pcdi_roi; +} + +bool Params::GetPcdiNormalize() +{ + return pcdi_normalize; +} + +int Params::GetPcdiIterations() +{ + return pcdi_iter; +} + +std::vector Params::GetTwinHalves() +{ + return twin_halves; +} + +std::vector Params::GetAlgSwitches() +{ + return alg_switches; +} + +bool Params::IsPlotErrors() +{ + return plot_errors; +} + +bool Params::IsResolution() +{ + return is_resolution; +} + +int Params::GetLowResolutionIter() +{ + return low_res_iterations; +} + +float Params::GetIterResSigmaFirst() +{ + return iter_res_sigma_first; +} + +float Params::GetIterResSigmaLast() +{ + return iter_res_sigma_last; +} + +float Params::GetIterResDetFirst() +{ + return iter_res_det_first; +} + +float Params::GetIterResDetLast() +{ + return iter_res_det_last; +} + +std::vector Params::GetUsedFlowSeq() +{ + return used_flow_seq; +} + +std::vector Params::GetFlowArray() +{ + //printf("flow vec len %i", flow_vec.size()); + return flow_vec; +} diff --git a/src_cpp/pcdi.cpp b/reccdi/src_cpp/pcdi.cpp similarity index 55% rename from src_cpp/pcdi.cpp rename to reccdi/src_cpp/pcdi.cpp index 239403a..d847916 100644 --- a/src_cpp/pcdi.cpp +++ b/reccdi/src_cpp/pcdi.cpp @@ -1,35 +1,45 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + #include "cstdio" #include "common.h" #include "pcdi.hpp" #include "util.hpp" -#include "arrayfire.h" #include "string" #include "sstream" +#include "parameters.hpp" -using namespace af; - -af::array kernel_array; -af::array roi_amplitudes_prev; -af::array roi_data_abs; -d_type sum_roi_data; -af::dim4 roi_dims; -af::dim4 dims; +PartialCoherence::PartialCoherence(Params *parameters, af::array coherence_array) +{ + params = parameters; + roi= params->GetPcdiRoi(); + algorithm = params->GetPcdiAlgorithm(); + normalize = params->GetPcdiNormalize(); + iteration_num = params->GetPcdiIterations(); + kernel_array = coherence_array; + if (Utils::IsNullArray(coherence_array)) + { + roi_dims = Utils::Int2Dim4(roi); + } + else + { + roi_dims = kernel_array.dims(); + } +} -PartialCoherence::PartialCoherence(std::vector roi_area, int * kernel_area, std::vector partial_coherence_trigger, int alg, bool pcdi_normalize, int pcdi_iter, bool pcdi_clip) +PartialCoherence::~PartialCoherence() { - roi= roi_area; - triggers = partial_coherence_trigger; - trigger_index = 0; - algorithm = alg; - kernel = kernel_area; - normalize = pcdi_normalize; - iteration_num = pcdi_iter; - clip = pcdi_clip; + kernel_array = af::array(); + roi_amplitudes_prev = af::array(); + roi_data_abs = af::array(); + roi.clear(); } void PartialCoherence::Init(af::array data) { - roi_dims = Utils::Int2Dim4(roi); dims = data.dims(); af::array data_centered = af::shift(data, dims[0]/2, dims[1]/2, dims[2]/2, dims[3]/2); @@ -37,8 +47,13 @@ void PartialCoherence::Init(af::array data) if (normalize) { sum_roi_data = sum(pow(roi_data_abs, 2)); - } - kernel_array = constant(0.5, roi_dims); + } + if (Utils::IsNullArray(kernel_array)) + { + d_type c = 0.5; + kernel_array = constant(c, roi_dims); + } + //dim4 kdim = kernel_array.dims(); } void PartialCoherence::SetPrevious(af::array abs_amplitudes) @@ -47,12 +62,7 @@ void PartialCoherence::SetPrevious(af::array abs_amplitudes) roi_amplitudes_prev = Utils::CropCenter(abs_amplitudes_centered, roi_dims).copy(); } -std::vector PartialCoherence::GetTriggers() -{ - return triggers; -} - -int PartialCoherence::GetTriggerAlgorithm() +int PartialCoherence::GetAlgorithm() { return algorithm; } @@ -62,38 +72,48 @@ std::vector PartialCoherence::GetRoi() return roi; } -int * PartialCoherence::GetKernel() -{ - return kernel; -} - -af::array PartialCoherence::ApplyPartialCoherence(af::array abs_amplitudes, int current_iteration) +af::array PartialCoherence::ApplyPartialCoherence(af::array abs_amplitudes) { - // check if trigger is set for this iteration, and if so update coherence - if ((trigger_index < triggers.size()) && (current_iteration == triggers[trigger_index])) - { - af::array abs_amplitudes_centered = shift(abs_amplitudes, dims[0]/2, dims[1]/2, dims[2]/2, dims[3]/2); - af::array roi_abs_amplitudes = Utils::CropCenter(abs_amplitudes_centered, roi_dims).copy(); - - af::array roi_combined_amp = 2*roi_abs_amplitudes - roi_amplitudes_prev; - OnTrigger(roi_combined_amp); // use_2k_1 from matlab program - printf("Updating coherence, current iter %i\n", current_iteration); - trigger_index++; - } - +try{ // apply coherence af::array abs_amplitudes_2 = pow(abs_amplitudes, 2); af::array converged_2 = af::fftConvolve(abs_amplitudes_2, kernel_array); af::array converged = sqrt(converged_2); //af::array converged = sqrt(fftConvolve(pow(abs_amplitudes, 2), kernel_array)); // implemented here, but works different than af::fftConvolve - printf("coherence norm %f\n", sum(pow(abs(kernel_array), 2))); - printf("converged norm %f\n", sum(pow(abs(converged), 2))); + //printf("coherence norm %f\n", sum(pow(abs(kernel_array), 2))); + //printf("converged norm %f\n", sum(pow(abs(converged), 2))); return converged; + +} +catch(af::exception& e) { + fprintf(stderr, "%s\n", e.what()); + printf("ApplyPartialCoherence\n"); + throw; + } +} + +void PartialCoherence::UpdatePartialCoherence(af::array abs_amplitudes) +{ +try{ + af::array abs_amplitudes_centered = shift(abs_amplitudes, dims[0]/2, dims[1]/2, dims[2]/2, dims[3]/2); + af::array roi_abs_amplitudes = Utils::CropCenter(abs_amplitudes_centered, roi_dims).copy(); + + af::array roi_combined_amp = 2*roi_abs_amplitudes - roi_amplitudes_prev; + OnTrigger(roi_combined_amp); // use_2k_1 from matlab program + //printf("Updating coherence\n"); + +} +catch(af::exception& e) { + fprintf(stderr, "%s\n", e.what()); + printf("UpdatePartialCoherence\n"); + throw; + } } void PartialCoherence::OnTrigger(af::array arr) { +try{ // assume calculating coherence across all three dimensions af::array amplitudes = arr; // if symmetrize data, recalculate roi_array and roi_data - not doing it now, since default to false @@ -103,7 +123,6 @@ void PartialCoherence::OnTrigger(af::array arr) d_type sum_ampl = sum(amplitudes_2); d_type ratio = sum_roi_data/sum_ampl; amplitudes = sqrt(amplitudes_2 * ratio); - //amplitudes = sqrt((pow(arr, 2)/sum(pow(arr, 2))) * sum_roi_data); } af::array coherence; @@ -114,11 +133,7 @@ void PartialCoherence::OnTrigger(af::array arr) } else if (algorithm == ALGORITHM_LUCY_PREV) { - // initialize, consider moving to init() - if (trigger_index == 1) - { - coherence = pow(roi_data_abs, 2); - } + // coherence = pow(roi_data_abs, 2); //coherence = DeconvLucy(coherence, pow(roi_data_abs, 2), iteration_num); } else @@ -126,32 +141,45 @@ void PartialCoherence::OnTrigger(af::array arr) //prinf("only LUCY algorithm is currently supported"); } } +catch(af::exception& e) { + fprintf(stderr, "%s\n", e.what()); + printf("OnTrigger\n"); + throw; + } +} af::array PartialCoherence::fftConvolve(af::array arr, af::array kernel) -{ +{ +try{ af::dim4 dims_input = arr.dims(); - af::dim4 dims_kernel = kernel.dims(); - - int dims_fft_padded [nD]; - - for(unsigned int i = 0; i < nD; i++) - { - dim_t d = Utils::GetDimension(dims_input[i] + dims_kernel[i] - 1); - dims_fft_padded[i] = Utils::GetDimension(dims_input[i] + dims_kernel[i] - 1); - } +// af::dim4 dims_kernel = kernel.dims(); +// +// int dims_fft_padded [nD]; +// +// for(unsigned int i = 0; i < nD; i++) +// { +// dim_t d = Utils::GetDimension(dims_input[i] + dims_kernel[i] - 1); +// dims_fft_padded[i] = Utils::GetDimension(dims_input[i] + dims_kernel[i] - 1); +// } d_type pad = 0; af::array kernel_padded = Utils::PadAround(kernel, dims_input, pad); af::array coh_padded = real( Utils::ifft( Utils::fft(arr) * Utils::fft(kernel_padded) ) ); return coh_padded; +} +catch(af::exception& e) { + fprintf(stderr, "%s\n", e.what()); + printf("fftConvolve\n"); + throw; + } } void PartialCoherence::DeconvLucy(af::array amplitudes, af::array data, int iterations) { +try{ // implementation based on Python code: https://github.com/scikit-image/scikit-image/blob/master/skimage/restoration/deconvolution.py - //af::array coherence = constant(0.5, amplitudes.dims()); //set it to the last coherence instead af::array coherence = kernel_array; af::array data_mirror = af::flip(af::flip(af::flip(af::flip(data, 0),1),2),3).copy(); @@ -159,28 +187,27 @@ void PartialCoherence::DeconvLucy(af::array amplitudes, af::array data, int iter for (int i = 0; i < iterations; i++) { af::array convolve = af::fftConvolve(coherence, data); - //af::array convolve = af::convolve3(im_deconv, psf); - convolve(convolve == 0) = 1.0; // added to the algorithm from scikit to prevet division by 0 + convolve(convolve == 0) = 1.0; // added to the algorithm from scikit to prevent division by 0 af::array relative_blurr = amplitudes/convolve; coherence *= af::fftConvolve(relative_blurr, data_mirror); } coherence = real(coherence); - // clip - if (clip) - { - coherence(coherence > 1) = 1; - coherence(coherence < -1) = -1; - } - //coherence = abs(coherence)/sum(abs(coherence)); d_type coh_sum = sum(abs(coherence)); coherence = abs(coherence)/coh_sum; - printf("coherence norm , %f\n", sum(pow(abs(coherence), 2))); + //printf("coherence norm , %f\n", sum(pow(abs(coherence), 2))); kernel_array = coherence; } +catch(af::exception& e) { + fprintf(stderr, "%s\n", e.what()); + printf("DeconvLucy\n"); + throw e; + } +} af::array PartialCoherence::GetKernelArray() { return kernel_array; -} \ No newline at end of file +} + diff --git a/reccdi/src_cpp/resolution.cpp b/reccdi/src_cpp/resolution.cpp new file mode 100644 index 0000000..e524a6d --- /dev/null +++ b/reccdi/src_cpp/resolution.cpp @@ -0,0 +1,50 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "stdio.h" +#include "util.hpp" +#include "common.h" +#include "resolution.hpp" +#include "parameters.hpp" +#include "arrayfire.h" + +using namespace af; + +Resolution::Resolution(Params* param) +{ + int iter = param->GetLowResolutionIter(); + dets = Utils::Linspace(iter, param->GetIterResDetFirst(), param->GetIterResDetLast()); + sigmas = Utils::Linspace(iter, param->GetIterResSigmaFirst(), param->GetIterResSigmaLast()); +} + +Resolution::~Resolution() +{ + dets.clear(); + sigmas.clear(); +} + +float Resolution::GetIterSigma(int iter) +{ + return sigmas[iter]; +} + +af::array Resolution::GetIterData(int iter, af::array data) +{ + int alpha = 1; + d_type *dim_sigmas = new d_type[nD]; + for (int i=0; i(distribution); + distribution = distribution/max_dist; + af::array data_shifted = Utils::ifftshift(data); + af::array masked = distribution*data_shifted; + return Utils::ifftshift(masked); +} + + diff --git a/reccdi/src_cpp/state.cpp b/reccdi/src_cpp/state.cpp new file mode 100644 index 0000000..73cd736 --- /dev/null +++ b/reccdi/src_cpp/state.cpp @@ -0,0 +1,76 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "stdio.h" +#include "vector" +#include "state.hpp" +#include "parameters.hpp" +#include "support.hpp" +#include "pcdi.hpp" +#include "arrayfire.h" + +using namespace af; + + +State::State(Params* parameters) +{ + params = parameters; + current_iter = -1; + total_iter_num = 0; + current_alg = 0; + alg_switch_index = 0; +} + +State::~State() +{ + errors.clear(); +} + +void State::Init() +{ + total_iter_num = params->GetNumberIterations(); + current_alg = params->GetAlgSwitches()[0].algorithm_id; +} + +int State::Next() +{ + if (current_iter++ == total_iter_num - 1) + { + return false; + } + // figure out current alg + if (params->GetAlgSwitches()[alg_switch_index].iterations == current_iter) + // switch to the next algorithm + { + alg_switch_index++; + current_alg = params->GetAlgSwitches()[alg_switch_index].algorithm_id; + } + + return true; +} + +int State::GetCurrentAlg() +{ + return current_alg; +} + +void State::RecordError(d_type error) +{ + errors.push_back(error); + //printf("iter, error %i %fl\n", current_iter, error); +} + +int State::GetCurrentIteration() +{ + return current_iter; +} + +std::vector State::GetErrors() +{ + return errors; +} + + diff --git a/reccdi/src_cpp/support.cpp b/reccdi/src_cpp/support.cpp new file mode 100644 index 0000000..b5bed27 --- /dev/null +++ b/reccdi/src_cpp/support.cpp @@ -0,0 +1,123 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "stdio.h" +#include "vector" +#include "support.hpp" +#include "parameters.hpp" +#include "util.hpp" + +Support::Support(const af::dim4 data_dim, Params *parameters, af::array support) +{ + params = parameters; + threshold = params->GetSupportThreshold(); + sigma = params->GetSupportSigma(); + algorithm = params->GetSupportAlg(); + af::array ones = constant(1, Utils::Int2Dim4(params->GetSupportArea()), u32); + init_support_array = Utils::PadAround(ones, data_dim, 0); + if (Utils::IsNullArray(support)) + { + support_array = init_support_array.copy(); + } + else + { + support_array = support; + } + update_iter = -1; + +/* if (algorithm == ALGORITHM_GAUSS) + { + int alpha = 1; + d_type *sigmas = new d_type[nD]; + for (int i=0; i(convag); + convag = convag/max_convag; + support_array = (convag >= threshold); + + last_sigma = sig; + update_iter = iter; + +} + +void Support::UpdatePhase(const af::array ds_image, int iter) +{ + //printf("phase trigger\n"); + af::array phase = atan2(imag(ds_image), real(ds_image)); + af::array phase_condition = ((phase > params->GetPhaseMin()) && (phase < params->GetPhaseMax())); + support_array *= phase_condition; +// printf("support sum %f\n", sum(support_array)); +} + +int Support::GetTriggerAlgorithm() +{ + return algorithm; +} + +float Support::GetThreshold() +{ + return threshold; +} + +af::array Support::GetSupportArray() +{ + return support_array; +} + +af::array Support::GetDistribution(const af::dim4 data_dim, d_type sigma) +{ + int alpha = 1; + d_type *sigmas = new d_type[nD]; + for (int i=0; i(ds_image_abs); + af::array shifted = Utils::ifftshift(ds_image_abs); + af::array rs_amplitudes = Utils::fft(shifted); + af::array rs_amplitudes_cent = Utils::ifftshift(rs_amplitudes); + + af::array amp_dist = rs_amplitudes_cent * distribution; + shifted = Utils::ifftshift(amp_dist); + af::array convag_compl = Utils::ifft(shifted); + af::array convag = (Utils::ifftshift(convag_compl)); + convag = real(convag); + convag(convag < 0) = 0; + d_type correction = image_sum/sum(convag); + convag *= correction; + return convag; +} + diff --git a/reccdi/src_cpp/util.cpp b/reccdi/src_cpp/util.cpp new file mode 100644 index 0000000..1493492 --- /dev/null +++ b/reccdi/src_cpp/util.cpp @@ -0,0 +1,202 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "util.hpp" +#include "arrayfire.h" +#include "common.h" +#include "algorithm" +#include "cstdio" + +using namespace af; + +int Utils::GetDimension(int dim) +{ + int new_dim = dim; + while (! IsDimCorrect(new_dim)) + { + new_dim++; + } + return new_dim; +} + +bool Utils::IsDimCorrect(int dim) +{ + int sub = dim; + while (sub % 2 == 0) + { + sub = sub/2; + } + while (sub % 3 == 0) + { + sub = sub/3; + } + while (sub % 5 == 0) + { + sub = sub/5; + } + if (sub == 1) + return true; + else + return false; +} + +af::dim4 Utils::Int2Dim4(std::vector dim) +{ + for (int i = dim.size(); i<4; i++) + { + dim.push_back(1); + } + return af::dim4(dim[0], dim[1], dim[2], dim[3]); +} + + +af::array Utils::CropCenter(af::array arr, af::dim4 roi) +{ + dim4 dims = arr.dims(); + int beginning[4]; //dim4 contains four dimensions + int ending[4]; + for (int i=0; i<4; i++) + { + beginning[i] = int((dims[i] - roi[i])/2); + ending[i] = beginning[i] + roi[i]; + } + return arr(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)); +} + +af::array Utils::fftshift(af::array arr) +{ + return af::shift(arr, ceil(arr.dims()[0]/2)-1, ceil(arr.dims()[1]/2)-1, ceil(arr.dims()[2]/2)-1, ceil(arr.dims()[3]/2)-1); +} + +af::array Utils::ifftshift(af::array arr) +{ + return af::shift(arr, ceil(arr.dims()[0]/2), ceil(arr.dims()[1]/2), ceil(arr.dims()[2]/2), ceil(arr.dims()[3]/2)); +} + +af::array Utils::fft(af::array arr) +{ + if (nD == 3) + { + return fft3(arr); + } + else if (nD == 2) + { + return fft2(arr); + } +} + +af::array Utils::ifft(af::array arr) +{ + + if (nD == 3) + { + return ifft3(arr); + } + else if (nD == 2) + { + return ifft2(arr); + } +} + +void Utils::GetMaxIndices(af::array arr, int* indices) +{ + //find indexes of max + d_type *arr_values = abs(arr).host(); + std::vector v(arr_values, arr_values + arr.elements()); + std::vector::iterator result = std::max_element(v.begin(), v.end()); + int max_offset = result - v.begin(); + //printf("maximum value, %i %f\n", max_offset, v[max_offset]); + delete [] arr_values; + indices[0] = max_offset % arr.dims()[0]; + indices[1] = max_offset / arr.dims()[0] % arr.dims()[1]; + indices[2] = max_offset/ (arr.dims()[0] * arr.dims()[1]); + //printf("offset, ind1, ind2 ind3 %i %i %i %i\n", max_offset, indices[0], indices[1], indices[2]); +} + +af::array Utils::GaussDistribution(af::dim4 data_dim, d_type * sgma, int alpha) +{ + // calculate multipliers + int dimension = nD; //this should be changed to determine size from sgma + //initialize first element of the grid, assuming at least one dimension + d_type multiplier = - 0.5 * alpha/pow(sgma[0],2); + af::array exponent = pow( (range(data_dim, 0)-(data_dim[0]-1)/2.0).as(f64) ,2)* multiplier; + af::array grid = exp(exponent); + + //add grid in other dimensions + for (int i = 1; i(grid); + grid = grid/grid_total; + return grid; +} + +af::array Utils::PadAround(af::array arr, af::dim4 new_dims, d_type pad) +{ + //af::array padded = constant(pad, new_dims, (af_dtype) dtype_traits::ctype); + + af::array padded = constant(pad, new_dims, arr.type()); + int beginning[4]; //dim4 contains four dimensions + int ending[4]; + for (int i=0; i<4; i++) + { + beginning[i] = int(new_dims[i]/2 - arr.dims()[i]/2); + ending[i] = beginning[i] + arr.dims()[i]; + } + + padded(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)) = arr; + return padded; +} + +af::array Utils::PadAround(af::array arr, af::dim4 new_dims, int pad) +{ + af::array padded = constant(pad, new_dims, u32); + int beginning[4]; //dim4 contains four dimensions + int ending[4]; + for (int i=0; i<4; i++) + { + beginning[i] = int(new_dims[i]/2 - arr.dims()[i]/2); + ending[i] = beginning[i] + arr.dims()[i]; + } + + padded(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)) = arr; + return padded; +} + +af::array Utils::GetRatio(af::array divident, af::array divisor) +{ + af::array divisor_copy = divisor.copy(); + divisor_copy(divisor == 0) = 1; + return divident/divisor_copy; +} + +bool Utils::IsNullArray(af::array arr) +{ + return (arr.elements() == 0); +} + +std::string Utils::GetFullFilename(const char * dir, const char * filename) +{ + std::string full_filename; + full_filename.append(dir); + full_filename.append("/"); + full_filename.append(filename); + return full_filename; +} + +std::vector Utils::Linspace(int iter, float start_val, float end_val) +{ + std::vector spaced; + for (int i = 0; i < iter; ++i) + { + spaced.push_back(start_val + (end_val - start_val)/iter * i); + } + return spaced; +} diff --git a/reccdi/src_cpp/worker.cpp b/reccdi/src_cpp/worker.cpp new file mode 100644 index 0000000..b042a68 --- /dev/null +++ b/reccdi/src_cpp/worker.cpp @@ -0,0 +1,430 @@ +/*** +Copyright (c) UChicago Argonne, LLC. All rights reserved. +See LICENSE file. +***/ +// Created by Barbara Frosik + +#include "fstream" +#include "unistd.h" +#include "stdio.h" +#include "worker.hpp" +#include "cstdio" +#include "cstdlib" +#include "math.h" +#include "vector" +#include "map" +#include "parameters.hpp" +#include "support.hpp" +#include "pcdi.hpp" +#include "state.hpp" +#include "common.h" +#include "util.hpp" +#include "resolution.hpp" + + +Reconstruction::Reconstruction(af::array image_data, af::array guess, Params* parameters, af::array support_array, af::array coherence_array) +{ + num_points = 0; + norm_data = 0; + current_iteration = 0; + aver_iter = 0; + current_error = 0.0; + data = image_data; + ds_image = guess; + params = parameters; + for (int i = 0; i < params->GetNumberIterations(); i++) + { + std::vector v; + iter_flow.push_back(v); + } + state = new State(params); + support = new Support(data.dims(), params, support_array); + + if (params->IsPcdi()) + { + partialCoherence = new PartialCoherence(params, coherence_array); + } + else + { + partialCoherence = NULL; + } + if (params->IsResolution()) + { + resolution = new Resolution(params); + } + else + { + resolution = NULL; + } +} + +Reconstruction::~Reconstruction() +{ + delete state; + delete support; + if (partialCoherence != NULL) + { + delete partialCoherence; + } + if (resolution != NULL) + { + delete resolution; + } + delete params; + + if (&data == &iter_data) + { + data = af::array(); + } + else + { + data = af::array(); + iter_data = af::array(); + } + ds_image = af::array(); + ds_image_raw = af::array(); + rs_amplitudes = af::array(); + + aver_v.clear(); + support_vector.clear(); + coherence_vector.clear(); + iter_flow.clear(); + algorithm_map.clear(); + Gc(); +} + +void Reconstruction::Init() +{ + // create map of algorithms ids to the algorithm functions + CreateAlgorithmMap(); + + std::map flow_ptr_map; + flow_ptr_map[(char *)"NextIter"] = &Reconstruction::NextIter; + flow_ptr_map[(char *)"ResolutionTrigger"] = &Reconstruction::ResolutionTrigger; + flow_ptr_map[(char *)"SupportTrigger"] = &Reconstruction::SupportTrigger; + flow_ptr_map[(char *)"PhaseTrigger"] = &Reconstruction::PhaseTrigger; + flow_ptr_map[(char *)"ToReciprocal"] = &Reconstruction::ToReciprocal; + flow_ptr_map[(char *)"PcdiTrigger"] = &Reconstruction::PcdiTrigger; + flow_ptr_map[(char *)"Pcdi"] = &Reconstruction::Pcdi; + flow_ptr_map[(char *)"NoPcdi"] = &Reconstruction::NoPcdi; + flow_ptr_map[(char *)"Gc"] = &Reconstruction::Gc; + flow_ptr_map[(char *)"SetPcdiPrevious"] = &Reconstruction::SetPcdiPrevious; + flow_ptr_map[(char *)"ToDirect"] = &Reconstruction::ToDirect; + flow_ptr_map[(char *)"RunAlg"] = &Reconstruction::ModulusConstrainEr; //This will be replaced by configured algorithm method + flow_ptr_map[(char *)"Twin"] = &Reconstruction::Twin; + flow_ptr_map[(char *)"Average"] = &Reconstruction::Average; + flow_ptr_map[(char *)"Prog"] = &Reconstruction::Progress; + + std::vector used_flow_seq = params->GetUsedFlowSeq(); + std::vector flow_array = params->GetFlowArray(); + int num_iter = params->GetNumberIterations(); + + for (uint i = 0; i < used_flow_seq.size(); i++) + { + int func_order = used_flow_seq[i]; + int offset = i * num_iter; + for (int j=0; j < num_iter; j++) + { + if (flow_array[offset + j]) + { + if (flow_array[offset + j] > 1) + { + fp func_ptr = algorithm_map[flow_array[offset + j]]; + iter_flow[j].push_back(func_ptr); + } + else + { + fp func_ptr = flow_ptr_map[flow_def[func_order].func_name]; + iter_flow[j].push_back(func_ptr); + } + } + } + } + + // initialize other components + state->Init(); + if (partialCoherence != NULL) + { + partialCoherence->Init(data); + } + + norm_data = GetNorm(data); + num_points = data.elements(); + // multiply the rs_amplitudes by max element of data array and the norm + d_type max_data = af::max(data); + ds_image *= max_data * GetNorm(ds_image); + +// the next two lines are for testing it sets initial guess to initial support + // af::array temp = support->GetSupportArray(); + // ds_image = complex(temp.as((af_dtype) dtype_traits::ctype), 0.0).as(c64); + + ds_image *= support->GetSupportArray(); + //printf("initial image norm %f\n", GetNorm(ds_image)); + +} + +void Reconstruction::CreateAlgorithmMap() +{ + algorithm_map[ALGORITHM_ER] = &Reconstruction::ModulusConstrainEr; + algorithm_map[ALGORITHM_HIO] = &Reconstruction::ModulusConstrainHio; +} + +int Reconstruction::Iterate() +{ + while (state->Next()) + { + current_iteration = state->GetCurrentIteration(); + if (access("stopfile", F_OK) == 0) + { + remove("stopfile"); + return (uint)(getpid()); + } + if (anyTrue(isNaN(ds_image))) + { + printf("the image array has NaN element, quiting this reconstruction process\n"); + return (uint)(getpid()); + } + for (uint i=0; i*iter_flow[current_iteration][i])(); + } + } + + if (aver_v.size() > 0) + { + af::array aver_a(ds_image.dims(), &aver_v[0]); + af::array ratio = Utils::GetRatio(aver_a, abs(ds_image)); + ds_image *= ratio/aver_iter; + } + ds_image *= support->GetSupportArray(); + VectorizeSupport(); + if (partialCoherence != NULL) + { + VectorizeCoherence(); + } + return 0; +} + +void Reconstruction::NextIter() +{ + iter_data = data; + sig = params->GetSupportSigma(); +// printf("NextIter %d\n", (uint)(getpid())); +} + +void Reconstruction::ResolutionTrigger() +{ + iter_data = resolution->GetIterData(current_iteration, data.copy()); + sig = resolution->GetIterSigma(current_iteration); +// printf("ResolutionTrigger %d\n", (uint)(getpid())); +} + +void Reconstruction::SupportTrigger() +{ + support->UpdateAmp(ds_image.copy(), sig, current_iteration); +// printf("SupportTrigger %d\n", (uint)(getpid())); +} + +void Reconstruction::PhaseTrigger() +{ + support->UpdatePhase(ds_image.copy(), current_iteration); +// printf("PhaseTrigger %d\n", (uint)(getpid())); +} + +void Reconstruction::ToReciprocal() +{ + rs_amplitudes = Utils::ifft(ds_image)*num_points; +// printf("data norm, ampl norm before ratio %fl %fl\n", GetNorm(iter_data), GetNorm(rs_amplitudes)); +// printf("ToReciprocal %d\n", (uint)(getpid())); +} + +void Reconstruction::PcdiTrigger() +{ + af::array abs_amplitudes = abs(rs_amplitudes).copy(); + partialCoherence->UpdatePartialCoherence(abs_amplitudes); +// printf("PcdiTrigger %d\n", (uint)(getpid())); +} + +void Reconstruction::Pcdi() +{ + af::array abs_amplitudes = abs(rs_amplitudes).copy(); + af::array converged = partialCoherence->ApplyPartialCoherence(abs_amplitudes); + af::array ratio = Utils::GetRatio(iter_data, abs(converged)); +// printf("ratio norm %f\n", GetNorm(ratio)); + current_error = GetNorm(abs(converged)(converged > 0)-iter_data(converged > 0))/GetNorm(iter_data); + state->RecordError(current_error); + rs_amplitudes *= ratio; +// printf("Pcdi %d\n", (uint)(getpid())); +} + +void Reconstruction::NoPcdi() +{ + af::array ratio = Utils::GetRatio(iter_data, abs(rs_amplitudes)); + current_error = GetNorm(abs(rs_amplitudes)(rs_amplitudes > 0)-iter_data(rs_amplitudes > 0))/GetNorm(iter_data); + state->RecordError(current_error); + rs_amplitudes *= ratio; +// printf("NoPcdi %d\n", (uint)(getpid())); +} + +void Reconstruction::Gc() +{ + af::deviceGC(); +// printf("Gc\n"); +} + +void Reconstruction::SetPcdiPrevious() +{ + partialCoherence->SetPrevious(abs(rs_amplitudes)); +// printf("SetPcdiPrevious\n"); +} + +void Reconstruction::ToDirect() +{ + ds_image_raw = Utils::fft(rs_amplitudes)/num_points; +// printf("ToDirect\n"); +} + +void Reconstruction::Twin() +{ + dim4 dims = data.dims(); + std::vector twin_halves = params->GetTwinHalves(); + af::array temp = constant(0, dims, u32); + int x_start = (twin_halves[0] == 0) ? 0 : dims[0]/2; + int x_end = (twin_halves[0] == 0) ? dims[0]/2 -1 : dims[0]-1; + int y_start = (twin_halves[1] == 0) ? 0 : dims[1]/2; + int y_end = (twin_halves[1] == 0) ? dims[1]/2 -1 : dims[1]-1; + temp( af::seq(x_start, x_end), af::seq(y_start, y_end), span, span) = 1; + ds_image = ds_image * temp; +// printf("Twin\n"); +} + +void Reconstruction::Average() +{ + aver_iter++; + af::array abs_image = abs(ds_image).copy(); + d_type *image_v = abs_image.host(); + std::vector v(image_v, image_v + ds_image.elements()); + if (aver_v.size() == 0) + { + for (uint i = 0; i < v.size(); i++) + { + aver_v.push_back(v[i]); + } + } + else + { + for (uint i = 0; i < v.size(); i++) + { + aver_v[i] += v[i]; + } + } + + delete [] image_v; +// printf("Average\n"); +} + +void Reconstruction::Progress() +{ + printf("------- current iteration %i, error %f -------\n", current_iteration, current_error); +} + +void Reconstruction::ModulusConstrainEr() +{ +// printf("er\n"); +// printf("image norm before support %fl\n", GetNorm(ds_image_raw)); + af::array support_array = support->GetSupportArray(); + ds_image = ds_image_raw * support_array; +// printf("er, image norm after support %fl %d\n", GetNorm(ds_image), (uint)(getpid())); +} + +void Reconstruction::ModulusConstrainHio() +{ +// printf("hio\n"); +// printf("image norm before support %fl\n",GetNorm(ds_image_raw)); + //ds_image(support->GetSupportArray(state->IsApplyTwin()) == 0) = (ds_image - ds_image_raw * params->GetBeta())(support->GetSupportArray(state->IsApplyTwin()) == 0); + af::array support_array = support->GetSupportArray(); + af::array adjusted_calc_image = ds_image_raw * params->GetBeta(); + af::array combined_image = ds_image - adjusted_calc_image; + ds_image = ds_image_raw; + ds_image(support_array == 0) = combined_image(support_array == 0); +// printf("hio, image norm after support %fl %d\n",GetNorm(ds_image), (uint)(getpid())); +} + +double Reconstruction::GetNorm(af::array arr) +{ + return sum(pow(abs(arr), 2)); +} + +void Reconstruction::VectorizeSupport() +{ + af::array a = support->GetSupportArray().as(f32); + float *support_v = a.host(); + std::vector v(support_v, support_v + a.elements()); + support_vector = v; + delete [] support_v; + } + +void Reconstruction::VectorizeCoherence() +{ + // get the partial coherence as double, so it will work for float and double data types + af::array a = partialCoherence->GetKernelArray().copy(); + d_type *coherence_v = a.host(); + std::vector v(coherence_v, coherence_v + a.elements()); + coherence_vector = v; + delete [] coherence_v; +} + +af::array Reconstruction::GetImage() +{ + return ds_image; +} + +af::array Reconstruction::GetSupportArray() +{ + return support->GetSupportArray(); +} + +af::array Reconstruction::GetCoherenceArray() +{ + return partialCoherence->GetKernelArray(); +} + +std::vector Reconstruction::GetErrors() +{ + return state->GetErrors(); +} + +std::vector Reconstruction::GetSupportVector() +{ + return support_vector; +} + +std::vector Reconstruction::GetCoherenceVector() +{ + return coherence_vector; +} + +std::vector Reconstruction::GetCoherenceVectorR() +{ + return coherence_vector; +} + +std::vector Reconstruction::GetCoherenceVectorI() +{ + return coherence_vector; +} + +af::array Reconstruction::GetReciprocal() +{ + return rs_amplitudes; +} + +std::vector Reconstruction::GetFlowVector() +{ + return params->GetUsedFlowSeq(); +} + +std::vector Reconstruction::GetIterFlowVector() +{ + return params->GetFlowArray(); +} diff --git a/src_py/controller/__init__.py b/reccdi/src_py/__init__.py similarity index 100% rename from src_py/controller/__init__.py rename to reccdi/src_py/__init__.py diff --git a/src_py/cyth/__init__.py b/reccdi/src_py/beamlines/__init__.py similarity index 100% rename from src_py/cyth/__init__.py rename to reccdi/src_py/beamlines/__init__.py diff --git a/reccdi/src_py/beamlines/aps_34id/__init__.py b/reccdi/src_py/beamlines/aps_34id/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/reccdi/src_py/beamlines/aps_34id/detectors.py b/reccdi/src_py/beamlines/aps_34id/detectors.py new file mode 100644 index 0000000..04c4329 --- /dev/null +++ b/reccdi/src_py/beamlines/aps_34id/detectors.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +__author__ = "Ross Harder" +__docformat__ = 'restructuredtext en' +__all__ = ['getdetclass'] + + +################################################################## +def getdetclass(detname, **args): + for cls in Detector.__subclasses__(): + # print(detname.strip(),cls, cls.name) + if cls.name == detname.strip(): + c=cls() + return c + + +#could start to encapsulate everything about a detector here. whitefield and dark and other things. Then we +#just set the detector in a config file and everything can use it. maybe add a config_det file to conf? +class Detector(object): + name=None + def __init__(self, det_name): + self.det_name = det_name + + +class Detector_34idcTIM2(Detector): + name="34idcTIM2:" + dims=(512,512) + pixel=(55.0e-6,55e-6) + pixelorientation=('x+','y-') #in xrayutilities notation + def __init__(self): + super(Detector_34idcTIM2, self).__init__('34idcTIM2:') + diff --git a/reccdi/src_py/beamlines/aps_34id/diffractometer.py b/reccdi/src_py/beamlines/aps_34id/diffractometer.py new file mode 100644 index 0000000..7cf14ef --- /dev/null +++ b/reccdi/src_py/beamlines/aps_34id/diffractometer.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +__author__ = "Ross Harder" +__docformat__ = 'restructuredtext en' +__all__ = ['getdiffclass'] + + +################################################################## +def getdiffclass(diffname, **args): + for cls in Diffractometer.__subclasses__(): + print(diffname.strip(),cls,cls.name) + if cls.name == diffname.strip(): + c=cls() + return c + + +class Diffractometer(object): + name=None + def __init__(self, det_name): + self.det_name = det_name + +class Diffractometer_34idc(Diffractometer): + name="34idc" + sampleaxes=('y+','z-','x-') #in xrayutilities notation + detectoraxes=('y+','x-') + incidentaxis=(0,0,1) + sampleaxes_name=('th','chi','phi') #using the spec mnemonics for scan id. + detectoraxes_name=('delta','gamma') + def __init__(self): + super(Diffractometer_34idc, self).__init__('34idc') + diff --git a/reccdi/src_py/beamlines/aps_34id/prep.py b/reccdi/src_py/beamlines/aps_34id/prep.py new file mode 100644 index 0000000..cd2d926 --- /dev/null +++ b/reccdi/src_py/beamlines/aps_34id/prep.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +__author__ = "Ross Harder" +__docformat__ = 'restructuredtext en' +__all__ = ['get_dir_list', + 'get_dark_white', + 'get_normalized_slice', + 'read_scan', + 'shift', + 'combine_part', + 'fit', + 'prep_data', + 'prepare'] + +import pylibconfig2 as cfg +import numpy as np +import copy +import scipy.fftpack as sf +import os +import glob +import tifffile as tif +import reccdi.src_py.beamlines.aps_34id.spec as spec +import reccdi.src_py.utilities.utils as ut + + +def get_dir_list(scans, map): + """ + Returns list of sub-directories in data_dir with names matching range of scans + It will exclude scans within exclude_scans list if provided, and directories with fewer files than + min_files, if provided + :param scans: + :param map: + :return: + """ + try: + min_files = map.min_files + except: + min_files = 0 + try: + exclude_scans = map.exclude_scans + except: + exclude_scans = [] + try: + data_dir = map.data_dir.strip() + except: + print ('please provide data_dir') + return + + dirs = [] + for name in os.listdir(data_dir): + subdir = os.path.join(data_dir, name) + if os.path.isdir(subdir): + # exclude directories with fewer tif files than min_files + if len(glob.glob1(subdir, "*.tif")) < min_files and len(glob.glob1(subdir, "*.tiff")) < min_files: + continue + try: + index = int(name[-4:]) + if index >= scans[0] and index <= scans[1] and not index in exclude_scans: + dirs.append(subdir) + except: + continue + return dirs + + +def get_dark_white(darkfile, whitefile, det_area1, det_area2): + if darkfile is not None: + # find the darkfield array + dark_full = tif.imread(darkfile).astype(float) + # crop the corresponding quad or use the whole array, depending on what info was parsed from spec file + dark = dark_full[slice(det_area1[0], det_area1[0] + det_area1[1]), slice(det_area2[0], det_area2[0] + det_area2[1])] + else: + dark = None + + if whitefile is not None: + # find the whitefield array + white_full = tif.imread(whitefile).astype(float) + # crop the corresponding quad or use the whole array, depending on what info was parsed from spec file + white = white_full[slice(det_area1[0], det_area1[0] + det_area1[1]), slice(det_area2[0], det_area2[0] + det_area2[1])] + # set the bad pixels to some large value + white = np.where(white<5000, 1e20, white) #Some large value + else: + white = None + + return dark, white + + +def get_normalized_slice(file, dark, white): + # file is a tuple of slice and either background slice or None + slice = tif.TiffFile(file[0]).asarray() + if file[1] is not None: + slice = slice - tif.TiffFile(file[1]).asarray() + if dark is not None: + slice = np.where(dark > 5, 0, slice) #Ignore cosmic rays + # here would be code for correction for dead time + if white is not None: + slice = slice/white + slice *= 1e5 #Some medium value + slice = np.where(np.isnan(slice), 0, slice) + return slice + + +def read_scan(dir, dark, white): + files = [] + files_dir = {} + for file in os.listdir(dir): + if file.endswith('tif') or file.endswith('tiff'): + temp = file.split('.') + #it's assumed that the files end with four digits and 'tif' or 'tiff' extension + key = temp[0][-4:] + files_dir[key] = file + + ordered_keys = sorted(list(files_dir.keys())) + + for key in ordered_keys: + file = files_dir[key] + file = os.path.join(dir, file) + bg_file = file.replace('.tif', '_bg.tif') + if not os.path.isfile(bg_file): + bg_file = None + + files.append((file, bg_file)) + + # look at slice0 to find out shape + n = 0 + slice0 = get_normalized_slice(files[n], dark, white) + shape = (slice0.shape[0], slice0.shape[1], len(files)) + arr = np.zeros(shape, dtype=slice0.dtype) + arr[:,:,0] = slice0 + + #for i in range (1, len(files)): + for file in files[1:]: + n = n + 1 + slice = get_normalized_slice(file, dark, white) + arr[:,:,n] = slice + return arr + + +def shift(arr, shifty): + # pass the FT of the fftshifted array you want to shift + # you get back the actual array, not the FT. + dims = arr.shape + # scipy does normalize ffts! + ftarr = sf.fftn(arr) + r=[] + for d in dims: + r.append(slice(int(np.ceil(-d/2.)), int(np.ceil(d/2.)), None)) + idxgrid = np.mgrid[r] + for d in range(len(dims)): + ftarr *= np.exp(-1j*2*np.pi*shifty[d]*sf.fftshift(idxgrid[d])/float(dims[d])) + + shifted_arr = sf.ifftn(ftarr) + return shifted_arr + + +def combine_part(part_f, slice_sum, refpart, part): + # get cross correlation and pixel shift + cross_correlation = sf.ifftn(refpart*np.conj(part_f)) + corelated = np.array(cross_correlation.shape) + amp = np.abs(cross_correlation) + intshift = np.unravel_index(amp.argmax(), corelated) + shifted = np.array(intshift) + pixelshift = np.where(shifted>=corelated/2, shifted-corelated, shifted) + return slice_sum + shift(part, pixelshift) + + +def fit(arr, det_area1, det_area2): + # The det_area parameters hold the [beginning of image, size] in both dimensions. + # the beginning of image is relative to the full sensor image 512 x 512. + # if the full sensor was used for the image (i.e. the data size is 512x512) + # or if the image overleaps multiple quads, + # the quadrants need to be shifted + # check whether the image was taken with a single quad, then no shift is needed + if (det_area1[0] + det_area1[1] <= 256) and (det_area2[0] + det_area2[1] <= 256): + return arr + else: + b = np.zeros((517,516,arr.shape[2]),float) + tmp = np.zeros((512,512,arr.shape[2]),float) + tmp[det_area1[0]:det_area1[0]+det_area1[1],det_area2[0]:det_area2[0]+det_area2[1],:] = arr + b[:256,:256,:] = tmp[:256,:256,:] #Quad top left unchanged + b[:256,260:,:] = tmp[:256,256:,:] #Quad top right moved 4 right + b[261:,:256,:] = tmp[256:,:256,:] #Quad bot left moved 6 down + b[261:,260:,:] = tmp[256:,256:,:] #Quad bot right + + return b[det_area1[0]:det_area1[0]+det_area1[1],det_area2[0]:det_area2[0]+det_area2[1],:] + + +def prep_data(experiment_dir, scans, map, det_area1, det_area2, *args): + if scans is None: + print ('scan info not provided') + return + + # build sub-directories map + if len(scans) == 1: + scans.append(scans[0]) + dirs = get_dir_list(scans, map) + if len(dirs) == 0: + print ('no data directories found') + return + else: + if not os.path.exists(experiment_dir): + os.makedirs(experiment_dir) + + try: + whitefile = (map.whitefile).strip() + except: + whitefile = None + + try: + darkfile = (map.darkfile).strip() + except: + darkfile = None + + dark, white = get_dark_white(darkfile, whitefile, det_area1, det_area2) + + if len(dirs) == 0: + print ('there are no data directories for given scans') + return + + if len(dirs) == 1: + arr = read_scan(dirs[0], dark, white) + arr = fit(arr, det_area1, det_area2) + else: + # make the first part a reference + part = read_scan(dirs[0], dark, white) + part = fit(part, det_area1, det_area2) + slice_sum = np.abs(copy.deepcopy(part)) + refpart = sf.fftn(part) + for i in range (1, len(dirs)): + #this will load scans from each directory into an array part + part = read_scan(dirs[i], dark, white) + part = fit(part, det_area1, det_area2) + # add the arrays together + part_f = sf.fftn(part) + slice_sum = combine_part(part_f, slice_sum, refpart, part) + arr = np.abs(slice_sum).astype(np.int32) + + #arr = fit(arr, det_area1, det_area2) + + #create directory to save prepared data ,/prep + prep_data_dir = os.path.join(experiment_dir, 'prep') + if not os.path.exists(prep_data_dir): + os.makedirs(prep_data_dir) + data_file = os.path.join(prep_data_dir, 'prep_data.tif') + + ut.save_tif(arr, data_file) + print ('done with prep, shape:', arr.shape) + + +def prepare(experiment_dir, scans, conf_file, *args): + try: + with open(conf_file, 'r') as f: + config_map = cfg.Config(f.read()) + except Exception as e: + print('Please check the configuration file ' + conf_file + '. Cannot parse ' + str(e)) + return + + scan_end = scans[len(scans)-1] + try: + specfile = config_map.specfile.strip() + # parse det1 and det2 parameters from spec + det_area1, det_area2 = spec.get_det_from_spec(specfile, scan_end) + except: + try: + det_quad = config_map.det_quad + if det_quad == 0: + det_area1 = (0, 512) + det_area2 = (0, 512) + elif det_quad == 1: + det_area1 = (0, 256) + det_area2 = (0, 256) + elif det_quad == 2: + det_area1 = (0, 256) + det_area2 = (256, 512) + elif det_quad == 3: + det_area1 = (256, 512) + det_area2 = (0, 256) + elif det_quad == 4: + det_area1 = (256, 512) + det_area2 = (256, 512) + else: + print('the detector quad can be configured as digt from 0 to 4') + return + except Exception as e: + print('spec file or scan is not configured, and detector quad is not configured') + return + + try: + separate_scans = config_map.separate_scans + except: + separate_scans = False + + # data prep + # if separate scans, prepare data in each scan separately in subdirectory + if separate_scans and len(scans) > 1: + for scan in range (scans[0], scans[1]+1): + single_scan = [scan] + scan_exp_dir = os.path.join(experiment_dir, 'scan_' + str(scan)) + prep_data(scan_exp_dir, single_scan, config_map, det_area1, det_area2, args) + else: + prep_data(experiment_dir, scans, config_map, det_area1, det_area2, args) + + diff --git a/reccdi/src_py/beamlines/aps_34id/spec.py b/reccdi/src_py/beamlines/aps_34id/spec.py new file mode 100644 index 0000000..1c7933e --- /dev/null +++ b/reccdi/src_py/beamlines/aps_34id/spec.py @@ -0,0 +1,52 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +__author__ = "Ross Harder" +__docformat__ = 'restructuredtext en' +__all__ = ['parse_spec', + 'get_det_from_spec'] + +from xrayutilities.io import spec as spec + +def parse_spec(specfile, scan): + # Scan numbers start at one but the list is 0 indexed + ss = spec.SPECFile(specfile)[scan - 1] + + # Stuff from the header + detector_name = str(ss.getheader_element('UIMDET')) + command = ss.command.split() + scanmot = command[1] + scanmot_del = (float(command[3]) - float(command[2])) / int(command[4]) + + # Motor stuff from the header + delta = ss.init_motor_pos['INIT_MOPO_Delta'] + gamma = ss.init_motor_pos['INIT_MOPO_Gamma'] + theta = ss.init_motor_pos['INIT_MOPO_Theta'] + phi = ss.init_motor_pos['INIT_MOPO_Phi'] + chi = ss.init_motor_pos['INIT_MOPO_Chi'] + detdist = ss.init_motor_pos['INIT_MOPO_camdist'] + energy = ss.init_motor_pos['INIT_MOPO_Energy'] + + # returning the scan motor name as well. Sometimes we scan things + # other than theta. So we need to expand the capability of the display + # code. + return delta, gamma, theta, phi, chi, scanmot, scanmot_del, detdist, detector_name, energy + + +def get_det_from_spec(specfile, scan): + # Scan numbers start at one but the list is 0 indexed + ss = spec.SPECFile(specfile)[scan - 1] + # Stuff from the header + try: + det_area = ss.getheader_element('UIMR5').split() + det_area1 = int(det_area[0]), int(det_area[1]) + det_area2 = int(det_area[2]), int(det_area[3]) + + return det_area1, det_area2 + except: + return None, None + + diff --git a/reccdi/src_py/beamlines/aps_34id/viz.py b/reccdi/src_py/beamlines/aps_34id/viz.py new file mode 100644 index 0000000..c118137 --- /dev/null +++ b/reccdi/src_py/beamlines/aps_34id/viz.py @@ -0,0 +1,413 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +import os +import sys +import numpy as np +import scipy.ndimage as ndi +import math as m +#import pyevtk.hl as vtk +from tvtk.api import tvtk +import xrayutilities.experiment as xuexp +import reccdi.src_py.utilities.utils as ut +from reccdi.src_py.utilities.utils import measure +import reccdi.src_py.beamlines.aps_34id.spec as sput +import reccdi.src_py.beamlines.aps_34id.detectors as det +import reccdi.src_py.beamlines.aps_34id.diffractometer as diff + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' + +class DispalyParams: + """ + This class encapsulates parameters defining image display. The parameters are +read from config file on construction or whereever they may exist. This class is +basically an information agglomerator for the viz generation. + """ + + def __init__(self, config): + """ + The constructor gets config file and fills out the class members. + + Parameters + ---------- + conf : str + configuration file name + + Returns + ------- + none + """ + deg2rad = np.pi / 180.0 + try: + specfile = config['specfile'] + last_scan = config['last_scan'] + #get stuff from the spec file. + delta, gamma, th, phi, chi, scanmot, scanmot_del, detdist, detector, energy = sput.parse_spec(specfile, last_scan) + self.delta = delta + self.gamma = gamma + self.th = th + self.phi = phi + self.chi = chi + self.detdist = detdist + self.energy = energy + self.scanmot=scanmot + self.scanmot_del=scanmot_del + self.detector=detector + + except Exception as e: + pass + # override the parsed parameters with entries in config file + try: + self.energy = config['energy'] + except KeyError: + print("energy from specfile") + pass + try: + self.diffractometer=config['diffractometer'] + except KeyError: + pass + try: + print("in params",self.diffractometer) + self.diffractometer_obj=diff.getdiffclass(self.diffractometer) + except: + print("Unexpected error:", sys.exc_info()[0]) + self.diffractometer_obj=None + try: + #this may not need to be in a try block + #get attributes out of the diffractometer class and instance + for attr in self.diffractometer_obj.__class__.__dict__.keys(): + print("atr", attr) + if not attr.startswith('__'): + self.__dict__[attr]=self.diffractometer_obj.__class__.__dict__[attr] + for attr in self.diffractometer_obj.__dict__.keys(): + print("atr", attr) + if not attr.startswith('__'): + self.__dict__[attr]=self.diffractometer_obj.__dict__[attr] + except: + pass + #get attribures out of config, overwriting things from diff object + try: + self.sampleaxes=tuple(config['sampleaxes']) + except KeyError: + pass + try: + self.detectoraxes = tuple(config['detectoraxes']) + except KeyError: + pass + try: + self.sampleaxes_name = tuple(config['sampleaxes_name']) + except KeyError: + pass + try: + self.detectoraxes_name = tuple(config['detectoraxes_name']) + except KeyError: + pass + try: + self.incidentaxis = tuple(config['incidentaxis']) + except KeyError: + pass + #axes values are set from the spec file, but if they are specified in the config file + #the vals from config take precedence. + for axis in self.detectoraxes_name: + if axis in config: + self.__dict__[axis] = config[axis] + for axis in self.sampleaxes_name: + if axis in config: + self.__dict__[axis] = config[axis] + + try: + self.scanmot = config['scanmot'] + except KeyError: + pass + try: + self.scanmot_del = config['scanmot_del'] + except KeyError: + pass + + try: + self.detector = config['detector'] + except KeyError: + pass + try: + self.detector_obj=det.getdetclass(self.detector) + except: + self.detector_obj = None + + try: + #this may not need to be in a try block + for attr in self.detector_obj.__class__.__dict__.keys(): + if not attr.startswith('__'): + self.__dict__[attr]=self.detector_obj.__class__.__dict__[attr] + for attr in self.detector_obj.__dict__.keys(): + if not attr.startswith('__'): + self.__dict__[attr]=self.detector_obj.__dict__[attr] + except: + pass + + try: + self.pixelorientation = tuple(config['pixelorientation']) + except KeyError: + pass + + try: + self.pixel = tuple(config['pixel']) + except KeyError: + pass + + #binning is used, but crop is currently not used. + try: + self.binning = [] + binning = config['binning'] + for i in range(len(binning)): + self.binning.append(binning[i]) + for _ in range(3 - len(self.binning)): + self.binning.append(1) + except KeyError: + self.binning = [1,1,1] + try: + self.crop = [] + crop = config['crop'] + for i in range(len(crop)): + self.crop.append(crop[i]) + for _ in range(3 - len(self.crop)): + self.crop.append(1.0) + crop[0], crop[1] = crop[1], crop[0] + except KeyError: + self.crop = None + + +class CXDViz(): + + cropx = 0.5 + cropy = 0.5 + cropz = 0.5 + dir_arrs={} + recip_arrs={} + + def __init__(self): + self.imd = tvtk.ImageData() + self.sg = tvtk.StructuredGrid() + self.recipsg = tvtk.StructuredGrid() + pass + + + #def set_geometry(self, lam, delta, gamma, dpx, dpy, dth): + @measure + def set_geometry(self, p, shape): + self.params = p + #DisplayParams is not expected to do any modifications of params (units, etc) + px = p.pixel[0]*p.binning[0] + py = p.pixel[1]*p.binning[1] + detdist=p.detdist/1000.0 #convert to meters + scanmot=p.scanmot.strip() + enfix=1 + #if energy is given in kev convert to ev for xrayutilities + if m.floor(m.log10(p.energy)) < 3: + enfix=1000 + energy=p.energy*enfix #x-ray energy in eV + print("energy", energy) + + print("running the new xrayutilities version and tvtk") +# self.qc=xuexp.QConversion(['y+','z-','x-'], ['y+','x-'],(0,0,1),en=energy) + #if scanmot is energy then we need to init Qconversion with an array. + #thinking about making everything an array by default + if scanmot=='en': + scanen=np.array( (energy,energy+p.scanmot_del*enfix)) + else: + scanen=np.array( (energy,) ) + self.qc=xuexp.QConversion(p.sampleaxes, p.detectoraxes,p.incidentaxis,en=scanen) + + + #compute for 4pixel (2x2) detector + self.qc.init_area(p.pixelorientation[0],p.pixelorientation[1], shape[0],shape[1], 2,2, distance=detdist, pwidth1=px, pwidth2=py) + + #I think q2 will always be (3,2,2,2) (vec, scanarr, px, py) + #should put some try except around this in case something goes wrong. + if scanmot=='en': #seems en scans always have to be treated differently since init is unique + q2=np.array(self.qc.area(p.th, p.chi,p.phi,p.delta,p.gamma,deg=True)) + elif scanmot in p.sampleaxes_name: #based on scanmot args are made for qc.area + args=[] + axisindex=p.sampleaxes_name.index(scanmot) + for n in range(len(p.sampleaxes_name)): + if n==axisindex: + scanstart=p.__dict__[scanmot] + args.append(np.array( (scanstart,scanstart+p.scanmot_del) )) + else: + args.append( p.__dict__[p.sampleaxes_name[n]] ) + for axis in p.detectoraxes_name: + args.append( p.__dict__[axis] ) + print("args", args) + q2=np.array(self.qc.area( *args,deg=True)) + else: + print("scanmot not in sample axes or energy") + + #I think q2 will always be (3,2,2,2) (vec, scanarr, px, py) + Astar=q2[:,0,1,0]-q2[:,0,0,0] + Bstar=q2[:,0,0,1]-q2[:,0,0,0] + Cstar=q2[:,1,0,0]-q2[:,0,0,0] + + print("Recip") + print(Astar, Bstar, Cstar) + #transform to lab coords from sample reference frame + Astar=self.qc.transformSample2Lab(Astar, p.th,p.chi,p.phi)*10.0 #convert to inverse nm. + Bstar=self.qc.transformSample2Lab(Bstar, p.th,p.chi,p.phi)*10.0 + Cstar=self.qc.transformSample2Lab(Cstar, p.th,p.chi,p.phi)*10.0 + print(Astar, Bstar, Cstar) + + denom = np.dot(Astar, np.cross(Bstar, Cstar)) + A = 2 * m.pi * np.cross(Bstar, Cstar) / denom + B = 2 * m.pi * np.cross(Cstar, Astar) / denom + C = 2 * m.pi * np.cross(Astar, Bstar) / denom + + self.Trecip = np.zeros(9) + self.Trecip.shape = (3, 3) + self.Trecip[:, 0] = Astar + self.Trecip[:, 1] = Bstar + self.Trecip[:, 2] = Cstar + print(self.Trecip) + + self.Tdir = np.zeros(9) + self.Tdir.shape = (3, 3) + self.Tdir = np.array((A, B, C)).transpose() + print("Direct") + print(A,B,C) + print(self.Tdir) + + self.dirspace_uptodate=0 + self.recipspace_uptodate=0 + + def update_dirspace(self, shape): + print("Updating dirspace coords") + dims = list(shape) + self.dxdir = 1.0 / shape[0] + self.dydir = 1.0 / shape[1] + self.dzdir = 1.0 / shape[2] + + r = np.mgrid[ + 0:dims[0] * self.dxdir:self.dxdir, \ + 0:dims[1] * self.dydir:self.dydir,\ + 0:dims[2] * self.dzdir:self.dzdir] + + origshape=r.shape + r.shape = 3, dims[0] * dims[1] * dims[2] + + self.dir_coords = np.dot(self.Tdir, r).transpose() + + print("dir shape", self.dir_coords.shape) + self.dirspace_uptodate=1 + + def update_recipspace(self, shape): + dims = list(shape) + q = np.mgrid[ 0:dims[0], 0:dims[1], 0:dims[2]] + + origshape=q.shape + q.shape = 3, dims[0] * dims[1] * dims[2] + + self.recip_coords = np.dot(self.Trecip, q).transpose() + self.recipspace_uptodate=1 + + def clear_direct_arrays(self): + self.dir_arrs.clear() + def clear_recip_arrays(self): + self.recip_arrs.clear() + + @measure + def add_ds_array(self, array, name, logentry=None): + + #Need to add something to ensure arrays are all the same dimension. + #Need to add crop of viz output arrays + if len(array.shape) < 3: + newdims = list(array.shape) + for i in range(3 - len(newdims)): + newdims.append(1) + array.shape = tuple(newdims) + print("adding array of shape ", array.shape) + self.dir_arrs[name]=array + if (not self.dirspace_uptodate): + self.update_dirspace(array.shape) + + @measure + def add_rs_array(self, array, name, logentry=None): + + #Need to add something to ensure arrays are all the same dimension. + #Need to add crop of viz output arrays + if len(array.shape) < 3: + newdims = list(array.shape) + for i in range(3 - len(newdims)): + newdims.append(1) + array.shape = tuple(newdims) + print("adding array of shape ", array.shape) + self.recip_arrs[name]=array + if (not self.recipspace_uptodate): + self.update_recipspace(array.shape) + + + def get_ds_structured_grid(self, **args): + arr0=self.dir_arrs[list(self.dir_arrs.keys())[0]] + dims = list(arr0.shape) + self.sg.points = self.dir_coords + for a in self.dir_arrs.keys(): + arr=tvtk.DoubleArray() + arr.from_array(self.dir_arrs[a].ravel()) + arr.name=a + self.sg.point_data.add_array(arr) + + self.sg.dimensions = (dims[2], dims[1], dims[0]) + self.sg.extent = 0, dims[2] - 1, 0, dims[1] - 1, 0, dims[0] - 1 + return self.sg + + def get_rs_structured_grid(self, **args): + arr0=self.recip_arrs[list(self.recip_arrs.keys())[0]] + dims = list(arr0.shape) + self.recipsg.points = self.recip_coords + for a in self.recip_arrs.keys(): + arr=tvtk.DoubleArray() + arr.from_array(self.recip_arrs[a].ravel()) + arr.name=a + self.recipsg.point_data.add_array(arr) + + self.recipsg.dimensions = (dims[2], dims[1], dims[0]) + self.recipsg.extent = 0, dims[2] - 1, 0, dims[1] - 1, 0, dims[0] - 1 + return self.recipsg + + def write_directspace(self, filename, **args): + sgwriter = tvtk.XMLStructuredGridWriter() + #sgwriter.file_type = 'binary' + if filename.endswith(".vtk"): + sgwriter.file_name = filename + else: + sgwriter.file_name = filename + '.vts' + sgwriter.set_input_data(self.get_ds_structured_grid()) + sgwriter.write() + print ('saved file', filename) + + def write_recipspace(self, filename, **args): + sgwriter = tvtk.XMLStructuredGridWriter() + #sgwriter.file_type = 'binary' + if filename.endswith(".vtk"): + sgwriter.file_name = filename + else: + sgwriter.file_name = filename + '.vts' + sgwriter.set_input_data(self.get_rs_structured_grid()) + sgwriter.write() + print ('saved file', filename) + +#Save until we finally give up in pyevtk +# @measure +# def write_directspace_pyevtk(self, filename, **args): +# print(self.dir_arrs.keys()) +# vtk.gridToVTK(filename, self.dir_coords[0,:,:,:].copy(), \ +# self.dir_coords[1,:,:,:].copy(), \ +# self.dir_coords[2,:,:,:].copy(), pointData=self.dir_arrs) +# vtk.imageToVTK(filename, pointData=self.dir_arrs) +# +# def write_recipspace_pyevtk(self, filename, **args): +# vtk.gridToVTK(filename, self.recip_coords[0,:,:,:].copy(), \ +# self.recip_coords[1,:,:,:].copy(), \ +# self.recip_coords[2,:,:,:].copy(), pointData=self.recip_arrs) +# vtk.imageToVTK(filename, pointData=self.recip_arrs) diff --git a/reccdi/src_py/controller/__init__.py b/reccdi/src_py/controller/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/reccdi/src_py/controller/data.py b/reccdi/src_py/controller/data.py new file mode 100644 index 0000000..b347e44 --- /dev/null +++ b/reccdi/src_py/controller/data.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module controls the reconstruction process. The user has to provide parameters such as type of processor, data, and configuration. +The processor specifies which library will be used by CFM (Calc Fast Module) that performs the processor intensive calculations. The module +can be run on cpu, or gpu. Depending on the gpu hardware and library, one can use opencl or cuda library. +The module starts the data preparation routines, calls for reconstruction using the CFM, and prepares the reconstructed data for +visualization. +""" + +import numpy as np +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.utilities.parse_ver as ver +import os + + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['prep'] + + +def prep(fname, conf_info): + + """ + This function prepares raw data for reconstruction. It uses configured parameters. The preparation consists of the following steps: + 1. clearing the noise - the values below an amplitude threshold are set to zero + 2. removing the "aliens" - aliens are areas that are effect of interference. The area is manually set in a configuration file + after inspecting the data. + 3. binning - adding amplitudes of several consecutive points. Binning can be done in any dimension. + 4. amplitudes are set to sqrt + 5. cropping and padding. If the adjust_dimention is negative in any dimension, the array is cropped in this dimension. + The cropping is followed by padding in the dimensions that have positive adjust dimension. After adjusting, the dimensions + are adjusted further to find the smallest dimension that is supported by opencl library (multiplier of 2, 3, and 5). + 6. centering - finding the greatest amplitude and locating it at a center of new array. If shift center is defined, the + center will be shifted accordingly. The shifted elements are rolled into the other end of array. + + The modified data is then saved in data directory. + + Parameters + ---------- + fname : str + tif file containing raw data + + conf_info : str + experiment directory or configuration file. If it is directory, the "conf/config_data" will be + appended to determine configuration file + + Returns + ------- + nothing + """ + # The data has been transposed when saved in tif format for the ImageJ to show the right orientation + data = ut.read_tif(fname) + + if os.path.isdir(conf_info): + experiment_dir = conf_info + conf = os.path.join(experiment_dir, 'conf', 'config_data') + # if the experiment contains separate scan directories + if not os.path.isfile(conf): + base_dir = os.path.abspath(os.path.join(experiment_dir, os.pardir)) + conf = os.path.join(base_dir, 'conf', 'config_data') + else: + #assuming it's a file + conf = conf_info + experiment_dir = None + + # verify the configuration file + if not ver.ver_config_data(conf): + return + + try: + config_map = ut.read_config(conf) + if config_map is None: + print ("can't read configuration file") + return + except: + print ('Please check the configuration file ' + conf + '. Cannot parse') + return + + # saving file for Kenley project - AI aliens removing + print ('saving for AI') + d_f = os.path.join(experiment_dir, 'prep', 'prep_data.npy') + np.save(d_f, data) + # zero out the ares defined by aliens + try: + aliens = config_map.aliens + print ('removing aliens') + for alien in aliens: + # The ImageJ swaps the x and y axis, so the aliens coordinates needs to be swapped, since ImageJ is used + # to find aliens + data[alien[1]:alien[4], alien[0]:alien[3], alien[2]:alien[5]] = 0 + # saving file for Kenley project - AI aliens removing + aliens_f = os.path.join(experiment_dir, 'prep', 'aliens') + with open(aliens_f, 'a') as a_f: + try: + with open(conf, 'r') as f: + for line in f: + if line.startswith('aliens'): + a_f.write(line + '\n') + break + f.close() + a_f.close() + except: + pass + # saving file for Kenley project - AI aliens removing + d_f = os.path.join(experiment_dir, 'prep', 'prep_no_aliens.npy') + np.save(d_f, data) + d_f = os.path.join(experiment_dir, 'prep', 'prep_no_aliens.tif') + ut.save_tif(data, d_f) + + except AttributeError: + pass + except: + print ('error in aliens configuration') + + try: + amp_threshold = config_map.amp_threshold + print ('applied threshold') + except AttributeError: + print ('define amplitude threshold. Exiting') + return + + # zero out the noise + prep_data = np.where(data <= amp_threshold, 0, data) + + # square root data + prep_data = np.sqrt(prep_data) + + try: + crops_pads = config_map.adjust_dimensions + # the adjust_dimention parameter list holds adjustment in each direction. Append 0s, if shorter + if len(crops_pads) < 6: + for _ in range (6 - len(crops_pads)): + crops_pads.append(0) + except AttributeError: + # the size still has to be adjusted to the opencl supported dimension + crops_pads = (0, 0, 0, 0, 0, 0) + # adjust the size, either pad with 0s or crop array + print ('cropping and/or padding dimensions') + pairs = [] + for i in range(int(len(crops_pads)/2)): + pair = crops_pads[2*i:2*i+2] + pairs.append(pair) + # change pairs x and y, as the ImageJ swaps the axes + pairs[0], pairs[1] = pairs[1], pairs[0] + prep_data = ut.adjust_dimensions(prep_data, pairs) + if prep_data is None: + print('check "adjust_dimensions" configuration') + return + + try: + center_shift = config_map.center_shift + print ('shift center') + prep_data = ut.get_centered(prep_data, center_shift) + except AttributeError: + prep_data = ut.get_centered(prep_data, [0,0,0]) + + try: + binsizes = config_map.binning + try: + bins = [] + for binsize in binsizes: + bins.append(binsize) + filler = len(prep_data.shape) - len(bins) + for _ in range(filler): + bins.append(1) + bins[0], bins[1] = bins[1], bins[0] + prep_data = ut.binning(prep_data, bins) + except: + print ('check "binning" configuration') + except AttributeError: + pass + + try: + data_dir = config_map.data_dir + except AttributeError: + data_dir = 'data' + if experiment_dir is not None: + data_dir = os.path.join(experiment_dir, data_dir) + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + # save data + data_file = os.path.join(data_dir, 'data.tif') + ut.save_tif(prep_data, data_file) + print ('data ready for reconstruction, data dims:', prep_data.shape) diff --git a/reccdi/src_py/controller/fast_module.py b/reccdi/src_py/controller/fast_module.py new file mode 100755 index 0000000..3376d3f --- /dev/null +++ b/reccdi/src_py/controller/fast_module.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module controls the reconstruction process. The user has to provide parameters such as type of processor, data, and configuration. +The processor specifies which library will be used by CFM (Calc Fast Module) that performs the processor intensive calculations. The module +can be run on cpu, or gpu. Depending on the gpu hardware and library, one can use opencl or cuda library. +The module starts the data preparation routines, calls for reconstruction using the CFM, and prepares the reconstructed data for +visualization. +""" + +import numpy as np +import scipy.fftpack as sf +# import reccdi.src_py.cyth.bridge_cpu as bridge_cpu +# import reccdi.src_py.cyth.bridge_opencl as bridge_opencl +# import reccdi.src_py.cyth.bridge_cuda as bridge_cuda +import copy + + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['fast_module_reconstruction',] + + +def fast_module_reconstruction(proc, device, conf, data, coh_dims, image=None, support=None, coherence=None): + """ + This function calls a bridge method corresponding to the requested processor type. The bridge method is an access + to the CFM (Calc Fast Module). When reconstruction is completed the function retrieves results from the CFM. + The data received is max centered and the array is ordered "C". The CFM requires data zero-frequency component at + the center of the spectrum and "F" array order. Thus the data is modified at the beginning. + + Parameters + ---------- + proc : str + a string indicating the processor type + + device : int + device id assigned to this reconstruction + + conf : dict + configuration map + + data : array + a 3D np array containing pre-processed experiment data + + coh_dims : tuple + shape of coherence array + + image : numpy array + initial image for reconstruction or None + + support : numpy array + support corresponding to image or None + + coherence : numpy array + coherence corresponding to image + + Returns + ------- + image : numpy array + reconstructed image + + support : numpy array + support for reconstructed image + + coherence : numpy array + coherence for reconstructed image + + er : list + a vector containing errors for each iteration + """ + if proc == 'cpu': + import reccdi.src_py.cyth.bridge_cpu as bridge_cpu + fast_module = bridge_cpu.PyBridge() + elif proc == 'opencl': + import reccdi.src_py.cyth.bridge_opencl as bridge_opencl + fast_module = bridge_opencl.PyBridge() + elif proc == 'cuda': + import reccdi.src_py.cyth.bridge_cuda as bridge_cuda + fast_module = bridge_cuda.PyBridge() + + # shift data + data = sf.fftshift(data) + dims = data.shape[::-1] + print("data dims", dims) + data_l = data.flatten().tolist() + if image is None: + # print("Running start_calc") + fast_module.start_calc(device, data_l, dims, conf) + elif support is None: + image = image.flatten() + fast_module.start_calc_with_guess(device, data_l, image.real.tolist(), image.imag.tolist(), dims, conf) + elif coherence is None: + image = image.flatten() + support = support.flatten() + fast_module.start_calc_with_guess_support(device, data_l, image.real.tolist(), image.imag.tolist(), support.tolist(), dims, conf) + else: + image = image.flatten() + support = support.flatten() + coh_dims1 = (coh_dims[2], coh_dims[1], coh_dims[0]) + coherence = coherence.flatten() + + fast_module.start_calc_with_guess_support_coh(device, data_l, image.real.tolist(), image.imag.tolist(), support.tolist(), dims, coherence.tolist(), coh_dims, conf) + + er = copy.deepcopy(fast_module.get_errors()) + if len(er) == 1 and er[0] == -1: + # run into Nan during reconstruction + fast_module.cleanup() + return None, None, None, None, None, None, None + image_r = np.asarray(fast_module.get_image_r()) + image_i = np.asarray(fast_module.get_image_i()) + image = image_r + 1j*image_i #no need to deepcopy the real and imag parts since this makes a new array + + # normalize image + mx = np.abs(image).max() + image = image/mx + + support = copy.deepcopy(np.asarray(fast_module.get_support())) + coherence = copy.deepcopy(np.asarray(fast_module.get_coherence())) + + image.shape=dims[::-1] + support = np.reshape(support, dims[::-1]) + if coherence.shape[0] > 1: + coherence = np.reshape(coherence, coh_dims[::-1]) + else: + coherence = None + + reciprocal_r = copy.deepcopy(np.asarray(fast_module.get_reciprocal_r())) + reciprocal_i = copy.deepcopy(np.asarray(fast_module.get_reciprocal_i())) + reciprocal = reciprocal_r + 1j*reciprocal_i + reciprocal = np.reshape(reciprocal, dims[::-1]) + reciprocal = sf.ifftshift(reciprocal) + + iter_array = copy.deepcopy(np.asarray(fast_module.get_iter_flow())) + flow = copy.deepcopy(list(fast_module.get_flow())) + flow_len = len(flow) + iter_array = np.reshape(iter_array, (flow_len, int(iter_array.shape[0]/flow_len))) + + fast_module.cleanup() + + return image, support, coherence, er, reciprocal, flow, iter_array + diff --git a/reccdi/src_py/controller/gen_rec.py b/reccdi/src_py/controller/gen_rec.py new file mode 100644 index 0000000..473d8bf --- /dev/null +++ b/reccdi/src_py/controller/gen_rec.py @@ -0,0 +1,400 @@ + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +This module controls the genetic algoritm process. +""" + +import numpy as np +import os +import reccdi.src_py.controller.reconstruction as single +import reccdi.src_py.controller.reconstruction_multi as multi +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.utilities.utils_ga as gut +import multiprocessing as mp +from functools import partial +import time + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['read_config', + 'reconstruction'] + + +class Generation: + """ + This class holds fields relevant to generations according to configuration. + """ + def __init__(self, config_map): + self.current_gen = 0 + try: + self.generations = config_map.generations + except AttributeError: + self.generations = 1 + + try: + self.metrics = tuple(config_map.ga_metrics) + if len(self.metrics) < self.generations: + self.metrics = self.metrics + ('chi',) * (self.generations - len(self.metrics)) + except AttributeError: + self.metrics = ('chi',) * self.generations + + try: + self.worst_remove_no = tuple(config_map.ga_removes) + if len(self.worst_remove_no) < self.generations: + self.worst_remove_no = self.worst_remove_no + (0,) * (self.generations - len(self.worst_remove_no)) + except AttributeError: + self.worst_remove_no = None + + try: + self.ga_support_thresholds = tuple(config_map.ga_support_thresholds) + if len(self.ga_support_thresholds) < self.generations: + try: + support_threshold = config_map.support_threshold + except: + support_threshold = .1 + self.ga_support_thresholds = self.ga_support_thresholds + (support_threshold,) * (self.generations - len(self.ga_support_thresholds)) + except AttributeError: + try: + support_threshold = config_map.support_threshold + except: + support_threshold = .1 + self.ga_support_thresholds = (support_threshold,) * (self.generations) + + try: + self.ga_support_sigmas = tuple(config_map.ga_support_sigmas) + if len(self.ga_support_sigmas) < self.generations: + try: + support_sigma = config_map.support_sigma + except: + support_sigma = 1.0 + self.ga_support_sigmas = self.ga_support_sigmas + (support_sigma,) * (self.generations - len(self.ga_support_sigmas)) + except AttributeError: + try: + support_sigma = config_map.support_sigma + except: + support_sigma = 1.0 + self.ga_support_sigmas = (support_sigma,) * (self.generations) + + try: + self.breed_modes = tuple(config_map.ga_breed_modes) + if len(self.breed_modes) < self.generations: + self.breed_modes = self.breed_modes + ('none',) * (self.generations - len(self.breed_modes)) + except AttributeError: + self.breed_modes = ('none',) * self.generations + + try: + self.sigmas = config_map.ga_low_resolution_sigmas + self.low_resolution_generations = len(self.sigmas) + except AttributeError: + self.low_resolution_generations = 0 + + if self.low_resolution_generations > 0: + try: + self.low_resolution_alg = config_map.ga_low_resolution_alg + except AttributeError: + self.low_resolution_alg = 'GAUSS' + + + def next_gen(self): + self.current_gen += 1 + + def get_data(self, data): + if self.current_gen >= self.low_resolution_generations: + return data + else: + gmask = self.get_gmask(data.shape) + return data * gmask + + + def get_gmask(self, shape): + if self.low_resolution_alg == 'GAUSS': + if self.sigmas[self.current_gen] < 1.0: + ut.gaussian(shape, self.sigmas[self.current_gen]) + else: + return np.ones(shape) + + + def get_metrics(self, images, errs): + metrics = [] + for i in range(len(images)): + pop_metric = {} + pop_metric['chi'] = errs[i][-1] + pop_metric['sharpness'] = sum(sum(sum(pow(abs(images[i]), 4)))) + pop_metric['summed_phase'] = sum(sum(gut.sum_phase_tight_support(images[i]))) + pop_metric['area'] = sum(sum(sum(ut.shrink_wrap(images[i], .2, .5)))) + metrics.append(pop_metric) + return metrics + + + def rank(self, images, errs): + print ('ranking generation ', self.current_gen) + rank_property = [] + + reconstructions = len(images) + metric = self.metrics[self.current_gen] + + for i in range (reconstructions): + image = images[i] + if metric == 'chi': + rank_property.append(errs[i][-1]) + elif metric == 'sharpness': + rank_property.append(sum(sum(sum(pow(abs(image), 4))))) + elif metric == 'summed_phase': + rank_property.append(sum(sum(gut.sum_phase_tight_support(image)))) + elif metric == 'area': + support = ut.shrink_wrap(image, .2, .5) + rank_property.append(sum(sum(sum(support)))) + else: + # metric is 'chi' + rank_property.append(errs[i][-1]) + + # ranks keeps indexes of reconstructions from best to worst + # for most of the metric types the minimum of the metric is best, but for + # 'summed_phase' and 'area' it is oposite, so reversing the order + ranks = np.argsort(rank_property).tolist() + if metric == 'summed_phase' or metric == 'area': + ranks.reverse() + return ranks + + + def order(self, images, supports, cohs, errs, recips): + start = time.time() + ranks = self.rank(images, errs) + ordered_images = [] + ordered_supports = [] + ordered_cohs = [] + ordered_errs = [] + ordered_recips = [] + for i in range(len(ranks)): + ordered_images.append(images[ranks[i]]) + ordered_supports.append(supports[ranks[i]]) + ordered_cohs.append(cohs[ranks[i]]) + ordered_errs.append(errs[ranks[i]]) + ordered_recips.append(recips[ranks[i]]) + + stop = time.time() + # print ('rank and order time', (stop - start)) + return ordered_images, ordered_supports, ordered_cohs, ordered_errs, ordered_recips + + + def breed_one(self, alpha, breed_mode, beta): + beta = gut.zero_phase(beta, 0) + alpha = gut.check_get_conj_reflect(beta, alpha) + alpha_s = gut.align_arrays(beta, alpha) + alpha_s = gut.zero_phase(alpha_s, 0) + ph_alpha = np.angle(alpha_s) + beta = gut.zero_phase_cc(beta, alpha_s) + ph_beta = np.angle(beta) + if breed_mode == 'sqrt_ab': + beta = np.sqrt(abs(alpha_s) * abs(beta)) * np.exp(0.5j * (ph_beta + ph_alpha)) + + elif breed_mode == 'dsqrt': + amp = pow(abs(beta), .5) + beta = amp * np.exp(1j * ph_beta) + + elif breed_mode == 'pixel_switch': + cond = np.random.random_sample(beta.shape) + beta = np.where((cond > 0.5), beta, alpha_s) + + elif breed_mode == 'b_pa': + beta = abs(beta) * np.exp(1j * (ph_alpha)) + + elif breed_mode == '2ab_a_b': + beta = 2 * (beta * alpha_s) / (beta + alpha_s) + + elif breed_mode == '2a_b_pa': + beta = (2 * abs(alpha_s) - abs(beta)) * np.exp(1j * ph_alpha) + + elif breed_mode == 'sqrt_ab_pa': + beta = np.sqrt(abs(alpha_s) * abs(beta)) * np.exp(1j * ph_alpha) + + elif breed_mode == 'sqrt_ab_pa_recip': + temp1 = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(beta))) + temp2 = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(alpha_s))) + temp = np.sqrt(abs(temp1) * abs(temp2)) * np.exp(1j * np.angle(temp2)) + beta = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(temp))) + + elif breed_mode == 'sqrt_ab_recip': + temp1 = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(beta))) + temp2 = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(alpha_s))) + temp = np.sqrt(abs(temp1) * abs(temp2)) * np.exp(.5j * np.angle(temp1)) * np.exp(.5j * np.angle(temp2)) + beta = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(temp))) + + elif breed_mode == 'max_ab': + beta = np.maximum(abs(alpha_s), abs(beta)) * np.exp(.5j * (ph_beta + ph_alpha)) + + elif breed_mode == 'max_ab_pa': + beta = np.maximum(abs(alpha_s), abs(beta)) * np.exp(1j * ph_alpha) + + elif breed_mode == 'min_ab_pa': + beta = np.minimum(abs(alpha_s), abs(beta)) * np.exp(1j * ph_alpha) + + elif breed_mode == 'avg_ab': + beta = 0.5 * (alpha_s + beta) + + elif breed_mode == 'avg_ab_pa': + beta = 0.5 * (abs(alpha_s) + abs(beta)) * np.exp(1j * (ph_alpha)) + + return beta + + def breed(self, images): + """ + This function ranks the multiple reconstruction. It breeds next generation by combining the reconstructed + images, centered + For each combined image the support is calculated and coherence is set to None. + The number of bred images matches the number of reconstructions. + Parameters + ---------- + images : list + ordered (best to worst) list of images arrays + supports : list + list of supports arrays + Returns + ------- + child_images : list + list of bred images + child_supports : list + list of calculated supports corresponding to child_images + child_cohs : list + list of child coherence, set to None + """ + print ('breeding generation ', (self.current_gen + 1)) + start = time.time() + child_images = [] + child_supports = [] + def collect_result(result): + for r in result: + if r is None: + continue + child_images.append(r) + child_supports.append(ut.shrink_wrap(r, threshold, sigma)) + + sigma = self.ga_support_sigmas[self.current_gen] + threshold = self.ga_support_thresholds[self.current_gen] + breed_mode = self.breed_modes[self.current_gen] + reconstructions = len(images) + if self.worst_remove_no is not None: + reconstructions = reconstructions - self.worst_remove_no[self.current_gen] + if breed_mode == 'none': + return images, None + + alpha = images[0] + alpha = gut.zero_phase(alpha, 0) + ims = images[1 : reconstructions] + + # put the best into the bred population + child_images.append(alpha) + child_supports.append(ut.shrink_wrap(alpha, threshold, sigma)) + + no_processes = min(len(ims), mp.cpu_count()) + func = partial(self.breed_one, alpha, breed_mode) + with mp.Pool(processes = no_processes) as pool: + pool.map_async(func, ims, callback=collect_result) + pool.close() + pool.join() + pool.terminate() + + stop = time.time() + # print ('breeding time', (stop - start)) + return child_images, child_supports + + +def reconstruction(proc, conf_file, datafile, dir, devices): + """ + This function controls reconstruction utilizing genetic algorithm. + Parameters + ---------- + generation : int + number of generations + proc : str + processor to run on (cpu, opencl, or cuda) + data : numpy array + initial data + conf_info : str + experiment directory or configuration file. If it is directory, the "conf/config_rec" will be + appended to determine configuration file + conf_map : dict + a dictionary from parsed configuration file + Returns + ------- + nothing + """ + data = ut.read_tif(datafile) + print ('data shape', data.shape) + + try: + config_map = ut.read_config(conf_file) + if config_map is None: + print("can't read configuration file " + conf_file) + return + except: + print('Cannot parse configuration file ' + conf_file + ' , check for matching parenthesis and quotations') + return + try: + reconstructions = config_map.reconstructions + except: + reconstructions = 1 + + gen_obj = Generation(config_map) + + try: + save_dir = config_map.save_dir + except AttributeError: + filename = conf_file.split('/')[-1] + save_dir = os.path.join(dir, filename.replace('config_rec', 'results')) + + try: + generations = config_map.generations + except: + print ('generations not configured') + return + + # init starting values + # if multiple reconstructions configured (typical for genetic algorithm), use "reconstruction_multi" module + if reconstructions > 1: + images = [] + supports = [] + cohs = [] + for _ in range(reconstructions): + images.append(None) + supports.append(None) + cohs.append(None) + rec = multi + # load parls configuration + for g in range(generations): + gen_data = gen_obj.get_data(data) + images, supports, cohs, errs, recips, flows, iter_arrs = rec.multi_rec(proc, gen_data, conf_file, config_map, devices, images, supports, cohs) + images, supports, cohs, errs, recips = gen_obj.order(images, supports, cohs, errs, recips) + metrics = gen_obj.get_metrics(images, errs) + # save the generation results + gen_save_dir = os.path.join(save_dir, 'g_' + str(g)) + ut.save_multiple_results(len(images), images, supports, cohs, errs, recips, flows, iter_arrs, gen_save_dir, metrics) + if g < generations - 1 and len(images) > 1: + images, shrink_supports = gen_obj.breed(images) + if shrink_supports is not None: + supports = shrink_supports + gen_obj.next_gen() + else: + image = None + support = None + coh = None + rec = single + + for g in range(generations): + gen_data = gen_obj.get_data(data) + image, support, coh, err, recip, flows, iter_arrs = rec.single_rec(proc, gen_data, conf_file, config_map, devices[0], image, support, coh) + if image is None: + return + # save the generation results + gen_save_dir = os.path.join(save_dir, 'g_' + str(g)) + ut.save_results(image, support, coh, err, recip, flows, iter_arrs, gen_save_dir) + gen_obj.next_gen() + + print ('done gen') diff --git a/reccdi/src_py/controller/reconstruction.py b/reccdi/src_py/controller/reconstruction.py new file mode 100644 index 0000000..c4c8f59 --- /dev/null +++ b/reccdi/src_py/controller/reconstruction.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module controls the reconstruction process. The user has to provide parameters such as type of processor, data, and configuration. +The processor specifies which library will be used by CFM (Calc Fast Module) that performs the processor intensive calculations. The module +can be run on cpu, or gpu. Depending on the gpu hardware and library, one can use opencl or cuda library. +The module starts the data preparation routines, calls for reconstruction using the CFM, and prepares the reconstructed data for +visualization. +""" + +import numpy as np +import os +import reccdi.src_py.controller.fast_module as calc +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.controller.reconstruction_multi as multi + + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['read_config', + 'reconstruction'] + + +def single_rec(proc, data, conf, config_map, dev, image, support, coh): + + """ + This function starts and returns results of reconstruction. The parameters must be initialized. + + Parameters + ---------- + proc : str + a string indicating the processor type + + data : numpy array + data array + + conf : str + configuration file name + + config_map : dict + parsed configuration + + image : numpy array + reconstructed image for further reconstruction, or None for initial + + support : numpy array + support of reconstructed image, or None + + coh : numpy array + coherence of reconstructed images, or None + + Returns + ------- + image : numpy array + reconstructed image + + support : numpy array + support of reconstructed images + + coh : numpy array + coherence of reconstructed images + + errs : list + list of errors (should we take the last error?) + """ + try: + coh_dims = tuple(config_map.partial_coherence_roi) + except: + coh_dims = None + image, support, coh, er, reciprocal, flow, iter_array = calc.fast_module_reconstruction(proc, dev, conf, data, coh_dims, image, support, coh) + + # errs contain errors for each iteration + return image, support, coh, er, reciprocal, flow, iter_array + + +def reconstruction(proc, conf_file, datafile, dir, dev): + """ + This function starts the reconstruction. It checks whether it is continuation of reconstruction defined by + configuration. If continuation, the arrays of image, support, coherence are read from cont_directory, + otherwise, they are initialized to None. After the arrays are initialized, they are passed for the reconstruction. + The results are saved in the configured directory. + + Parameters + ---------- + proc : str + a string indicating the processor type (cpu, opencl, cuda) + + data : numpy array + data array + + conf_info : str + configuration file name or experiment directory. If directory, the configuration file is + defined as /conf/config_rec + + config_map : dict + parsed configuration + + Returns + ------- + nothing + """ + data = ut.read_tif(datafile) + print ('data shape', data.shape) + + try: + config_map = ut.read_config(conf_file) + if config_map is None: + print("can't read configuration file " + conf_file) + return + except: + print('Cannot parse configuration file ' + conf_file + ' , check for matching parenthesis and quotations') + return + + cont = False + try: + if config_map.cont: + try: + continue_dir = config_map.continue_dir + image, support, coh = ut.read_results(continue_dir) + cont = True + except: + print("continue_dir not configured") + return None + except: + pass + + if not cont: + image = None + support = None + coh = None + + image, support, coh, errs, recips, flow, iter_array = single_rec(proc, data, conf_file, config_map, dev[0], image, support, coh) + if image is None: + return + + try: + save_dir = config_map.save_dir + except AttributeError: + filename = conf_file.split('/')[-1] + save_dir = os.path.join(dir, filename.replace('config_rec', 'results')) + + ut.save_results(image, support, coh, np.asarray(errs), recips, flow, iter_array, save_dir) diff --git a/reccdi/src_py/controller/reconstruction_multi.py b/reccdi/src_py/controller/reconstruction_multi.py new file mode 100644 index 0000000..37ee01e --- /dev/null +++ b/reccdi/src_py/controller/reconstruction_multi.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module controls the reconstruction process. The user has to provide parameters such as type of processor, data, and configuration. +The processor specifies which library will be used by CFM (Calc Fast Module) that performs the processor intensive calculations. The module +can be run on cpu, or gpu. Depending on the gpu hardware and library, one can use opencl or cuda library. +The module starts the data preparation routines, calls for reconstruction using the CFM, and prepares the reconstructed data for +visualization. +""" + +import os +import numpy as np +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.controller.fast_module as calc +import time +from multiprocessing import Pool, Queue +from functools import partial + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['read_config', + 'reconstruction'] + + +def single_rec_process(proc, conf, data, coh_dims, prev): + """ + This function runs in the reconstruction palarellized by Parsl. + + Parameters + ---------- + proc : str + string defining library used 'cpu' or 'opencl' or 'cuda' + + device : int + device allocated to this reconstruction or -1 if not configured + + conf : str + configuration file + + data : numpy array + data array + + coh_dims : tuple + shape of coherence array + + prev_image : numpy array or None + previously reconstructed image (if continuation or genetic algorithm) or None + + prev_support : numpy array or None + support of previously reconstructed image (if continuation or genetic algorithm) or None + + prev_coh : numpy array or None + coherence of previously reconstructed image (if continuation or genetic algorithm) or None + + Returns + ------- + image : numpy array + reconstructed image + + support : numpy array + support of reconstructed image + + coherence : coherence of reconstructed image + + error : list containing errors for iterations + """ + prev_image, prev_support, prev_coh = prev + image, support, coherence, errors, reciprocal, flow, iter_array = calc.fast_module_reconstruction(proc, gpu, conf, data, coh_dims, + prev_image, prev_support, prev_coh) + return image, support, coherence, errors, reciprocal, flow, iter_array + + +def assign_gpu(*args): + q = args[0] + global gpu + gpu = q.get() + + +def multi_rec(proc, data, conf, config_map, devices, prev_images, prev_supports, prev_cohs=None): + + """ + This function controls the multiple reconstructions. It invokes a loop to execute parallel resconstructions, + wait for all reconstructions to deliver results, and store te results. + + Parameters + ---------- + proc : str + a string indicating the processor type + + data : numpy array + data array + + conf : str + configuration file name + + config_map : dict + parsed configuration + + images : list + list of numpy arrays containing reconstructed images for further reconstruction, or None for initial + + supports : list + list of numpy arrays containing support of reconstructed images, or None + + cohs : list + list of numpy arrays containing coherence of reconstructed images, or None + + Returns + ------- + images : list + list of numpy arrays containing reconstructed images + + supports : list + list of numpy arrays containing support of reconstructed images + + cohs : list + list of numpy arrays containing coherence of reconstructed images + + errs : list + list of lists of errors (now each element is another list by iterations, but should we take the last error?) + """ + images = [] + supports = [] + cohs = [] + errs = [] + recips = [] + flows = [] + iter_arrs = [] + def collect_result(result): + for r in result: + if r[0] is None: + continue + images.append(r[0]) + supports.append(r[1]) + cohs.append(r[2]) + errs.append(r[3]) + recips.append(r[4]) + flows.append(r[5]) + iter_arrs.append(r[6]) + + reconstructions = config_map.reconstructions + + try: + coh_dims = tuple(config_map.partial_coherence_roi) + except: + coh_dims = None + + iterable = [] + for i in range(reconstructions): + if prev_cohs is None: + coh = None + else: + coh = prev_cohs[i] + iterable.append((prev_images[i], prev_supports[i], coh)) + + func = partial(single_rec_process, proc, conf, data, coh_dims) + q = Queue() + for device in devices: + q.put(device) + with Pool(processes = len(devices),initializer=assign_gpu, initargs=(q,)) as pool: + pool.map_async(func, iterable, callback=collect_result) + pool.close() + pool.join() + pool.terminate() + + # return only error from last iteration for each reconstruction + return images, supports, cohs, errs, recips, flows, iter_arrs + + +def reconstruction(proc, conf_file, datafile, dir, devices): +# proc, datafile, dir, conf_file, devices + """ + This function starts the reconstruction. It checks whether it is continuation of reconstruction defined by + configuration. If continuation, the lists contaning arrays of images, supports, coherence for multiple reconstructions + are read from cont_directory, otherwise, they are initialized to None. + After the lists are initialized, they are passed for the multi-reconstruction. + The results are saved in the configured directory. + + Parameters + ---------- + reconstructions : int + number of reconstructions + + proc : str + a string indicating the processor type (cpu, opencl, cuda) + + data : numpy array + data array + + conf_info : str + configuration file name or experiment directory. If directory, the configuration file is + defined as /conf/config_rec + + config_map : dict + parsed configuration + + Returns + ------- + nothing + """ + data = ut.read_tif(datafile) + print ('data shape', data.shape) + + try: + config_map = ut.read_config(conf_file) + if config_map is None: + print("can't read configuration file " + conf_file) + return + except: + print('Cannot parse configuration file ' + conf_file + ' , check for matching parenthesis and quotations') + return + + try: + reconstructions = config_map.reconstructions + except: + reconstructions = 1 + + images = [] + supports = [] + cohs = [] + try: + if config_map.cont: + try: + continue_dir = config_map.continue_dir + for sub in os.listdir(continue_dir): + image, support, coh = ut.read_results(os.path.join(continue_dir, sub) + '/') + images.append(image) + supports.append(support) + cohs.append(coh) + except: + print("continue_dir not configured") + return None + except: + for _ in range(reconstructions): + images.append(None) + supports.append(None) + cohs.append(None) + + new_images, new_supports, new_cohs, errs, recips, flows, iter_arrs = multi_rec(proc, data, conf_file, config_map, devices, images, supports, cohs) + + try: + save_dir = config_map.save_dir + except AttributeError: + filename = conf_file.split('/')[-1] + save_dir = os.path.join(dir, filename.replace('config_rec', 'results')) + + ut.save_multiple_results(len(new_images), new_images, new_supports, new_cohs, errs, recips, flows, iter_arrs, save_dir) diff --git a/reccdi/src_py/cyth/__init__.py b/reccdi/src_py/cyth/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/reccdi/src_py/cyth/bridge_cpu.pyx b/reccdi/src_py/cyth/bridge_cpu.pyx new file mode 100644 index 0000000..615cc26 --- /dev/null +++ b/reccdi/src_py/cyth/bridge_cpu.pyx @@ -0,0 +1,72 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +# distutils: language = c++ +# distutils: include_dirs = ['reccdi/include', '/home/beams/CXDUSER/CDI/arrayfire/include', '/home/beams/CXDUSER/CDI/libconfig/include',] +# distutils: sources = ['reccdi/src_cpp/bridge.cpp', 'reccdi/src_cpp/manager.cpp', 'reccdi/src_cpp/parameters.cpp', 'reccdi/src_cpp/pcdi.cpp', 'reccdi/src_cpp/resolution.cpp', 'reccdi/src_cpp/state.cpp', 'reccdi/src_cpp/support.cpp', 'reccdi/src_cpp/util.cpp', 'reccdi/src_cpp/worker.cpp'] +# distutils: libraries = ['afcpu', 'config++',] +# distutils: library_dirs = ['/home/beams/CXDUSER/CDI/arrayfire/lib64', '/home/beams/CXDUSER/CDI/libconfig/lib',] + +from libcpp.vector cimport vector +from libcpp.string cimport string + + +cdef extern from "../include/bridge.hpp": + cdef cppclass Bridge: + Bridge() except + + void StartCalcWithGuess(int, vector[float], vector[float], vector[float], vector[int], string) + void StartCalcWithGuessSupport(int, vector[float], vector[float], vector[float], vector[int], vector[int], string) + void StartCalcWithGuessSupportCoh(int, vector[float], vector[float], vector[float], vector[int], vector[int], vector[float], vector[int], string) + void StartCalc(int, vector[float], vector[int], string) + vector[double] GetImageR() + vector[double] GetImageI() + vector[double] GetErrors() + vector[float] GetSupportV() + vector[double] GetCoherenceV() + vector[double] GetReciprocalR() + vector[double] GetReciprocalI() + vector[int] GetFlowV() + vector[int] GetIterFlowV() + void Cleanup() + + +cdef class PyBridge: + cdef Bridge *thisptr + def __cinit__(self): + self.thisptr = new Bridge() + def __dealloc__(self): + del self.thisptr + def start_calc_with_guess(self, device, data_r, guess_r, guess_i, dims, config): + self.thisptr.StartCalcWithGuess(device, data_r, guess_r, guess_i, dims, config.encode()) + def start_calc_with_guess_support(self, device, data_r, guess_r, guess_i, support, dims, config): + self.thisptr.StartCalcWithGuessSupport(device, data_r, guess_r, guess_i, support, dims, config.encode()) + def start_calc_with_guess_support_coh(self, device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config): + self.thisptr.StartCalcWithGuessSupportCoh(device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config.encode()) + def start_calc(self, device, data_r, dims, config): + self.thisptr.StartCalc(device, data_r, dims, config.encode()) + def get_image_r(self): + return self.thisptr.GetImageR() + def get_image_i(self): + return self.thisptr.GetImageI() + def get_errors(self): + return self.thisptr.GetErrors() + def get_support(self): + return self.thisptr.GetSupportV() + def get_coherence(self): + return self.thisptr.GetCoherenceV() + def get_reciprocal_r(self): + return self.thisptr.GetReciprocalR() + def get_reciprocal_i(self): + return self.thisptr.GetReciprocalI() + def get_flow(self): + return self.thisptr.GetFlowV() + def get_iter_flow(self): + return self.thisptr.GetIterFlowV() + def cleanup(self): + self.thisptr.Cleanup() + + diff --git a/reccdi/src_py/cyth/bridge_cuda.pyx b/reccdi/src_py/cyth/bridge_cuda.pyx new file mode 100755 index 0000000..7ea8cbc --- /dev/null +++ b/reccdi/src_py/cyth/bridge_cuda.pyx @@ -0,0 +1,70 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +# distutils: language = c++ +# distutils: include_dirs = ['reccdi/include', '/home/beams/CXDUSER/CDI/arrayfire/include', '/home/beams/CXDUSER/CDI/libconfig/include',] +# distutils: sources = ['reccdi/src_cpp/bridge.cpp', 'reccdi/src_cpp/manager.cpp', 'reccdi/src_cpp/parameters.cpp', 'reccdi/src_cpp/pcdi.cpp', 'reccdi/src_cpp/resolution.cpp', 'reccdi/src_cpp/state.cpp', 'reccdi/src_cpp/support.cpp', 'reccdi/src_cpp/util.cpp', 'reccdi/src_cpp/worker.cpp'] +# distutils: libraries = ['afcuda', 'config++',] +# distutils: library_dirs = ['/home/beams/CXDUSER/CDI/arrayfire/lib64', '/home/beams/CXDUSER/CDI/libconfig/lib',] + +from libcpp.vector cimport vector +from libcpp.string cimport string + +cdef extern from "../include/bridge.hpp": + cdef cppclass Bridge: + Bridge() except + + void StartCalcWithGuess(int, vector[float], vector[float], vector[float], vector[int], string) + void StartCalcWithGuessSupport(int, vector[float], vector[float], vector[float], vector[int], vector[int], string) + void StartCalcWithGuessSupportCoh(int, vector[float], vector[float], vector[float], vector[int], vector[int], vector[float], vector[int], string) + void StartCalc(int, vector[float], vector[int], string) + vector[double] GetImageR() + vector[double] GetImageI() + vector[double] GetErrors() + vector[float] GetSupportV() + vector[double] GetCoherenceV() + vector[double] GetReciprocalR() + vector[double] GetReciprocalI() + vector[int] GetFlowV() + vector[int] GetIterFlowV() + void Cleanup() + + +cdef class PyBridge: + cdef Bridge *thisptr + def __cinit__(self): + self.thisptr = new Bridge() + def __dealloc__(self): + del self.thisptr + def start_calc_with_guess(self, device, data_r, guess_r, guess_i, dims, config): + self.thisptr.StartCalcWithGuess(device, data_r, guess_r, guess_i, dims, config.encode()) + def start_calc_with_guess_support(self, device, data_r, guess_r, guess_i, support, dims, config): + self.thisptr.StartCalcWithGuessSupport(device, data_r, guess_r, guess_i, support, dims, config.encode()) + def start_calc_with_guess_support_coh(self, device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config): + self.thisptr.StartCalcWithGuessSupportCoh(device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config.encode()) + def start_calc(self, device, data_r, dims, config): + self.thisptr.StartCalc(device, data_r, dims, config.encode()) + def get_image_r(self): + return self.thisptr.GetImageR() + def get_image_i(self): + return self.thisptr.GetImageI() + def get_errors(self): + return self.thisptr.GetErrors() + def get_support(self): + return self.thisptr.GetSupportV() + def get_coherence(self): + return self.thisptr.GetCoherenceV() + def get_reciprocal_r(self): + return self.thisptr.GetReciprocalR() + def get_reciprocal_i(self): + return self.thisptr.GetReciprocalI() + def get_flow(self): + return self.thisptr.GetFlowV() + def get_iter_flow(self): + return self.thisptr.GetIterFlowV() + def cleanup(self): + self.thisptr.Cleanup() + diff --git a/reccdi/src_py/cyth/bridge_opencl.pyx b/reccdi/src_py/cyth/bridge_opencl.pyx new file mode 100755 index 0000000..ddcc913 --- /dev/null +++ b/reccdi/src_py/cyth/bridge_opencl.pyx @@ -0,0 +1,70 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +# distutils: language = c++ +# distutils: include_dirs = ['reccdi/include', '/home/beams/CXDUSER/CDI/arrayfire/include', '/home/beams/CXDUSER/CDI/libconfig/include',] +# distutils: sources = ['reccdi/src_cpp/bridge.cpp', 'reccdi/src_cpp/manager.cpp', 'reccdi/src_cpp/parameters.cpp', 'reccdi/src_cpp/pcdi.cpp', 'reccdi/src_cpp/resolution.cpp', 'reccdi/src_cpp/state.cpp', 'reccdi/src_cpp/support.cpp', 'reccdi/src_cpp/util.cpp', 'reccdi/src_cpp/worker.cpp'] +# distutils: libraries = ['afopencl', 'config++',] +# distutils: library_dirs = ['/home/beams/CXDUSER/CDI/arrayfire/lib64', '/home/beams/CXDUSER/CDI/libconfig/lib',] + +from libcpp.vector cimport vector +from libcpp.string cimport string + +cdef extern from "../include/bridge.hpp": + cdef cppclass Bridge: + Bridge() except + + void StartCalcWithGuess(int, vector[float], vector[float], vector[float], vector[int], string) + void StartCalcWithGuessSupport(int, vector[float], vector[float], vector[float], vector[int], vector[int], string) + void StartCalcWithGuessSupportCoh(int, vector[float], vector[float], vector[float], vector[int], vector[int], vector[float], vector[int], string) + void StartCalc(int, vector[float], vector[int], string) + vector[double] GetImageR() + vector[double] GetImageI() + vector[double] GetErrors() + vector[float] GetSupportV() + vector[double] GetCoherenceV() + vector[double] GetReciprocalR() + vector[double] GetReciprocalI() + vector[int] GetFlowV() + vector[int] GetIterFlowV() + void Cleanup() + + +cdef class PyBridge: + cdef Bridge *thisptr + def __cinit__(self): + self.thisptr = new Bridge() + def __dealloc__(self): + del self.thisptr + def start_calc_with_guess(self, device, data_r, guess_r, guess_i, dims, config): + self.thisptr.StartCalcWithGuess(device, data_r, guess_r, guess_i, dims, config.encode()) + def start_calc_with_guess_support(self, device, data_r, guess_r, guess_i, support, dims, config): + self.thisptr.StartCalcWithGuessSupport(device, data_r, guess_r, guess_i, support, dims, config.encode()) + def start_calc_with_guess_support_coh(self, device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config): + self.thisptr.StartCalcWithGuessSupportCoh(device, data_r, guess_r, guess_i, support, dims, coh, coh_dims, config.encode()) + def start_calc(self, device, data_r, dims, config): + self.thisptr.StartCalc(device, data_r, dims, config.encode()) + def get_image_r(self): + return self.thisptr.GetImageR() + def get_image_i(self): + return self.thisptr.GetImageI() + def get_errors(self): + return self.thisptr.GetErrors() + def get_support(self): + return self.thisptr.GetSupportV() + def get_coherence(self): + return self.thisptr.GetCoherenceV() + def get_reciprocal_r(self): + return self.thisptr.GetReciprocalR() + def get_reciprocal_i(self): + return self.thisptr.GetReciprocalI() + def get_flow(self): + return self.thisptr.GetFlowV() + def get_iter_flow(self): + return self.thisptr.GetIterFlowV() + def cleanup(self): + self.thisptr.Cleanup() + diff --git a/reccdi/src_py/run_scripts/__init__.py b/reccdi/src_py/run_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reccdi/src_py/run_scripts/everything.py b/reccdi/src_py/run_scripts/everything.py new file mode 100644 index 0000000..acae99e --- /dev/null +++ b/reccdi/src_py/run_scripts/everything.py @@ -0,0 +1,29 @@ +import sys +import argparse +import reccdi.src_py.run_scripts.run_data as run_dt +import reccdi.src_py.run_scripts.run_rec as run_rc +import reccdi.src_py.run_scripts.run_disp as run_dp +import reccdi.src_py.run_scripts.run_34id_prepare as prep + + +def main(arg): + parser = argparse.ArgumentParser() + parser.add_argument("dev", help="processor to run on (cpu, opencl, cuda)") + parser.add_argument("prefix", help="prefix id") + parser.add_argument("scans", help="scans to preocess") + parser.add_argument("conf_dir", help="directory with configuration files") + args = parser.parse_args() + dev = args.dev + prefix = args.prefix + scans = args.scans + conf_dir = args.conf_dir + + experiment_dir = prep.parse_and_prepare(prefix, scans, conf_dir) + run_dt.data(experiment_dir) + run_rc.manage_reconstruction(dev, experiment_dir) + run_dp.to_vtk(experiment_dir) + + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/reccdi/src_py/run_scripts/everything.sh b/reccdi/src_py/run_scripts/everything.sh new file mode 100644 index 0000000..69b1fba --- /dev/null +++ b/reccdi/src_py/run_scripts/everything.sh @@ -0,0 +1,17 @@ +#!/bin/sh + + +#export LD_LIBRARY_PATH=/usr/local/lib:/local/libconfig/lib:/local/af/lib + +#export LD_LIBRARY_PATH=/home/beams/CXDUSER/CDI/libconfig/lib:/usr/lib64:/home/beams/CXDUSER/CDI/ArrayFire-v3.4.2/lib:/usr/local-linux/cuda-8.0/lib64:/usr/local-linux/cuda-8.0/nvvm/lib64 + +#newest arrayfire version will work with cuda 9.1 +#export LD_LIBRARY_PATH=/home/beams/CXDUSER/CDI/arrayfire/lib:/home/beams/CXDUSER/CDI/libconfig/lib:/local/cuda-9.0/lib64:/local/cuda-9.0/nvvm/lib64 + +dev=$1 +prefix=$2 +scans=$3 +conf_dir=$4 + +python everything.py $dev $prefix $scans $conf_dir + diff --git a/reccdi/src_py/run_scripts/run_34id_prepare.py b/reccdi/src_py/run_scripts/run_34id_prepare.py new file mode 100755 index 0000000..e230cd6 --- /dev/null +++ b/reccdi/src_py/run_scripts/run_34id_prepare.py @@ -0,0 +1,119 @@ +import argparse +import pylibconfig2 as cfg +import sys +import os +import reccdi.src_py.beamlines.aps_34id.prep as prep +import reccdi.src_py.utilities.parse_ver as ver +import shutil + + +def prepare(experiment_dir, scan_range, conf_file): + prep.prepare(experiment_dir, scan_range, conf_file) + + return experiment_dir + + +def copy_conf(src, dest): + try: + main_conf = os.path.join(src, 'config_prep') + shutil.copy(main_conf, dest) + conf_data = os.path.join(src, 'config_data') + shutil.copy(conf_data, dest) + conf_rec = os.path.join(src, 'config_rec') + shutil.copy(conf_rec, dest) + conf_disp = os.path.join(src, 'config_disp') + shutil.copy(conf_disp, dest) + except: + pass + + +def parse_and_prepare(prefix, scan, conf_dir): + id = prefix + '_' + scan + print ('reading data files for experiment ' + id) + + if not os.path.isdir(conf_dir): + print ('configured directory ' + conf_dir + ' does not exist') + return + + main_conf = os.path.join(conf_dir, 'config') + if not os.path.isfile(main_conf): + print ('the configuration directory does not contain "config" file') + return + + try: + # convert it to list of int + scan_range = scan.split('-') + scan_num = [] + for i in range(len(scan_range)): + scan_num.append(int(scan_range[i])) + except: + print ('enter numeric values for scan range') + return + + if not ver.ver_config_prep(main_conf): + return + + try: + with open(main_conf, 'r') as f: + config_map = cfg.Config(f.read()) + except Exception as e: + print ('Please check the configuration file ' + main_conf + '. Cannot parse ' + str(e)) + return + + try: + working_dir = config_map.working_dir.strip() + except: + working_dir = os.getcwd() + + experiment_dir = os.path.join(working_dir, id) + if not os.path.exists(experiment_dir): + os.makedirs(experiment_dir) + # copy config_data, config_rec, cofig_disp files from cofig directory into the experiment conf directory + experiment_conf_dir = os.path.join(experiment_dir, 'conf') + if not os.path.exists(experiment_conf_dir): + os.makedirs(experiment_conf_dir) + + experiment_main_config = os.path.join(experiment_conf_dir, 'config') + conf_map = {} + conf_map['working_dir'] = '"' + working_dir + '"' + conf_map['experiment_id'] = '"' + prefix + '"' + conf_map['scan'] = '"' + scan + '"' + temp_file = os.path.join(experiment_conf_dir, 'temp') + with open(temp_file, 'a') as f: + for key in conf_map: + value = conf_map[key] + if len(value) > 0: + f.write(key + ' = ' + conf_map[key] + '\n') + f.close() + if not ver.ver_config(temp_file): +# os.remove(temp_file) + print('please check the entered parameters. Cannot save this format') + else: + shutil.copy(temp_file, experiment_main_config) + os.remove(temp_file) + + copy_conf(conf_dir, experiment_conf_dir) + prep_conf = os.path.join(experiment_conf_dir, 'config_prep') + if os.path.isfile(prep_conf): + prep.prepare(experiment_dir, scan_num, prep_conf) + else: + print ('missing ' + prep_conf + ' file') + + return experiment_dir + + +def main(arg): + parser = argparse.ArgumentParser() + parser.add_argument("id", help="prefix to name of the experiment/data reconstruction") + parser.add_argument("scan", help="a range of scans to prepare data from") + parser.add_argument("conf_dir", help="directory where the configuration files are located") + args = parser.parse_args() + scan = args.scan + id = args.id + conf_dir = args.conf_dir + + return parse_and_prepare(id, scan, conf_dir) + + +if __name__ == "__main__": + exit(main(sys.argv[1:])) diff --git a/reccdi/src_py/run_scripts/run_data.py b/reccdi/src_py/run_scripts/run_data.py new file mode 100755 index 0000000..87064dd --- /dev/null +++ b/reccdi/src_py/run_scripts/run_data.py @@ -0,0 +1,29 @@ +import reccdi.src_py.controller.data as dt +import sys +import argparse +import os + + +def data(experiment_dir): + print ('formating data') + prep_file = os.path.join(experiment_dir, 'prep', 'prep_data.tif') + if os.path.isfile(prep_file): + dt.prep(prep_file, experiment_dir) + else: + dirs = os.listdir(experiment_dir) + for dir in dirs: + if dir.startswith('scan'): + scan_dir = os.path.join(experiment_dir, dir) + prep_file = os.path.join(scan_dir, 'prep', 'prep_data.tif') + dt.prep(prep_file, scan_dir) + + +def main(arg): + parser = argparse.ArgumentParser() + parser.add_argument("experiment_dir", help="experiment directory") + args = parser.parse_args() + data(args.experiment_dir) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/reccdi/src_py/run_scripts/run_rec.py b/reccdi/src_py/run_scripts/run_rec.py new file mode 100755 index 0000000..d0477a9 --- /dev/null +++ b/reccdi/src_py/run_scripts/run_rec.py @@ -0,0 +1,229 @@ +import sys +import signal +import os +import argparse +from multiprocessing import Process, Queue +import reccdi.src_py.controller.reconstruction as rec +import reccdi.src_py.controller.gen_rec as gen_rec +import reccdi.src_py.controller.reconstruction_multi as mult_rec +import reccdi.src_py.utilities.utils as ut +import reccdi.src_py.utilities.parse_ver as ver +import time +from functools import reduce + + +MEM_FACTOR = 1500 +ADJUST = 0.0 + +def interrupt_thread(): + """ + This function is part of interrupt mechanism. It detects ctl-c signal and creates an empty file named "stopfile". + The file is discovered by fast module and the discovery prompts termonation of the process. + """ + def int_handler(signal, frame): + while not os.path.isfile('stopfile'): + open('stopfile', 'a').close() + time.sleep(.3) + + # #remove the file at the end + if os.path.isfile('stopfile'): + os.remove('stopfile') + + def term_handler(signal, frame): + pass + + signal.signal(signal.SIGINT, int_handler) + signal.signal(signal.SIGTERM, term_handler) + signal.pause() + + +def rec_process(proc, conf_file, datafile, dir, gpus, r, q): + if r == 'g': + gen_rec.reconstruction(proc, conf_file, datafile, dir, gpus) + elif r == 'm': + mult_rec.reconstruction(proc, conf_file, datafile, dir, gpus) + elif r == 's': + rec.reconstruction(proc, conf_file, datafile, dir, gpus) + q.put((os.getpid(), gpus)) + + +def get_gpu_use(devices, no_dir, no_rec, data_size): + rec_mem_size = data_size / MEM_FACTOR + gpu_load = ut.get_gpu_load(rec_mem_size, devices) + no_runs = no_dir * no_rec + gpu_distribution = ut.get_gpu_distribution(no_runs, gpu_load) + gpu_use = [] + available = reduce((lambda x,y: x+y), gpu_distribution) + dev_index = 0 + i = 0 + while i < available: + if gpu_distribution[dev_index] > 0: + gpu_use.append(devices[dev_index]) + gpu_distribution[dev_index] = gpu_distribution[dev_index] -1 + i += 1 + dev_index += 1 + dev_index = dev_index % len(devices) + if no_dir > 1: + gpu_use = [gpu_use[x:x+no_rec] for x in range(0, len(gpu_use), no_rec)] + + return gpu_use + + +def manage_reconstruction(proc, experiment_dir, rec_id=None): + """ + This function starts the interruption discovery thread and the recontruction thread. + + It reads configuration file defined as /conf/config_rec. + If multiple generations are configured, it will start reconstruction from "reconstruction_multi" + script, otherwise from "reconstruction" script. + """ + if os.path.exists('stopfile'): + os.remove('stopfile') + print ('starting reconstruction') + # find how many reconstruction configurations are in config directory + # if more than one, it will run in separate processes + conf_dir = os.path.join(experiment_dir, 'conf') + if rec_id is None: + conf_file = os.path.join(conf_dir, 'config_rec') + else: + conf_file = os.path.join(conf_dir, rec_id + '_config_rec') + + # check if file exists + if not os.path.isfile(conf_file): + print ('no configuration file ' + conf_file + ' found') + return + + # verify the configuration file + if not ver.ver_config_rec(conf_file): + # if not verified, the ver will print message + return + + try: + config_map = ut.read_config(conf_file) + if config_map is None: + print("can't read configuration file " + conf_file) + return + except: + print('Cannot parse configuration file ' + conf_file + ' , check for matching parenthesis and quotations') + return + + exp_dirs_data = [] + # experiment may be multi-scan in which case will run a reconstruction for each scan + for dir in os.listdir(experiment_dir): + if dir.startswith('scan'): + datafile = os.path.join(experiment_dir, dir, 'data', 'data.tif') + if os.path.isfile(datafile): + exp_dirs_data.append((datafile, os.path.join(experiment_dir, dir))) + # if there are no scan directories, assume it is combined scans experiment + if len(exp_dirs_data) == 0: + # in typical scenario data_dir is not configured, and it is defaulted to /data + # the data_dir is ignored in multi-scan scenario + try: + data_dir = config_map.data_dir + except AttributeError: + data_dir = os.path.join(experiment_dir, 'data') + datafile = os.path.join(data_dir, 'data.tif') + if os.path.isfile(datafile): + exp_dirs_data.append((datafile, experiment_dir)) + no_runs = len(exp_dirs_data) + + try: + generations = config_map.generations + except: + generations = 0 + try: + reconstructions = config_map.reconstructions + except: + reconstructions = 1 + try: + devices = config_map.device + except: + devices = [-1] + + if (no_runs > 1 or reconstructions > 1) and devices[0] != -1: + from functools import reduce + # find size of data array + data_shape = ut.read_tif(exp_dirs_data[0][0]).shape + data_size = reduce((lambda x,y: x*y), data_shape) + gpu_use = get_gpu_use(devices, no_runs, reconstructions, data_size) + else: + gpu_use = devices + + if generations > 1: + r = gen_rec + elif reconstructions > 1: + r = mult_rec + else: + r = rec + + # start the interrupt process + interrupt_process = Process(target=interrupt_thread, args=()) + interrupt_process.start() + + if no_runs == 1: + dir_data = exp_dirs_data[0] + datafile = dir_data[0] + dir = dir_data[1] + r.reconstruction(proc, conf_file, datafile, dir, gpu_use) + else: + # check if is it worth to use last chunk + if len(gpu_use[0]) > len(gpu_use[-1])*2: + gpu_use = gpu_use[0:-1] + + if generations > 1: + r = 'g' + elif reconstructions > 1: + r = 'm' + else: + r = 's' + + q = Queue() + for gpus in gpu_use: + q.put((None, gpus)) + # index keeps track of the multiple directories + index = 0 + processes = {} + while index < no_runs: + pid, gpus = q.get() + if pid is not None: + os.kill(pid, signal.SIGKILL) + del processes[pid] + datafile = exp_dirs_data[index][0] + dir = exp_dirs_data[index][1] + p = Process(target = rec_process, args = (proc, conf_file, datafile, dir, gpus, r, q)) + p.start() + processes[p.pid] = index + index += 1 + + # close the queue + while len(processes.items()) > 0: + pid, gpus = q.get() + os.kill(pid, signal.SIGKILL) + del processes[pid] + q.close() + + interrupt_process.terminate() + print ('finished reconstruction') + + +def main(arg): + parser = argparse.ArgumentParser() + parser.add_argument("proc", help="the processor the code will run on, can be 'cpu', 'opencl', or 'cuda'.") + parser.add_argument("experiment_dir", help="experiment directory.") + parser.add_argument("--rec_id", help="reconstruction id, a prefix to '_results' directory") + args = parser.parse_args() + proc = args.proc + experiment_dir = args.experiment_dir + + if args.rec_id: + manage_reconstruction(proc, experiment_dir, args.rec_id) + else: + manage_reconstruction(proc, experiment_dir) + + +if __name__ == "__main__": + print (sys.argv[1:]) + main(sys.argv[1:]) + +#python run_rec.py opencl experiment_dir + diff --git a/reccdi/src_py/run_scripts/run_rec.sh b/reccdi/src_py/run_scripts/run_rec.sh new file mode 100644 index 0000000..895f2df --- /dev/null +++ b/reccdi/src_py/run_scripts/run_rec.sh @@ -0,0 +1,12 @@ +#!/bin/sh + + +export LD_LIBRARY_PATH=/usr/local/lib:/local/libconfig/lib:/local/af/lib + +#export LD_LIBRARY_PATH=/home/beams/CXDUSER/CDI/libconfig/lib:/usr/lib64:/home/beams/CXDUSER/CDI/ArrayFire-v3.4.2/lib:/usr/local-linux/cuda-8.0/lib64:/usr/local-linux/cuda-8.0/nvvm/lib64 + +#newest arrayfire version will work with cuda 9.1 +#export LD_LIBRARY_PATH=/home/beams/CXDUSER/CDI/arrayfire/lib:/home/beams/CXDUSER/CDI/libconfig/lib:/local/cuda-9.0/lib64:/local/cuda-9.0/nvvm/lib64 + +python run_rec.py $1 $2 +#python run_rec.py "opencl" \ No newline at end of file diff --git a/src_py/utilities/__init__.py b/reccdi/src_py/utilities/__init__.py similarity index 100% rename from src_py/utilities/__init__.py rename to reccdi/src_py/utilities/__init__.py diff --git a/reccdi/src_py/utilities/parse_ver.py b/reccdi/src_py/utilities/parse_ver.py new file mode 100644 index 0000000..bb030a7 --- /dev/null +++ b/reccdi/src_py/utilities/parse_ver.py @@ -0,0 +1,861 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + + +""" +verification of configuration files +""" + +import reccdi.src_py.utilities.utils as ut +import os + + +def ver_list_int(param_name, param_value): + if not issubclass(type(param_value), list): + print (param_name + ' is not a list') + return False + for e in param_value: + if type(e) != int: + print (param_name + ' should be list of integer values') + return False + return True + + +def ver_list_float(param_name, param_value): + if not issubclass(type(param_value), list): + print (param_name + ' is not a list') + return False + for e in param_value: + if type(e) != float: + print (param_name + ' should be list of float values') + return False + return True + + +def ver_config(fname): + + """ + This function verifies config file + + Parameters + ---------- + conf_info : str + configuration file + + Returns + ------- + True if configuration is correct, False otherwise + """ + if not os.path.isfile(fname): + print ('no configuration file ' + fname + ' found') + return False + + try: + config_map = ut.read_config(fname) + if config_map is None: + print ("can't read configuration file") + return False + except: + print ('Cannot parse ' + fname + ' configuration file. Check paranthesis and quotations.') + return False + + try: + working_dir = config_map.working_dir + if type(working_dir) != str: + print('working_dir parameter should be string') + return False + except AttributeError: + pass + except: + print ('working_dir parameter parsing error') + return False + + try: + experiment_id = config_map.experiment_id + if type(experiment_id) != str: + print('experiment_id parameter should be string') + return False + except AttributeError: + pass + except: + print ('experiment_id parameter parsing error') + return False + + try: + scan = config_map.scan + if type(scan) != str: + print('scan parameter should be string') + return False + except AttributeError: + pass + except: + print ('scan parameter parsing error') + return False + + return True + + +def ver_config_rec(fname): + + """ + This function verifies config_rec file + + Parameters + ---------- + conf_info : str + configuration file + + Returns + ------- + True if configuration is correct, False otherwise + """ + if not os.path.isfile(fname): + print ('no configuration file ' + fname + ' found') + return False + + try: + config_map = ut.read_config(fname) + if config_map is None: + print ("can't read configuration file") + return False + except: + print ('Cannot parse ' + fname + ' configuration file. Check paranthesis and quotations.') + return False + + try: + data_dir = config_map.data_dir + if type(data_dir) != str: + print('data_dir parameter should be string') + return False + except AttributeError: + pass + except: + print ('data_dir parameter parsing error') + return False + + try: + save_dir = config_map.save_dir + if type(save_dir) != str: + print('save_dir parameter should be string') + return False + except AttributeError: + pass + except: + print ('save_dir parameter parsing error') + return False + + try: + cont = config_map.cont + if type(cont) != bool: + print ('cont parameter should be true or false') + return False + try: + continue_dir = config_map.continue_dir + if type(continue_dir) != str: + print('continue_dir parameter should be string') + return False + except AttributeError: + pass + except: + print('continue_dir parameter parsing error') + return False + except AttributeError: + pass + except: + print ('cont parameter parsing error') + return False + + try: + reconstructions = config_map.reconstructions + if type(reconstructions) != int: + print('reconstructions parameter should be int') + return False + except AttributeError: + pass + except: + print ('reconstructions parameter parsing error') + return False + + try: + device = config_map.device + if not ver_list_int('device', device): + return False + except AttributeError: + pass + except: + print ('device parameter parsing error') + return False + + try: + garbage_trigger = config_map.garbage_trigger + if not ver_list_int('garbage_trigger', garbage_trigger): + return False + except AttributeError: + pass + except: + print ('garbage_trigger parameter parsing error') + return False + + try: + algorithm_sequence = config_map.algorithm_sequence + if not issubclass(type(algorithm_sequence), list): + print ('algorithm_sequence should be a list') + return False + for s in algorithm_sequence: + for i in range(len(s)): + # the first element in each sub-list is the repeat factor and should be int + if i== 0 and type(s[i]) != int: + print ('algorithm_sequence configuration error, the repeat factor should be int') + return False + if i > 0: + if not issubclass(type(s[i]), list): + print ('algorithm_sequence configuration error, the sequence element should be a list') + return False + algorithm = s[i][0] + if type(algorithm) != str: + print ('algorithm_sequence configuration error, algorithm should be str') + return False + algorithm_options = ["ER", "HIO"] + if algorithm not in algorithm_options: + print ('algorithm_sequence configuration error, algorithm should be "ER" or "HIO"') + return False + algorithm_repeat = s[i][1] + if type(algorithm_repeat) != int: + print ('algorithm_sequence configuration error, algorithm repeat should be int') + return False + except AttributeError: + print ('missing mandatory algorithm_sequence parameter') + return False + except: + print ('algorithm_sequence parameter parsing error') + return False + + try: + beta = config_map.beta + if type(beta) != float: + print('beta parameter should be float') + return False + except AttributeError: + pass + except: + print ('beta parameter parsing error') + return False + + try: + generations = config_map.generations + if type(generations) != int: + print('generations parameter should be int') + return False + try: + ga_metrics = config_map.ga_metrics + if not issubclass(type(ga_metrics), list): + print (ga_metrics + ' is not a list') + return False + metrics_options = ['chi', 'sharpness', 'summed_phase', 'area'] + for metric in ga_metrics: + if metric not in metrics_options: + print ("ga_metrics list can include only following strings: 'chi', 'sharpness', 'summed_phase', 'area'") + except AttributeError: + pass + except: + print('ga_metrics parameter parsing error') + return False + + try: + ga_breed_modes = config_map.ga_breed_modes + if not issubclass(type(ga_breed_modes), list): + print (ga_breed_modes + ' is not a list') + return False + breed_options = ['sqrt_ab', 'max_all', 'Dhalf', 'Dhalf-best', 'dsqrt', 'pixel_switch',\ + 'b_pa', '2ab_a_b', '2a-b_pa', 'sqrt_ab_pa', 'sqrt_ab_pa_recip', 'sqrt_ab_recip',\ + 'max_ab', 'max_ab_pa', 'min_ab_pa', 'avg_ab', 'avg_ab_pa', 'sqrt_abg', + 'sqrt_abg_pa', 'max_abg','max_abg_pa', 'avg_abg', 'avg_abg_pa', 'avg_sqrt'] + for breed in ga_breed_modes: + if breed not in breed_options: + print ("ga_breed_modes list can include only following strings: 'sqrt_ab', 'max_all',\ + 'Dhalf', 'Dhalf-best', 'dsqrt', 'pixel_switch',\ + 'b_pa', '2ab_a_b', '2a-b_pa', 'sqrt_ab_pa', 'sqrt_ab_pa_recip', 'sqrt_ab_recip',\ + 'max_ab', 'max_ab_pa', 'min_ab_pa', 'avg_ab', 'avg_ab_pa', 'sqrt_abg',\ + 'sqrt_abg_pa', 'max_abg','max_abg_pa', 'avg_abg', 'avg_abg_pa', 'avg_sqrt'") + except AttributeError: + pass + except: + print('ga_breed_modes parameter parsing error') + return False + + try: + ga_cullings = config_map.ga_cullings + if not ver_list_int('ga_cullings', ga_cullings): + return False + except AttributeError: + pass + except: + print('ga_cullings parameter parsing error') + return False + + try: + ga_support_thresholds = config_map.ga_support_thresholds + if not ver_list_float('ga_support_thresholds', ga_support_thresholds): + return False + except AttributeError: + pass + except: + print('ga_support_thresholds parameter parsing error') + return False + + try: + ga_support_sigmas = config_map.ga_support_sigmas + if not ver_list_float('ga_support_sigmas', ga_support_sigmas): + return False + except AttributeError: + pass + except: + print('ga_support_sigmas parameter parsing error') + return False + + try: + ga_low_resolution_sigmas = config_map.ga_low_resolution_sigmas + if not ver_list_float('ga_low_resolution_sigmas', ga_low_resolution_sigmas): + return False + except AttributeError: + pass + except: + print('ga_low_resolution_sigmas parameter parsing error') + return False + except AttributeError: + pass + except: + print ('generations parameter parsing error') + return False + + + try: + twin_trigger = config_map.twin_trigger + if not ver_list_int('twin_trigger', twin_trigger): + return False + else: + try: + twin_halves = config_map.twin_halves + if not ver_list_int('twin_halves', twin_halves): + return False + except AttributeError: + pass + except: + print('twin_halves parameter parsing error') + return False + + except AttributeError: + pass + + try: + if not ver_list_int('amp_support_trigger', config_map.amp_support_trigger): + return False + else: + try: + support_type = config_map.support_type + if type(support_type) != str: + print ('support_type parameter should be string') + return False + if support_type != "GAUSS": + print ('support_type parameter can be configured "GAUSS"') + return False + except AttributeError: + pass + except: + print('support_type parameter parsing error') + return False + + try: + support_threshold = config_map.support_threshold + if type(support_threshold) != float: + print('support_threshold should be float') + return False + except AttributeError: + pass + except: + print('support_threshold parameter parsing error') + return False + + try: + support_sigma = config_map.support_sigma + if type(support_sigma) != float: + print('support_sigma should be float') + return False + except AttributeError: + pass + except: + print('support_sigma parameter parsing error') + return False + + try: + support_area = config_map.support_area + if not issubclass(type(support_area), list): + print('support_area should be list') + return False + for e in support_area: + if type(e) != int and type(e) !=float: + print('support_area should be a list of int or float') + return False + except AttributeError: + pass + except: + print('support_area parameter parsing error') + return False + + except AttributeError: + pass + + try: + if not ver_list_int('phase_support_trigger', config_map.phase_support_trigger): + return False + else: + try: + phase_min = config_map.phase_min + if type(phase_min) != float: + print('phase_min should be float') + return False + except AttributeError: + pass + except: + print('phase_min parameter parsing error') + return False + + try: + phase_max = config_map.phase_max + if type(phase_max) != float: + print('phase_max should be float') + return False + except AttributeError: + pass + except: + print('phase_max parameter parsing error') + return False + + except AttributeError: + pass + + try: + if not ver_list_int('pcdi_trigger', config_map.pcdi_trigger): + return False + else: + try: + partial_coherence_type = config_map.partial_coherence_type + if type(partial_coherence_type) != str: + print ('partial_coherence_type parameter should be string') + return False + if partial_coherence_type != "LUCY": + print ('partial_coherence_type parameter can be configured "LUCY"') + return False + except AttributeError: + pass + except: + print('partial_coherence_type parameter parsing error') + return False + + try: + partial_coherence_iteration_num = config_map.partial_coherence_iteration_num + if type(partial_coherence_iteration_num) != int: + print('partial_coherence_iteration_num should be int') + return False + except AttributeError: + pass + except: + print('partial_coherence_iteration_num parameter parsing error') + return False + + try: + partial_coherence_normalize = config_map.partial_coherence_normalize + if type(partial_coherence_normalize) != bool: + print ('partial_coherence_normalize parameter should be true or false') + return False + except AttributeError: + pass + except: + print('partial_coherence_normalize parameter parsing error') + return False + + try: + partial_coherence_roi = config_map.partial_coherence_roi + if not ver_list_int('partial_coherence_roi', partial_coherence_roi): + return False + except AttributeError: + pass + except: + print("'partial_coherence_roi' parameter parsing error") + return False + + except AttributeError: + pass + + try: + if not ver_list_int('resolution_trigger', config_map.resolution_trigger): + return False + else: + try: + iter_res_sigma_range = config_map.iter_res_sigma_range + if not ver_list_float('iter_res_sigma_range', iter_res_sigma_range): + return False + except AttributeError: + pass + except: + print("'iter_res_sigma_range' parameter parsing error") + return False + + try: + iter_res_det_range = config_map.iter_res_det_range + if not ver_list_float('iter_res_det_range', iter_res_det_range): + return False + except AttributeError: + pass + except: + print("'iter_res_det_range' parameter parsing error") + return False + + except AttributeError: + pass + + try: + if not ver_list_int('average_trigger', config_map.average_trigger): + return False + except AttributeError: + pass + + try: + if not ver_list_int('progress_trigger', config_map.progress_trigger): + return False + except AttributeError: + pass + + return True + + +def ver_config_data(fname): + + """ + This function verifies config_data file + + Parameters + ---------- + conf_info : str + configuration file + + Returns + ------- + True if configuration is correct, False otherwise + """ + if not os.path.isfile(fname): + print ('no configuration file ' + fname + ' found') + return False + + try: + config_map = ut.read_config(fname) + if config_map is None: + print ("can't read configuration file") + return False + except: + print ('Cannot parse ' + fname + ' configuration file. Check paranthesis and quotations.') + return False + + try: + data_dir = config_map.data_dir + if type(data_dir) != str: + print('data_dir parameter should be string') + return False + except AttributeError: + pass + except: + print ('data_dir parameter parsing error') + return False + + try: + if not ver_list_int('pad_crop', config_map.adjust_dimensions): + return False + except AttributeError: + pass + try: + if not ver_list_int('center_shift', config_map.center_shift): + return False + except AttributeError: + pass + try: + if not ver_list_int('binning', config_map.binning): + return False + except AttributeError: + pass + + try: + amp_threshold = config_map.amp_threshold + if type(amp_threshold) != float and type(amp_threshold) != int: + print('amp_threshold should be float') + return False + except AttributeError: + pass + except: + print('amp_threshold parameter parsing error') + return False + + try: + aliens = config_map.aliens + if not issubclass(type(aliens), list): + print('aliens should be a list of aliens(lists)') + return False + for a in aliens: + if not issubclass(type(a), list): + print ('aliens should be a list of aliens(lists)') + return False + if not ver_list_int('aliens', a): + return False + if (len(a) < 6): + print('each alien is defined by list of six int') + except AttributeError: + pass + except: + print('amp_threshold parameter parsing error') + return False + + return True + + +def ver_config_prep(fname): + + """ + This function verifies config_prep file + + Parameters + ---------- + conf_info : str + configuration file + + Returns + ------- + True if configuration is correct, False otherwise + """ + if not os.path.isfile(fname): + print ('no configuration file ' + fname + ' found') + return False + + try: + config_map = ut.read_config(fname) + if config_map is None: + print ("can't read configuration file") + return False + except: + print ('Cannot parse ' + fname + ' configuration file. Check paranthesis and quotations.') + return False + + try: + data_dir = config_map.data_dir + if type(data_dir) != str: + print('data_dir parameter should be string') + return False + except AttributeError: + pass + except: + print ('data_dir parameter parsing error') + return False + + try: + specfile = config_map.specfile + if type(specfile) != str: + print('specfile parameter should be string') + return False + except AttributeError: + pass + except: + print ('specfile parameter parsing error') + return False + + try: + darkfile = config_map.darkfile + if type(darkfile) != str: + print('darkfile parameter should be string') + return False + except AttributeError: + pass + except: + print ('darkfile parameter parsing error') + return False + + try: + whitefile = config_map.whitefile + if type(whitefile) != str: + print('whitefile parameter should be string') + return False + except AttributeError: + pass + except: + print ('whitefile parameter parsing error') + return False + + try: + if not ver_list_int('exclude_scans', config_map.exclude_scans): + return False + except AttributeError: + pass + + try: + min_files = config_map.min_files + if type(min_files) != int: + print('min_files should be int') + return False + except AttributeError: + pass + except: + print('min_files parameter parsing error') + return False + + try: + det_quad = config_map.det_quad + quad_options = [0,1,2,3,4] + if det_quad not in quad_options: + print('det_quad should be one of the following: 0, 1, 2, 3, 4') + return False + except AttributeError: + pass + except: + print('det_quad parameter parsing error') + return False + + try: + separate_scans = config_map.separate_scans + if type(separate_scans) != bool: + print('separate_scans parameter should be true or false') + return False + except AttributeError: + pass + except: + print('separate_scans parameter parsing error') + return False + + return True + + +def ver_config_disp(fname): + + """ + This function verifies config_disp file + + Parameters + ---------- + conf_info : str + configuration file + + Returns + ------- + True if configuration is correct, False otherwise + """ + if not os.path.isfile(fname): + print ('no configuration file ' + fname + ' found') + return False + + try: + config_map = ut.read_config(fname) + if config_map is None: + print ("can't read configuration file") + return False + except: + print ('Cannot parse ' + fname + ' configuration file. Check paranthesis and quotations.') + return False + + try: + crop = config_map.crop + if not issubclass(type(crop), list): + print('crop should be list') + return False + for e in crop: + if type(e) != int and type(e) != float: + print('crop should be a list of int or float') + return False + except AttributeError: + pass + except: + print('crop parameter parsing error') + return False + + try: + specfile = config_map.specfile + if type(specfile) != str: + print('specfile parameter should be string') + return False + except AttributeError: + pass + except: + print ('specfile parameter parsing error') + return False + + try: + energy = config_map.energy + if type(energy) != float: + print('energy should be float') + return False + except AttributeError: + pass + except: + print('energy parameter parsing error') + return False + + try: + delta = config_map.delta + if type(delta) != float: + print('delta should be float') + return False + except AttributeError: + pass + except: + print('delta parameter parsing error') + return False + + try: + gamma = config_map.gamma + if type(gamma) != float: + print('gamma should be float') + return False + except AttributeError: + pass + except: + print('gamma parameter parsing error') + return False + + try: + arm = config_map.arm + if type(arm) != float: + print('arm should be float') + return False + except AttributeError: + pass + except: + print('arm parameter parsing error') + return False + + try: + dth = config_map.dth + if type(dth) != float: + print('dth should be float') + return False + except AttributeError: + pass + except: + print('dth parameter parsing error') + return False + + try: + pixel = config_map.pixel + if not issubclass(type(pixel), list): + print('pixel should be a list') + return False + if type(pixel[0]) != float or type(pixel[1]) != float: + print('pixel values should be float') + return False + except AttributeError: + pass + except: + print('pixel parameter parsing error') + return False + + return True diff --git a/reccdi/src_py/utilities/prep_noconfig.py b/reccdi/src_py/utilities/prep_noconfig.py new file mode 100644 index 0000000..0d6cc36 --- /dev/null +++ b/reccdi/src_py/utilities/prep_noconfig.py @@ -0,0 +1,216 @@ +import numpy as np +import copy +import scipy.fftpack as sf +import os +import glob +import tifffile as tif +#import reccdi.src_py.utilities.spec as spec +#import reccdi.src_py.utilities.utils as ut + + +def get_dir_list(scans, map): + """ + Returns list of sub-directories in data_dir with names matching range of scans + It will exclude scans within exclude_scans list if provided, and directories with fewer files than + min_files, if provided + :param scans: + :param map: + :return: + """ + try: + min_files = map.min_files + except: + min_files = 0 + try: + exclude_scans = map.exclude_scans + except: + exclude_scans = [] + try: + data_dir = map.data_dir + except: + print ('please provide data_dir') + + dirs = [] + for name in os.listdir(data_dir): + subdir = os.path.join(data_dir, name) + if os.path.isdir(subdir): + # exclude directories with fewer tif files than min_files + if len(glob.glob1(subdir, "*.tif")) < min_files and len(glob.glob1(subdir, "*.tiff")) < min_files: + continue + try: + index = int(name[-4:]) + if index >= scans[0] and index <= scans[1] and not index in exclude_scans: + dirs.append(subdir) + except: + continue + return dirs + + +def get_dark_white(darkfile, whitefile, det_area1, det_area2): + if darkfile is not None: + # find the darkfield array + dark_full = tif.imread(darkfile).astype(float) + # crop the corresponding quad or use the whole array, depending on what info was parsed from spec file + dark = dark_full[slice(det_area1[0], det_area1[1]), slice(det_area2[0], det_area2[1])] + else: + dark = None + + if whitefile is not None: + # find the whitefield array + white_full = tif.imread(whitefile).astype(float) + # crop the corresponding quad or use the whole array, depending on what info was parsed from spec file + white = white_full[slice(det_area1[0], det_area1[1]), slice(det_area2[0], det_area2[1])] + # set the bad pixels to some large value + white = np.where(white<5000, 1e20, white) #Some large value + else: + white = None + + return dark, white + + +def get_normalized_slice(file, dark, white): + # file is a tuple of slice and either background slice or None + slice = tif.TiffFile(file[0]).asarray() + if file[1] is not None: + slice = slice - tif.TiffFile(file[1]).asarray() + if dark is not None: + slice = np.where(dark > 5, 0, slice) #Ignore cosmic rays + # here would be code for correction for dead time + if white is not None: + slice = slice/white + slice *= 1e5 #Some medium value + slice = np.where(np.isnan(slice), 0, slice) + return slice + + +def read_scan(dir, dark, white): + files = [] + files_dir = {} + for file in os.listdir(dir): + if file.endswith('tif') or file.endswith('tiff'): + temp = file.split('.') + #it's assumed that the files end with four digits and 'tif' or 'tiff' extension + key = temp[0][-4:] + files_dir[key] = file + + ordered_keys = sorted(list(files_dir.keys())) + + for key in ordered_keys: + file = files_dir[key] + file = os.path.join(dir, file) + bg_file = file.replace('.tif', '_bg.tif') + if not os.path.isfile(bg_file): + bg_file = None + + files.append((file, bg_file)) + + # look at slice0 to find out shape + n = 0 + slice0 = get_normalized_slice(files[n], dark, white).transpose() + shape = (slice0.shape[0], slice0.shape[1], len(files)) + arr = np.zeros(shape, dtype=slice0.dtype) + arr[:,:,0] = slice0 + print("slice shape",slice0.shape) + + #for i in range (1, len(files)): + for file in files[1:]: + n = n + 1 + slice = get_normalized_slice(file, dark, white).transpose() + arr[:,:,n] = slice + return arr + + +def shift(arr, shifty): + # pass the FT of the fftshifted array you want to shift + # you get back the actual array, not the FT. + dims = arr.shape + # scipy does normalize ffts! + ftarr = sf.fftn(arr) + r=[] + for d in dims: + r.append(slice(int(np.ceil(-d/2.)), int(np.ceil(d/2.)), None)) + idxgrid = np.mgrid[r] + for d in range(len(dims)): + ftarr *= np.exp(-1j*2*np.pi*shifty[d]*sf.fftshift(idxgrid[d])/float(dims[d])) + + shiftedarr = sf.ifftn(ftarr) + return shiftedarr + + +def combine_part(part_f, slice_sum, refpart, part): + # get cross correlation and pixel shift + cross_correlation = sf.ifftn(refpart*np.conj(part_f)) + corelated = np.array(cross_correlation.shape) + amp = np.abs(cross_correlation) + intshift = np.unravel_index(amp.argmax(), corelated) + shifted = np.array(intshift) + pixelshift = np.where(shifted>=corelated/2, shifted-corelated, shifted) + return slice_sum + shift(part, pixelshift) + + +def fit(arr, det_area1, det_area2): + # if the full sensor was used for the image (i.e. the data size is 512x512) + # the quadrants need to be shifted + if det_area1[0] == 0 and det_area1[1] == 512 and det_area1[0] == 0 and det_area2[1] == 512: + b = np.zeros((arr.shape[0],517,516),float) + b[:,:256,:256] = arr[:,:256,:256] #Quad top left unchanged + b[:,:256,260:] = arr[:,:256,256:] #Quad top right moved 4 right + b[:,261:,:256] = arr[:,256:,:256] #Quad bot left moved 6 down + b[:,261:,260:] = arr[:,256:,256:] #Quad bot right + else: + b = arr + return b + + +def prep_data(experiment_dir, scans, map, det_area1, det_area2, *args): + if scans is None: + print ('scan info not provided') + return + + # build sub-directories map + if len(scans) == 1: + scans.append(scans[0]) + dirs = get_dir_list(scans, map) + + try: + whitefile = map.whitefile + except: + whitefile = None + + try: + darkfile = map.darkfile + except: + darkfile = None + + dark, white = get_dark_white(darkfile, whitefile, det_area1, det_area2) + + if len(dirs) == 0: + print ('there are no data directories for given scans') + return + + if len(dirs) == 1: + arr = read_scan(dirs[0], dark, white) + else: + # make the first part a reference + part = read_scan(dirs[0], dark, white) + slice_sum = np.abs(copy.deepcopy(part)) + refpart = sf.fftn(part) + for i in range (1, len(dirs)): + #this will load scans from each directory into an array part + part = read_scan(dirs[i], dark, white) + # add the arrays together + part_f = sf.fftn(part) + slice_sum = combine_part(part_f, slice_sum, refpart, part) + arr = np.abs(slice_sum).astype(np.int32) + + arr = fit(arr, det_area1, det_area2) + + #create directory to save prepared data ,/prep + prep_data_dir = os.path.join(experiment_dir, 'prep') + if not os.path.exists(prep_data_dir): + os.makedirs(prep_data_dir) + data_file = os.path.join(prep_data_dir, 'prep_data.tif') + + ut.save_tif(arr, data_file) + print ('done with prep, shape:', arr.shape) + diff --git a/reccdi/src_py/utilities/simple.py b/reccdi/src_py/utilities/simple.py new file mode 100644 index 0000000..41ce888 --- /dev/null +++ b/reccdi/src_py/utilities/simple.py @@ -0,0 +1,20 @@ +#! /usr/bin/env python +from pyevtk.hl import gridToVTK +import numpy as np + +# Dimensions +nx, ny, nz = 11, 11, 11 + +X = np.linspace(1., -1., nx) +Y = np.linspace(-1., 1., ny) +Z = np.linspace(-1., 1., nz) + +x, y, z = np.meshgrid(X, Y, Z, indexing='ij') + +r = np.sqrt(x**2 + y**2 + z**2) + +gridToVTK('./structured', x, y, z, pointData={'r': r, 'x': x}) + +r = 1.-np.sqrt(x**2 + y**2 + z**2) + +gridToVTK('./structured2', x, y, z, pointData={'r': r, 'x': x}) diff --git a/reccdi/src_py/utilities/tools.py b/reccdi/src_py/utilities/tools.py new file mode 100644 index 0000000..aa13a3b --- /dev/null +++ b/reccdi/src_py/utilities/tools.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module is a suite of utility mehods. +""" + +import tifffile as tf +import numpy as np +import os +import logging +import stat + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['read_tif', + 'get_opencl_dim', + 'binning', + 'get_centered', + 'adjust_dimensions', + 'crop_center', + 'flip'] + + +def get_logger(name, ldir=''): + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + log_file = os.path.join(ldir, 'default.log') + fh = logging.FileHandler(log_file) + fh.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + return logger + + +def read_tif(filename): + """ + This method reads tif type file containing experiment data and returns the data as array. + Parameters + ---------- + filename : str + a filename containing the experiment data + Returns + ------- + data : array + an array containing the experiment data + """ + + ar = tf.imread(filename) + ar = np.swapaxes(ar, 0, 2) +# ar = np.swapaxes(ar, 0, 1) + return ar + + +def save_tif(arr, tif_file): + arr = np.swapaxes(arr, 0, 2) + arr = np.swapaxes(arr, 1, 2) + tf.imsave(tif_file, arr.astype(np.int32)) + + +def get_good_dim(dim): + """ + This function calculates the dimension supported by opencl library (i.e. is multiplier of 2,3, or 5) and is closest to the + given starting dimension, and spaced by the given step. + If the dimension is not supported the function adds step value and verifies the new dimension. It iterates until it finds + supported value. + Parameters + ---------- + dim : int + a dimension that needs to be tranformed to one that is supported by the opencl library, if it is not already + + step : int + a delta to increase the dimension + Returns + ------- + dim : int + a dimension that is supported by the opencl library, and closest to the original dimension by n*step + """ + + def is_correct(x): + sub = x + if sub % 3 == 0: + sub = sub / 3 + if sub % 3 == 0: + sub = sub / 3 + if sub % 5 == 0: + sub = sub / 5 + while sub % 2 == 0: + sub = sub / 2 + return sub == 1 + + new_dim = dim + if new_dim % 2 == 1: + new_dim += 1 + while not is_correct(new_dim): + new_dim += 2 + return new_dim + + +def get_opencl_dim1(dim, step): + """ + This function calculates the dimension supported by opencl library (i.e. is multiplier of 2,3, or 5) and is closest to the + given starting dimension, and spaced by the given step. + If the dimension is not supported the function adds step value and verifies the new dimension. It iterates until it finds + supported value. + Parameters + ---------- + dim : int + a dimension that needs to be tranformed to one that is supported by the opencl library, if it is not already + + step : int + a delta to increase the dimension + Returns + ------- + dim : int + a dimension that is supported by the opencl library, and closest to the original dimension by n*step + """ + + def is_correct(x): + sub = x + while sub % 2 == 0: + sub = sub / 2 + while sub % 3 == 0: + sub = sub / 3 + while sub % 5 == 0: + sub = sub / 5 + return sub == 1 + + new_dim = dim + while not is_correct(new_dim): + new_dim += step + return new_dim + + +def binning(array, binsizes): + """ + This function does the binning of the array. The array is binned in each dimension by the corresponding binsizes elements. + If binsizes list is shorter than the array dimensions, the remaining dimensions are not binned. The elements in + a bucket are summed. + Parameters + ---------- + array : array + the original array to be binned + + binsizes : list + a list defining binning buckets for corresponding dimensions + Returns + ------- + array : array + the binned array + """ + + data_dims = array.shape + # trim array + for ax in range(len(binsizes)): + cut_slices = range(data_dims[ax] - data_dims[ax] % binsizes[ax], data_dims[ax]) + array = np.delete(array, cut_slices, ax) + + binned_array = array + new_shape = list(array.shape) + + for ax in range(len(binsizes)): + if binsizes[ax] > 1: + new_shape[ax] = binsizes[ax] + new_shape.insert(ax, int(array.shape[ax] / binsizes[ax])) + binned_array = np.reshape(binned_array, tuple(new_shape)) + binned_array = np.sum(binned_array, axis=ax + 1) + new_shape = list(binned_array.shape) + return binned_array + + +# ar = np.asarray([1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9]) +# ar.resize((5,9)) +# print ('ar', ar) +# b = binning(ar, (2,2)) +# print ('b',b) +# c = binning1(ar,(2,2)) +# print ('c',c) + + +def get_centered(arr, center_shift): + """ + This function finds a greatest value in the array, and puts it in a center of a new array. If center_shift is + not zeros, the array will be shifted accordingly. The shifted elements are rolled into the other end of array. + Parameters + ---------- + arr : array + the original array to be centered + center_shift : list + a list defining shift of the center + Returns + ------- + array : array + the centered array + """ + max_coordinates = list(np.unravel_index(np.argmax(arr), arr.shape)) + max_coordinates = np.add(max_coordinates, center_shift) + shape = arr.shape + centered = arr + for i in range(len(max_coordinates)): + centered = np.roll(centered, int(shape[i] / 2) - max_coordinates[i], i) + + return centered + + +def get_centered_both(arr, support): + """ + This function finds a greatest value in the array, and puts it in a center of a new array. If center_shift is + not zeros, the array will be shifted accordingly. The shifted elements are rolled into the other end of array. + Parameters + ---------- + arr : array + the original array to be centered + support : array + the associated array shifted the same way centered array is + Returns + ------- + centered : array + the centered array + """ + max_coordinates = list(np.unravel_index(np.argmax(arr), arr.shape)) + shape = arr.shape + centered = arr + centered_supp = support + for i in range(len(max_coordinates)): + centered = np.roll(centered, int(shape[i] / 2) - max_coordinates[i], i) + centered_supp = np.roll(centered_supp, int(shape[i] / 2) - max_coordinates[i], i) + + return centered, centered_supp + + +def get_zero_padded_centered(arr, new_shape): + """ + This function pads the array with zeros to the new shape with the array in the center. + Parameters + ---------- + arr : array + the original array to be padded and centered + new_shape : tuple + a list of new dimensions + Returns + ------- + array : array + the zero padded centered array + """ + shape = arr.shape + pad = [] + c_vals = [] + for i in range(len(new_shape)): + pad.append((0, new_shape[i] - shape[i])) + c_vals.append((0.0, 0.0)) + arr = np.lib.pad(arr, (pad), 'constant', constant_values=c_vals) + + centered = arr + for i in range(len(new_shape)): + centered = np.roll(centered, int((new_shape[i] - shape[i] + 1) / 2), i) + + return centered + + +def adjust_dimensions(arr, pads): + """ + This function adds to or subtracts from each dimension of the array elements defined by pad. If the pad is positive, + the array is padded in this dimension. If the pad is negative, the array is cropped. + The dimensions of the new array are supported by the opencl library. + Parameters + ---------- + arr : array + the array to pad/crop + pad : list + list of three pad values, for each dimension + Returns + ------- + array : array + the padded/cropped array + """ + # logger = get_logger('adjust_dimensions') + old_dims = arr.shape + start = [] + stop = [] + for i in range(len(old_dims)): + pad = pads[i] + first = max(0, -pad[0]) + last = old_dims[i] - max(0, -pad[1]) + if first >= last: + print ('the crop exceeds size, please change the crop and run again') + return None + else: + start.append(first) + stop.append(last) + + cropped = arr[ start[0]:stop[0], start[1]:stop[1], start[2]:stop[2] ] + # logger.info('cutting from to ' + str(crop[0]) + ', ' + str(old_dims[0]-crop[1]) + ', ' + str(crop[2]) + ', ' \ + # + str(old_dims[1]-crop[3]) + ', ' + str(crop[4]) + ', ' + str(old_dims[2]-crop[5])) + dims = cropped.shape + c_vals = [] + new_pad = [] + for i in range(len(dims)): + pad = pads[i] + # find a good dimension and find padding + temp_dim = old_dims[i] + pad[0] + pad[1] + new_dim = get_good_dim(temp_dim) + added = new_dim - temp_dim + # if the pad is positive + pad_front = max(0, pad[0]) + int(added / 2) + pad_end = new_dim - dims[i] - pad_front + new_pad.append((pad_front, pad_end)) + c_vals.append((0.0, 0.0)) + adjusted = np.pad(cropped, new_pad, 'constant', constant_values=c_vals) + + # logger.info('pads ' + str(new_pad[0]) + ', ' + str(new_pad[1]) + ', ' + str(new_pad[2]) + ', ' + str(new_pad[3]) \ + # + ', ' + str(new_pad[4]) + ', ' + str(new_pad[5])) + # logger.info('old dim, new dim (' + str(dims[0]) + ',' + str(dims[1]) + ',' + str(dims[2]) + ') (' + str(arr.shape[0]) +\ + # ',' + str(arr.shape[1]) + ',' + str(arr.shape[1]) + ')') + + return adjusted + +# ar = np.zeros((256,256,90)) +# pads = (-100,-100,0,0,0,0) +# ar = np.zeros((81,256,256)) +# pads = (5,-7,-20,-30,4,-20) +# arr = adjust_dimensions(ar,pads) +# print (arr.shape) + +def crop_center(arr, new_size): + size = arr.shape + cropped = arr + for i in range(len(size)): + crop_front = int((size[i] - new_size[i]) / 2) + crop_end = crop_front + new_size[i] + splitted = np.split(cropped, [crop_front, crop_end], axis=i) + cropped = splitted[1] + + return cropped + + +# ar = np.zeros((81,256,256)) +# new_size = (40, 200,100) +# arr = crop_center(ar,new_size) +# print (arr.shape) + +def get_norm(arr): + return sum(sum(sum(abs(arr) ** 2))) + + +def flip(m, axis): + """ + Copied from numpy 1.12.0. + """ + if not hasattr(m, 'ndim'): + m = np.asarray(m) + indexer = [slice(None)] * m.ndim + try: + indexer[axis] = slice(None, None, -1) + except IndexError: + raise ValueError("axis=%i is invalid for the %i-dimensional input array" + % (axis, m.ndim)) + return m[tuple(indexer)] + + +def gaussian(shape, sigmas, alpha=1): + grid = np.full(shape, 1.0) + for i in range(len(shape)): + # prepare indexes for tile and transpose + tile_shape = list(shape) + tile_shape.pop(i) + tile_shape.append(1) + trans_shape = list(range(len(shape) - 1)) + trans_shape.insert(i, len(shape) - 1) + + multiplier = - 0.5 * alpha / pow(sigmas[i], 2) + line = np.linspace(-(shape[i] - 1) / 2.0, (shape[i] - 1) / 2.0, shape[i]) + gi = np.tile(line, tile_shape) + gi = np.transpose(gi, tuple(trans_shape)) + exponent = np.power(gi, 2) * multiplier + gi = np.exp(exponent) + grid = grid * gi + + grid_total = np.sum(grid) + return grid / grid_total + + +def gauss_conv_fft(arr, sigmas): + arr_sum = np.sum(abs(arr)) + arr_f = np.fft.ifftshift(np.fft.fftn(np.fft.ifftshift(arr))) + shape = list(arr.shape) + for i in range(len(sigmas)): + sigmas[i] = shape[i] / 2.0 / np.pi / sigmas[i] + convag = arr_f * gaussian(shape, sigmas) + convag = np.fft.ifftshift(np.fft.ifftn(np.fft.ifftshift(convag))) + convag = convag.real + convag = np.clip(convag, 0, None) + correction = arr_sum / np.sum(convag) + convag *= correction + return convag + + +def shrink_wrap(arr, threshold, sigma, type='gauss'): + sigmas = [sigma] * len(arr.shape) + if type == 'gauss': + convag = gauss_conv_fft(abs(arr), sigmas) + max_convag = np.amax(convag) + convag = convag / max_convag + support = np.where(convag >= threshold, 1, 0) + return support + else: + return None + + +def read_results(read_dir): + try: + imagefile = os.path.join(read_dir, 'image.npy') + image = np.load(imagefile) + + supportfile = os.path.join(read_dir, 'support.npy') + support = np.load(supportfile) + + try: + cohfile = os.path.join(read_dir, 'coherence.npy') + coh = np.load(cohfile) + except: + coh = None + except: + pass + + return image, support, coh + + +def save_metrics(errs, dir, metrics=None): + metric_file = os.path.join(dir, 'summary') + if os.path.isfile(metric_file): + os.remove(metric_file) + with open(metric_file, 'a') as f: + if metrics is not None: + f.write('metric result\n') + for key in metrics: + value = metrics[key] + f.write(key + ' = ' + str(value) + '\n') + f.write('\nerrors by iteration\n') + for er in errs: + f.write(str(er) + ' ') + f.close() + +def write_plot_errors(save_dir): + plot_file = os.path.join(save_dir, 'plot_errors.py') + f = open(plot_file, 'w+') + f.write("#! /usr/bin/env python\n") + f.write("import matplotlib.pyplot as plt\n") + f.write("import numpy as np\n") + f.write("import sys\n") + f.write("import os\n") + f.write("current_dir = sys.path[0]\n") + f.write("errs = np.load(os.path.join(current_dir, 'errors.npy')).tolist()\n") + f.write("errs.pop(0)\n") + f.write("plt.plot(errs)\n") + f.write("plt.ylabel('errors')\n") + f.write("plt.show()") + f.close() + st = os.stat(plot_file) + os.chmod(plot_file, st.st_mode | stat.S_IEXEC) + + +def save_results(image, support, coh, errs, reciprocal, save_dir, metrics=None): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + image_file = os.path.join(save_dir, 'image') + np.save(image_file, image) + support_file = os.path.join(save_dir, 'support') + np.save(support_file, support) + errs_file = os.path.join(save_dir, 'errors') + np.save(errs_file, errs) + if not coh is None: + coh_file = os.path.join(save_dir, 'coherence') + np.save(coh_file, coh) + reciprocal_file = os.path.join(save_dir, 'reciprocal') + np.save(reciprocal_file, reciprocal) + write_plot_errors(save_dir) + if metrics is not None: + save_metrics(errs, save_dir, metrics) + else: + save_metrics(errs, save_dir) + + +def save_multiple_results(samples, images, supports, cohs, errs, reciprocals, save_dir, metrics=None): + """ + This function saves results of multiple reconstructions to directory tree in save_dir. + Parameters + ---------- + samples : int + number of reconstruction sets results + images : list + list of numpy arrays containing reconstructed images + supports : list + list of numpy arrays containing support of reconstructed images + cohs : list + list of numpy arrays containing coherence of reconstructed images + save_dir : str + a directory to save the results + Returns + ------- + nothing + """ + for i in range(samples): + subdir = os.path.join(save_dir, str(i)) + if metrics is None: + save_results(images[i], supports[i], cohs[i], np.asarray(errs[i]), reciprocals[i], subdir) + else: + save_results(images[i], supports[i], cohs[i], np.asarray(errs[i]), reciprocals[i], subdir, metrics[i]) + + +def sub_pixel_shift(arr, row_shift, col_shift, z_shift): + # arr is 3D + buf2ft = np.fft.fftn(arr) + shape = arr.shape + Nr = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[0] / 2)), shape[0] - int(np.floor(shape[0] / 2)))))) + Nc = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[1] / 2)), shape[1] - int(np.floor(shape[1] / 2)))))) + Nz = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[2] / 2)), shape[2] - int(np.floor(shape[2] / 2)))))) + [Nc, Nr, Nz] = np.meshgrid(Nc, Nr, Nz) + Greg = buf2ft * np.exp(1j * 2 * np.pi * (-row_shift * Nr / shape[0] - col_shift * Nc / shape[1] - z_shift * Nz / shape[2])) + return np.fft.ifftn(Greg) + + +def arr_property(arr): + arr1 = abs(arr) + print ('norm', np.sum(pow(abs(arr),2))) + max_coordinates = list(np.unravel_index(np.argmax(arr1), arr.shape)) + print ('max coords, value', max_coordinates, arr[max_coordinates[0], max_coordinates[1],max_coordinates[2]]) diff --git a/reccdi/src_py/utilities/transtest.py b/reccdi/src_py/utilities/transtest.py new file mode 100644 index 0000000..f4756e3 --- /dev/null +++ b/reccdi/src_py/utilities/transtest.py @@ -0,0 +1,22 @@ +import numpy as np + +dims=(5,5,5) +dxdir=1 +dydir=1 +dzdir=1 + +r = np.mgrid[(dims[0] - 1) * dxdir:-dxdir:-dxdir, \ + 0:dims[1] * dydir:dydir,\ + 0:dims[2] * dzdir:dzdir] + +origshape=r.shape +r.shape = 3, dims[0] * dims[1] * dims[2] +r = r.transpose() + +Tdir=np.array( [[0.1,0,0],[0,1,0],[0,0,1]]) +print( Tdir) +dir_coords = np.dot(r, Tdir) + +dir_coords = dir_coords.transpose() +dir_coords.shape=origshape + diff --git a/reccdi/src_py/utilities/utils.py b/reccdi/src_py/utilities/utils.py new file mode 100644 index 0000000..a8e10e9 --- /dev/null +++ b/reccdi/src_py/utilities/utils.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module is a suite of utility mehods. +""" + +import tifffile as tf +import pylibconfig2 as cfg +import numpy as np +import scipy.fftpack as sf +import os +import logging +import stat +from functools import reduce +import GPUtil + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['read_tif', + 'get_opencl_dim', + 'binning', + 'get_centered', + 'adjust_dimensions', + 'crop_center', + 'flip'] + + +def get_logger(name, ldir=''): + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + log_file = os.path.join(ldir, 'default.log') + fh = logging.FileHandler(log_file) + fh.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + return logger + + +def read_tif(filename): + """ + This method reads tif type file containing experiment data and returns the data as array. + Parameters + ---------- + filename : str + a filename containing the experiment data + Returns + ------- + data : array + an array containing the experiment data + """ + + ar = tf.imread(filename).transpose() + #ar = np.swapaxes(ar, 0, 2) + #ar = np.swapaxes(ar, 0, 1) + return ar + + +def save_tif(arr, tif_file): + # arr = np.swapaxes(arr, 0, 2) + # arr = np.swapaxes(arr, 1, 2) + #arr = np.swapaxes(arr, 0, 1) + #arr = np.swapaxes(arr, 0, 2) + tf.imsave(tif_file, arr.transpose().astype(np.float32)) + + +def read_config(config): + """ + This function gets configuration file. It checks if the file exists and parses it into a map. + Parameters + ---------- + config : str + configuration file name, including path + Returns + ------- + config_map : dict + a map containing parsed configuration, None if the given file does not exist + """ + + if os.path.isfile(config): + print(config) + with open(config, 'r') as f: + config_map = cfg.Config(f.read()) + return config_map; + else: + return None + + +def get_good_dim(dim): + """ + This function calculates the dimension supported by opencl library (i.e. is multiplier of 2,3, or 5) and is closest to the + given starting dimension, and spaced by the given step. + If the dimension is not supported the function adds step value and verifies the new dimension. It iterates until it finds + supported value. + Parameters + ---------- + dim : int + a dimension that needs to be tranformed to one that is supported by the opencl library, if it is not already + + step : int + a delta to increase the dimension + Returns + ------- + dim : int + a dimension that is supported by the opencl library, and closest to the original dimension by n*step + """ + + def is_correct(x): + sub = x + if sub % 3 == 0: + sub = sub / 3 + if sub % 3 == 0: + sub = sub / 3 + if sub % 5 == 0: + sub = sub / 5 + while sub % 2 == 0: + sub = sub / 2 + return sub == 1 + + new_dim = dim + if new_dim % 2 == 1: + new_dim += 1 + while not is_correct(new_dim): + new_dim += 2 + return new_dim + + +def binning(array, binsizes): + """ + This function does the binning of the array. The array is binned in each dimension by the corresponding binsizes elements. + If binsizes list is shorter than the array dimensions, the remaining dimensions are not binned. The elements in + a bucket are summed. + Parameters + ---------- + array : array + the original array to be binned + + binsizes : list + a list defining binning buckets for corresponding dimensions + Returns + ------- + array : array + the binned array + """ + + data_dims = array.shape + # trim array + for ax in range(len(binsizes)): + cut_slices = range(data_dims[ax] - data_dims[ax] % binsizes[ax], data_dims[ax]) + array = np.delete(array, cut_slices, ax) + + binned_array = array + new_shape = list(array.shape) + + for ax in range(len(binsizes)): + if binsizes[ax] > 1: + new_shape[ax] = binsizes[ax] + new_shape.insert(ax, int(array.shape[ax] / binsizes[ax])) + binned_array = np.reshape(binned_array, tuple(new_shape)) + binned_array = np.sum(binned_array, axis=ax + 1) + new_shape = list(binned_array.shape) + return binned_array + + +# ar = np.asarray([1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9,1,2,3,4,5,6,7,8,9]) +# ar.resize((5,9)) +# print ('ar', ar) +# b = binning(ar, (2,2)) +# print ('b',b) +# c = binning1(ar,(2,2)) +# print ('c',c) + + +def get_centered(arr, center_shift): + """ + This function finds a greatest value in the array, and puts it in a center of a new array. If center_shift is + not zeros, the array will be shifted accordingly. The shifted elements are rolled into the other end of array. + Parameters + ---------- + arr : array + the original array to be centered + center_shift : list + a list defining shift of the center + Returns + ------- + array : array + the centered array + """ + max_coordinates = list(np.unravel_index(np.argmax(arr), arr.shape)) + max_coordinates = np.add(max_coordinates, center_shift) + shape = arr.shape + centered = arr + for i in range(len(max_coordinates)): + centered = np.roll(centered, int(shape[i] / 2) - max_coordinates[i], i) + + return centered + + +def get_centered_both(arr, support): + """ + This function finds a greatest value in the array, and puts it in a center of a new array. If center_shift is + not zeros, the array will be shifted accordingly. The shifted elements are rolled into the other end of array. + Parameters + ---------- + arr : array + the original array to be centered + support : array + the associated array shifted the same way centered array is + Returns + ------- + centered : array + the centered array + """ + max_coordinates = list(np.unravel_index(np.argmax(arr), arr.shape)) + shape = arr.shape + centered = arr + centered_supp = support + for i in range(len(max_coordinates)): + centered = np.roll(centered, int(shape[i] / 2) - max_coordinates[i], i) + centered_supp = np.roll(centered_supp, int(shape[i] / 2) - max_coordinates[i], i) + + return centered, centered_supp + + +def get_zero_padded_centered(arr, new_shape): + """ + This function pads the array with zeros to the new shape with the array in the center. + Parameters + ---------- + arr : array + the original array to be padded and centered + new_shape : tuple + a list of new dimensions + Returns + ------- + array : array + the zero padded centered array + """ + shape = arr.shape + pad = [] + c_vals = [] + for i in range(len(new_shape)): + pad.append((0, new_shape[i] - shape[i])) + c_vals.append((0.0, 0.0)) + arr = np.lib.pad(arr, (pad), 'constant', constant_values=c_vals) + + centered = arr + for i in range(len(new_shape)): + centered = np.roll(centered, int((new_shape[i] - shape[i] + 1) / 2), i) + + return centered + + +def adjust_dimensions(arr, pads): + """ + This function adds to or subtracts from each dimension of the array elements defined by pad. If the pad is positive, + the array is padded in this dimension. If the pad is negative, the array is cropped. + The dimensions of the new array are supported by the opencl library. + Parameters + ---------- + arr : array + the array to pad/crop + pad : list + list of three pad values, for each dimension + Returns + ------- + array : array + the padded/cropped array + """ + # logger = get_logger('adjust_dimensions') + old_dims = arr.shape + start = [] + stop = [] + for i in range(len(old_dims)): + pad = pads[i] + first = max(0, -pad[0]) + last = old_dims[i] - max(0, -pad[1]) + if first >= last: + print ('the crop exceeds size, please change the crop and run again') + return None + else: + start.append(first) + stop.append(last) + + cropped = arr[ start[0]:stop[0], start[1]:stop[1], start[2]:stop[2] ] + # logger.info('cutting from to ' + str(crop[0]) + ', ' + str(old_dims[0]-crop[1]) + ', ' + str(crop[2]) + ', ' \ + # + str(old_dims[1]-crop[3]) + ', ' + str(crop[4]) + ', ' + str(old_dims[2]-crop[5])) + dims = cropped.shape + c_vals = [] + new_pad = [] + for i in range(len(dims)): + pad = pads[i] + # find a good dimension and find padding + temp_dim = old_dims[i] + pad[0] + pad[1] + new_dim = get_good_dim(temp_dim) + added = new_dim - temp_dim + # if the pad is positive + pad_front = max(0, pad[0]) + int(added / 2) + pad_end = new_dim - dims[i] - pad_front + new_pad.append((pad_front, pad_end)) + c_vals.append((0.0, 0.0)) + adjusted = np.pad(cropped, new_pad, 'constant', constant_values=c_vals) + + # logger.info('pads ' + str(new_pad[0]) + ', ' + str(new_pad[1]) + ', ' + str(new_pad[2]) + ', ' + str(new_pad[3]) \ + # + ', ' + str(new_pad[4]) + ', ' + str(new_pad[5])) + # logger.info('old dim, new dim (' + str(dims[0]) + ',' + str(dims[1]) + ',' + str(dims[2]) + ') (' + str(arr.shape[0]) +\ + # ',' + str(arr.shape[1]) + ',' + str(arr.shape[1]) + ')') + + return adjusted + +# ar = np.zeros((256,256,90)) +# pads = (-100,-100,0,0,0,0) +# ar = np.zeros((81,256,256)) +# pads = (5,-7,-20,-30,4,-20) +# arr = adjust_dimensions(ar,pads) +# print (arr.shape) + +def crop_center(arr, new_size): + size = arr.shape + cropped = arr + for i in range(len(size)): + crop_front = int((size[i] - new_size[i]) / 2) + crop_end = crop_front + new_size[i] + splitted = np.split(cropped, [crop_front, crop_end], axis=i) + cropped = splitted[1] + + return cropped + + +# ar = np.zeros((81,256,256)) +# new_size = (40, 200,100) +# arr = crop_center(ar,new_size) +# print (arr.shape) + +def get_norm(arr): + return sum(sum(sum(abs(arr) ** 2))) + + +def flip(m, axis): + """ + Copied from numpy 1.12.0. + """ + if not hasattr(m, 'ndim'): + m = np.asarray(m) + indexer = [slice(None)] * m.ndim + try: + indexer[axis] = slice(None, None, -1) + except IndexError: + raise ValueError("axis=%i is invalid for the %i-dimensional input array" + % (axis, m.ndim)) + return m[tuple(indexer)] + + +def gaussian(shape, sigmas, alpha=1): + grid = np.full(shape, 1.0) + for i in range(len(shape)): + # prepare indexes for tile and transpose + tile_shape = list(shape) + tile_shape.pop(i) + tile_shape.append(1) + trans_shape = list(range(len(shape) - 1)) + trans_shape.insert(i, len(shape) - 1) + + multiplier = - 0.5 * alpha / pow(sigmas[i], 2) + line = np.linspace(-(shape[i] - 1) / 2.0, (shape[i] - 1) / 2.0, shape[i]) + gi = np.tile(line, tile_shape) + gi = np.transpose(gi, tuple(trans_shape)) + exponent = np.power(gi, 2) * multiplier + gi = np.exp(exponent) + grid = grid * gi + + grid_total = np.sum(grid) + return grid / grid_total + + +def gauss_conv_fft(arr, sigmas): + arr_sum = np.sum(abs(arr)) + arr_f = sf.ifftshift(sf.fftn(sf.ifftshift(arr))) + shape = list(arr.shape) + for i in range(len(sigmas)): + sigmas[i] = shape[i] / 2.0 / np.pi / sigmas[i] + convag = arr_f * gaussian(shape, sigmas) + convag = np.fft.ifftshift(sf.ifftn(np.fft.ifftshift(convag))) + convag = convag.real + convag = np.clip(convag, 0, None) + correction = arr_sum / np.sum(convag) + convag *= correction + return convag + + +def shrink_wrap(arr, threshold, sigma, type='gauss'): + sigmas = [sigma] * len(arr.shape) + if type == 'gauss': + convag = gauss_conv_fft(abs(arr), sigmas) + max_convag = np.amax(convag) + convag = convag / max_convag + support = np.where(convag >= threshold, 1, 0) + return support + else: + return None + + +def read_results(read_dir): + try: + imagefile = os.path.join(read_dir, 'image.npy') + image = np.load(imagefile) + + supportfile = os.path.join(read_dir, 'support.npy') + support = np.load(supportfile) + + try: + cohfile = os.path.join(read_dir, 'coherence.npy') + coh = np.load(cohfile) + except: + coh = None + except: + pass + + return image, support, coh + + +def save_metrics(errs, dir, metrics=None): + metric_file = os.path.join(dir, 'summary') + if os.path.isfile(metric_file): + os.remove(metric_file) + with open(metric_file, 'a') as f: + if metrics is not None: + f.write('metric result\n') + for key in metrics: + value = metrics[key] + f.write(key + ' = ' + str(value) + '\n') + f.write('\nerrors by iteration\n') + for er in errs: + f.write(str(er) + ' ') + f.close() + +def write_plot_errors(save_dir): + plot_file = os.path.join(save_dir, 'plot_errors.py') + f = open(plot_file, 'w+') + f.write("#! /usr/bin/env python\n") + f.write("import matplotlib.pyplot as plt\n") + f.write("import numpy as np\n") + f.write("import sys\n") + f.write("import os\n") + f.write("current_dir = sys.path[0]\n") + f.write("errs = np.load(os.path.join(current_dir, 'errors.npy')).tolist()\n") + f.write("errs.pop(0)\n") + f.write("plt.plot(errs)\n") + f.write("plt.ylabel('errors')\n") + f.write("plt.show()") + f.close() + st = os.stat(plot_file) + os.chmod(plot_file, st.st_mode | stat.S_IEXEC) + + +def save_results(image, support, coh, errs, reciprocal, flow, iter_array, save_dir, metrics=None): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + print("image shape", image.shape) + + image_file = os.path.join(save_dir, 'image') + np.save(image_file, image) + save_tif(np.abs(image), image_file+".tif") + support_file = os.path.join(save_dir, 'support') + np.save(support_file, support) + save_tif(np.abs(support), support_file+".tif") + errs_file = os.path.join(save_dir, 'errors') + np.save(errs_file, errs) + if not coh is None: + coh_file = os.path.join(save_dir, 'coherence') + np.save(coh_file, coh) + reciprocal_file = os.path.join(save_dir, 'reciprocal') + np.save(reciprocal_file, reciprocal) + write_plot_errors(save_dir) + + graph_dir = os.path.join(save_dir, 'graph') + if not os.path.exists(graph_dir): + os.makedirs(graph_dir) + flow_file = os.path.join(graph_dir, 'flow') + np.save(flow_file, np.asarray(flow)) + iter_array_file = os.path.join(graph_dir, 'iter_array') + np.save(iter_array_file, iter_array) + + if metrics is not None: + save_metrics(errs, save_dir, metrics) + else: + save_metrics(errs, save_dir) + + +def save_multiple_results(samples, images, supports, cohs, errs, reciprocals, flows, iter_arrs, save_dir, metrics=None): + """ + This function saves results of multiple reconstructions to directory tree in save_dir. + Parameters + ---------- + samples : int + number of reconstruction sets results + images : list + list of numpy arrays containing reconstructed images + supports : list + list of numpy arrays containing support of reconstructed images + cohs : list + list of numpy arrays containing coherence of reconstructed images + save_dir : str + a directory to save the results + Returns + ------- + nothing + """ + for i in range(samples): + subdir = os.path.join(save_dir, str(i)) + if metrics is None: + save_results(images[i], supports[i], cohs[i], np.asarray(errs[i]), reciprocals[i], flows[i], iter_arrs[i], subdir) + else: + save_results(images[i], supports[i], cohs[i], np.asarray(errs[i]), reciprocals[i], flows[i], iter_arrs[i], subdir, metrics[i]) + + +def sub_pixel_shift(arr, row_shift, col_shift, z_shift): + # arr is 3D + buf2ft = sf.fftn(arr) + shape = arr.shape + Nr = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[0] / 2)), shape[0] - int(np.floor(shape[0] / 2)))))) + Nc = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[1] / 2)), shape[1] - int(np.floor(shape[1] / 2)))))) + Nz = np.fft.ifftshift(np.array(list(range(-int(np.floor(shape[2] / 2)), shape[2] - int(np.floor(shape[2] / 2)))))) + [Nc, Nr, Nz] = np.meshgrid(Nc, Nr, Nz) + Greg = buf2ft * np.exp(1j * 2 * np.pi * (-row_shift * Nr / shape[0] - col_shift * Nc / shape[1] - z_shift * Nz / shape[2])) + return sf.ifftn(Greg) + + +def arr_property(arr): + arr1 = abs(arr) + print ('norm', np.sum(pow(abs(arr),2))) + max_coordinates = list(np.unravel_index(np.argmax(arr1), arr.shape)) + print ('max coords, value', max_coordinates, arr[max_coordinates[0], max_coordinates[1],max_coordinates[2]]) + + +def get_gpu_load(mem_size, ids): + gpus = GPUtil.getGPUs() + total_avail = 0 + available_dir = {} + for gpu in gpus: + if gpu.id in ids: + free_mem = gpu.memoryFree + avail_runs = int(free_mem / mem_size) + if avail_runs > 0: + total_avail += avail_runs + available_dir[gpu.id] = avail_runs + available = [] + for id in ids: + try: + avail_runs = available_dir[id] + except: + avail_runs = 0 + available.append(avail_runs) + return available + + +def get_gpu_distribution(runs, available): + all_avail = reduce((lambda x,y: x+y), available) + while runs < all_avail: + # balance distribution + for i in range(len(available)): + if available[i] > 0: + available[i] -= 1 + all_avail -= 1 + if all_avail == runs: + break + return available + + +#https://stackoverflow.com/questions/51503672/decorator-for-timeit-timeit-method/51503837#51503837 +from functools import wraps +from time import time +def measure(func): + @wraps(func) + def _time_it(*args, **kwargs): + start = int(round(time() * 1000)) + try: + return func(*args, **kwargs) + finally: + end_ = int(round(time() * 1000)) - start + print(f"Total execution time: {end_ if end_ > 0 else 0} ms") + return _time_it + diff --git a/reccdi/src_py/utilities/utils_ga.py b/reccdi/src_py/utilities/utils_ga.py new file mode 100644 index 0000000..747ad80 --- /dev/null +++ b/reccdi/src_py/utilities/utils_ga.py @@ -0,0 +1,219 @@ + +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +""" +Please make sure the installation :ref:`pre-requisite-reference-label` are met. +This module is a suite of utility mehods. +""" + +import scipy as sci +import numpy as np +import reccdi.src_py.utilities.utils as ut + + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' +__all__ = ['get_array_from_tif', + 'get_opencl_dim', + 'binning', + 'get_centered', + 'adjust_dimensions', + 'crop_center', + 'flip'] + +def cross_correlation(a, b): + A = np.fft.ifftshift(np.fft.fftn(np.fft.fftshift(conj_reflect(a)))) + B = np.fft.ifftshift(np.fft.fftn(np.fft.fftshift(b))) + CC = A * B + return np.fft.ifftshift(np.fft.ifftn(np.fft.fftshift(CC))) + + +def conj_reflect(arr): + F = np.fft.ifftshift(np.fft.fftn(np.fft.fftshift(arr))) + return np.fft.ifftshift(np.fft.ifftn(np.fft.fftshift(np.conj(F)))) + + +def check_get_conj_reflect(arr1, arr2): + support1 = ut.shrink_wrap(abs(arr1), .1, .1) + support2 = ut.shrink_wrap(abs(arr2), .1, .1) + cc1 = cross_correlation(support1, ut.shrink_wrap(conj_reflect(arr2), .1, .1)) + cc2 = cross_correlation(support1, support2) + if np.amax(cc1) > np.amax(cc2): + return conj_reflect(arr2) + else: + return arr2 + + +def dftups(arr, nor=-1, noc=-1, usfac=2, roff=0, coff=0): + # arr is 2D + [nr,nc] = arr.shape + if nor < 0: + nor = nr + if noc < 0: + noc = nc + + # Compute kernels and obtain DFT by matrix products + yl = list(range(-int(np.floor(nc/2)), nc - int(np.floor(nc/2)))) + y = np.fft.ifftshift(np.array(yl)) * (-2j * np.pi/(nc * usfac)) + xl = list(range(-coff, noc - coff)) + x = np.array(xl) + yt = np.tile(y, (len(xl), 1)) + xt = np.tile(x, (len(yl), 1)) + kernc = np.exp(yt.T * xt) + + yl = list(range(-roff, nor - roff)) + y = np.array(yl) + xl = list(range(-int(np.floor(nr/2)), nr - int(np.floor(nr/2)))) + x = np.fft.ifftshift(np.array(xl)) + yt = np.tile(y, (len(xl), 1)) + xt = np.tile(x, (len(yl), 1)) + kernr = np.exp(yt * xt.T * (-2j * np.pi/(nr * usfac))) + + return np.dot(np.dot(kernr.T, arr), kernc) + + +def dftregistration(ref_arr, arr, usfac=2): + #arrays are 2D + # based on Matlab dftregistration by Manuel Guizar (Portions of this code were taken from code written by + # Ann M. Kowalczyk and James R. Fienup. + if usfac < 2: + print ('usfac less than 2 not supported') + # will throw exception + return + # First upsample by a factor of 2 to obtain initial estimate + # Embed Fourier data in a 2x larger array + shape = ref_arr.shape + large_shape = tuple(2 * x for x in ref_arr.shape) + c_c = ut.get_zero_padded_centered(np.fft.fftshift(ref_arr) * np.conj(np.fft.fftshift(arr)), large_shape) + + # Compute crosscorrelation and locate the peak + c_c = np.fft.ifft2(np.fft.ifftshift(c_c)) + max_coord = list(np.unravel_index(np.argmax(c_c), c_c.shape)) + + if max_coord[0] > shape[0]: + row_shift = max_coord[0] - large_shape[0] + else: + row_shift = max_coord[0] + if max_coord[1] > shape[1]: + col_shift = max_coord[1] - large_shape[1] + else: + col_shift = max_coord[1] + + row_shift = row_shift/2 + col_shift = col_shift/2 + + #If upsampling > 2, then refine estimate with matrix multiply DFT + if usfac > 2: + # DFT computation + # Initial shift estimate in upsampled grid + row_shift = round(row_shift * usfac)/usfac + col_shift = round(col_shift * usfac)/usfac + dftshift = np.fix(np.ceil(usfac * 1.5)/2) # Center of output array at dftshift + # Matrix multiply DFT around the current shift estimate + c_c = np.conj(dftups(arr * np.conj(ref_arr), int(np.ceil(usfac * 1.5)), int(np.ceil(usfac * 1.5)), usfac, + int(dftshift-row_shift * usfac), int(dftshift-col_shift * usfac)))/\ + (int(np.fix(shape[0]/2)) * int(np.fix(shape[1]/2)) * usfac^2) + # Locate maximum and map back to original pixel grid + max_coord = list(np.unravel_index(np.argmax(c_c), c_c.shape)) + [rloc, cloc] = max_coord + + rloc = rloc - dftshift + cloc = cloc - dftshift + row_shift = row_shift + rloc/usfac + col_shift = col_shift + cloc/usfac + + return row_shift, col_shift + + +def register_3d_reconstruction(ref_arr, arr): + r_shift_2, c_shift_2 = dftregistration(np.fft.fft2(np.sum(ref_arr, 2)), np.fft.fft2(np.sum(arr, 2)), 100) + r_shift_1, c_shift_1 = dftregistration(np.fft.fft2(np.sum(ref_arr, 1)), np.fft.fft2(np.sum(arr, 1)), 100) + r_shift_0, c_shift_0 = dftregistration(np.fft.fft2(np.sum(ref_arr, 0)), np.fft.fft2(np.sum(arr, 0)), 100) + + shift_2 = sum([r_shift_2, r_shift_1]) * 0.5 + shift_1 = sum([c_shift_2, r_shift_0]) * 0.5 + shift_0 = sum([c_shift_1, c_shift_0]) * 0.5 + return shift_2, shift_1, shift_0 + + +def print_max(arr): + max_coord = list(np.unravel_index(np.argmax(abs(arr)), arr.shape)) + print ('max coord, value', abs(arr[max_coord[0],max_coord[1],max_coord[2]]), max_coord) + +def zero_phase(arr, val=0): + ph = np.angle(arr) + support = ut.shrink_wrap(abs(arr), .2, .5) #get just the crystal, i.e very tight support + avg_ph = np.sum(ph * support)/np.sum(support) + ph = ph - avg_ph + val + return abs(arr) * np.exp(1j * ph) + + +def zero_phase_cc(arr1, arr2): + # will set array1 avg phase to array2 + c_c = np.conj(arr1) * arr2 + c_c_tot = np.sum(c_c) + ph = np.angle(c_c_tot) + arr = arr1 * np.exp(1j * ph) + return arr + + +def align_arrays(ref_arr, arr): + (shift_2, shift_1, shift_0) = register_3d_reconstruction(abs(ref_arr), abs(arr)) + return ut.sub_pixel_shift(arr, shift_2, shift_1, shift_0) + +# ref_arr = np.load('/home/phoebus/BFROSIK/temp/test/A_78-97/results/image.npy') +# arr = np.load('/home/phoebus/BFROSIK/temp/test/B_78-97/results/image.npy') +# l = align_arrays(ref_arr, arr) + +def sum_phase_tight_support(arr): + arr = zero_phase(arr) + ph = np.arctan2(arr.imag, arr.real) + support = ut.shrink_wrap(abs(arr), .2, .5) + return sum( abs(ph * support)) + + +def get_arr_characteristic(arr): + lev1_norm = sum(abs(arr)) + sharpness = sum(abs(arr)^4) + summed_phase = sum_phase_tight_support(arr) + support = ut.shrink_wrap(arr, .2, .5) + area = sum(support) + gradients = np.gradient(arr) + TV = np.zeros(arr.shape) + for gr in gradients: + TV += abs(gr) + return lev1_norm, sharpness, summed_phase, area, TV + + +# def align_iterates(arrs): +# #assume arrs[0] is the referrence array +# alpha = arrs[0] +# for i in range(1, len(arrs)): +# arr = check_get_conj_reflect(abs(alpha), abs(arr)) +# shift_2, shift_1, shift_0 = register_3d_reconstruction(abs(alpha), abs(arr)) +# arrs[i] = sub_pixel_shift(arr, round(shift_2), round(shift_1), round(shift_2)) +# return arrs + + +# def test(a,b): +# alpha = zero_phase(a, 0) +# beta = zero_phase(b, 0) +# alpha = check_get_conj_reflect(beta, alpha) +# alpha_s = align_arrays(beta, alpha) +# alpha_s = zero_phase(alpha_s, 0) +# ph_alpha = np.angle(alpha_s) +# beta = zero_phase_cc(beta, alpha_s) +# ph_beta = np.angle(beta) +# beta = np.sqrt(abs(alpha_s) * abs(beta)) * np.exp(0.5j * (ph_beta + ph_alpha)) +# +# a = np.load('/home/phoebus/BFROSIK/temp/test/A_78-97/results/image.npy') +# b = np.load('/home/phoebus/BFROSIK/temp/test/B_78-97/results/image.npy') +# test(a, b) \ No newline at end of file diff --git a/reccdi/src_py/utilities/viz_util.py b/reccdi/src_py/utilities/viz_util.py new file mode 100644 index 0000000..6d9e02c --- /dev/null +++ b/reccdi/src_py/utilities/viz_util.py @@ -0,0 +1,71 @@ +# ######################################################################### +# Copyright (c) , UChicago Argonne, LLC. All rights reserved. # +# # +# See LICENSE file. # +# ######################################################################### + +import os +import numpy as np +import scipy.ndimage as ndi +import math as m +import reccdi.src_py.utilities.utils as ut +from reccdi.src_py.utilities.utils import measure + +__author__ = "Barbara Frosik" +__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." +__docformat__ = 'restructuredtext en' + + +def shift(arr, s0, s1, s2): + shifted = np.roll(arr, s0, axis=0) + shifted = np.roll(shifted, s1, axis=1) + return np.roll(shifted, s2, axis=2) + + +@measure +def remove_ramp(arr, ups=1): + new_shape = list(arr.shape) + # pad zeros around arr, to the size of 3 times (ups = 3) of arr size + for i in range(len(new_shape)): + new_shape[i] = ups * new_shape[i] + padded = ut.get_zero_padded_centered(arr, new_shape) + padded_f = np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(padded))) + com = ndi.center_of_mass(np.power(np.abs(padded_f), 2)) + sub_pixel_shifted = ut.sub_pixel_shift(padded_f, new_shape[0]/2.0-com[0], new_shape[1]/2.0-com[1], +new_shape[2]/2.0-com[2]) + ramp_removed_padded = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(sub_pixel_shifted))) + ramp_removed = ut.crop_center(ramp_removed_padded, arr.shape) + + return ramp_removed + + +@measure +def center(image, support): + dims = image.shape + image, support = ut.get_centered_both(image, support) + + # place center of mass image*support in the center + for ax in range(len(dims)): + com = ndi.center_of_mass(np.absolute(image) * support) + image = shift(image, int(dims[0]/2 - com[0]), int(dims[1]/2 - com[1]), int(dims[2]/2 - com[2])) + support = shift(support, int(dims[0]/2 - com[0]), int(dims[1]/2 - com[1]), int(dims[2]/2 - com[2])) + + # set center phase to zero, use as a reference + phi0 = m.atan2(image.imag[int(dims[0]/2), int(dims[1]/2), int(dims[2]/2)], image.real[int(dims[0]/2), int(dims[1]/2), +int(dims[2]/2)]) + image = image * np.exp(-1j * phi0) + + return image, support + + +def get_crop(params, shape): + crop = [] + for i in range(len(shape)): + if params.crop is None: + crop.append(shape[i]) + else: + crop.append(params.crop[i]) + if isinstance(crop[i], float): + crop[i] = int(crop[i]*shape[i]) + return crop + diff --git a/setup.py b/setup.py index 6d85ffa..c82f23c 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,15 @@ from distutils.core import setup -from distutils.extension import Extension from Cython.Build import cythonize -setup(ext_modules = cythonize( - ["src_py/cyth/bridge_cpu.pyx", "src_py/cyth/bridge_opencl.pyx", "src_py/cyth/bridge_cuda.pyx",], - language="c++",) -) - +setup(ext_modules=cythonize( + ["reccdi/src_py/cyth/bridge_cpu.pyx", "reccdi/src_py/cyth/bridge_opencl.pyx", "reccdi/src_py/cyth/bridge_cuda.pyx", ],), + name='reccdi', + author = 'Barbara Frosik', + author_email = 'bfrosik@anl.gov', + url='https://github.com/advancedPhotonSource/cdi', + version='1.0', + packages=['reccdi.src_py.beamlines.aps_34id', 'reccdi.src_py.controller', 'reccdi.src_py.cyth', 'reccdi.src_py.run_scripts', 'reccdi.src_py.utilities'], + package_data={'reccdi' : ['*.pyx',], 'reccdi.src_py.cyth' : ['*.pyx',]} +) diff --git a/src_cpp/algorithm.cpp b/src_cpp/algorithm.cpp deleted file mode 100644 index 35021dc..0000000 --- a/src_cpp/algorithm.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "iostream" -#include "algorithm.hpp" -#include "worker.hpp" -#include "common.h" - -Reconstruction * rec; - -void Algorithm::RunAlgorithm(Reconstruction * reconstruction) -{ - rec = reconstruction; - ModulusConstrain(ModulusProjection()); -} - -af::array Algorithm::ModulusProjection() -{ - return rec->ModulusProjection(); -} - -void Algorithm::ModulusConstrain(af::array arr) -{ - -} - -//------------------------------------------------------------------- -// Hio sub-class - -void Hio::ModulusConstrain(af::array arr) -{ - rec->ModulusConstrainHio(arr); -} - -//------------------------------------------------------------------- -// Er sub-class - -void Er::ModulusConstrain(af::array arr) -{ - rec->ModulusConstrainEr(arr); -} - diff --git a/src_cpp/bridge.cpp b/src_cpp/bridge.cpp deleted file mode 100644 index 68b63f6..0000000 --- a/src_cpp/bridge.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// -// Reconstruction.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/12/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - - -#include "sstream" -#include "bridge.hpp" -#include "manager.hpp" - -Manager mgr; - -void Bridge::StartCalcWithGuess(std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config) -{ - std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); - std::vector guess_i(guess_buffer_i.begin(), guess_buffer_i.end()); - std::vector guess_r(guess_buffer_r.begin(), guess_buffer_r.end()); - mgr.StartCalc(data_r, guess_r, guess_i, dim, config); - //mgr.StartCalc(data_buffer_r, guess_buffer_r, guess_buffer_i, dim, config); -} - -void Bridge::StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config) -{ - std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); - mgr.StartCalc(data_r, dim, config); -// mgr.StartCalc(data_buffer_r, dim, config); -} - -void Bridge::StartCalcMultiple(std::vector data_buffer_r, std::vector dim, std::string const & config, int nu_threads) -{ - std::vector data_r(data_buffer_r.begin(), data_buffer_r.end()); - mgr.StartCalc(data_r, dim, config, nu_threads); -// mgr.StartCalc(data_buffer_r, dim, config, nu_threads); -} - -std::vector Bridge::GetImageR() -{ - return mgr.GetImageR(); -} - -std::vector Bridge::GetImageI() -{ - return mgr.GetImageI(); -} - -std::vector Bridge::GetErrors() -{ - return mgr.GetErrors(); -} - -std::vector Bridge::GetSupportV() -{ - return mgr.GetSupportV(); -} - -std::vector Bridge::GetCoherenceV() -{ - return mgr.GetCoherenceV(); -} - - - - - diff --git a/src_cpp/manager.cpp b/src_cpp/manager.cpp deleted file mode 100644 index 16a67f1..0000000 --- a/src_cpp/manager.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// -// Reconstruction.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/12/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#include "typeinfo" -#include "arrayfire.h" -#include "worker.hpp" -#include "manager.hpp" -#include "util.hpp" - - -using namespace af; - -void Manager::StartCalc(std::vector data_buffer_r, std::vector guess_buffer_r, std::vector guess_buffer_i, std::vector dim, const std::string & config) -{ - dim4 af_dims = Utils::Int2Dim4(dim); - af::array real_d(af_dims, &data_buffer_r[0]); - //saving abs(data) - af::array data = abs(real_d); - - - af::array real_g(af_dims, &guess_buffer_r[0]); - af::array imag_g(af_dims, &guess_buffer_i[0]); - af::array guess = complex(real_g, imag_g); - - Reconstruction reconstruction(data, guess, config.c_str()); - reconstruction.Init(); - rec = &reconstruction; - timer::start(); - - reconstruction.Iterate(); - - printf("iterate function took %g seconds\n", timer::stop()); -} - -void Manager::StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config) -{ - StartCalc(data_buffer_r, dim, config, 1); -} - -void Manager::StartCalc(std::vector data_buffer_r, std::vector dim, std::string const & config, int nu_threads) -{ - af::array real_d(Utils::Int2Dim4(dim), &data_buffer_r[0]); - //saving abs(data) - af::array data = abs(real_d); - - - af::randomEngine r(AF_RANDOM_ENGINE_MERSENNE, (uint)time(0)); - d_type test1 = 0; - double test2 = 0; - af::array guess; - if (typeid(test1) == typeid(test2)) - { - guess = randu(data.dims(), c64, r); - } - else - { - guess = randu(data.dims(), c32, r); - } - Reconstruction reconstruction(data, guess, config.c_str()); - reconstruction.Init(); - rec = &reconstruction; - - timer::start(); - reconstruction.Iterate(); - printf("iterate function took %g seconds\n", timer::stop()); - -} - -std::vector Manager::GetImageR() -{ - af::array image = rec->GetImage(); - - d_type *image_r = real(image).host(); - std::vector v(image_r, image_r + image.elements()); - - delete [] image_r; - return v; -} - -std::vector Manager::GetImageI() -{ - af::array image = rec->GetImage(); - - d_type *image_i = imag(image).host(); - std::vector v(image_i, image_i + image.elements()); - - delete [] image_i; - return v; -} - -std::vector Manager::GetErrors() -{ - return rec->GetErrors(); -} - -std::vector Manager::GetSupportV() -{ - return rec->GetSupportVector(); -} - -std::vector Manager::GetCoherenceV() -{ - return rec->GetCoherenceVector(); -} - - diff --git a/src_cpp/parameters.cpp b/src_cpp/parameters.cpp deleted file mode 100644 index 3a63734..0000000 --- a/src_cpp/parameters.cpp +++ /dev/null @@ -1,484 +0,0 @@ -// -// parameters.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#include "arrayfire.h" -#include -#include "iostream" -#include "libconfig.h++" -#include "map" -#include "algorithm" -#include "parameters.hpp" -#include "support.hpp" -#include "pcdi.hpp" -#include "common.h" -#include "util.hpp" - -using namespace af; -using namespace libconfig; -Config cfg; - -// maps algorithm name to algorithm number -std::map algorithm_id_map; -// vector holding algorithm run sequence, where algorithm run is a pair of algorithm and number of iterations -std::vector alg_switches; - -std::string data_type; - -//use the matlab order (fft/ifft in modulus projector) -bool matlab_order = true; - -// amplitude threshold -d_type amp_threshold; -bool amp_threshold_fill_zeros; - -d_type phase_min = 120; -d_type phase_max = 10; -float beta = .9; - -// support -Support *support_attr; -PartialCoherence *partial_coherence = NULL; - -bool d_type_precision; - -// number of iterates to average -int avg_iterations; - -// calculated number of iterations -int number_iterations; - -int twin; - -int regularized_amp = REGULARIZED_AMPLITUDE_NONE; -int gc = -1; - - - -Params::Params(const char* config_file, const dim4 data_dim) -{ - BuildAlgorithmMap(); - - // Read the file. If there is an error, report. - try - { - cfg.readFile(config_file); - } - catch(const FileIOException &fioex) - { - printf("file I/O exception"); - } - catch(const ParseException &pex) - { - printf("parsing exception"); - } - - try { - const Setting& root = cfg.getRoot(); - const Setting &tmp = root["algorithm_sequence"]; - int count = tmp.getLength(); - - int switch_iter = 0; - int iter = 0; - for (int i = 0; i < count; ++i) - { - int repeat = tmp[i][0]; - for (int k = 0; k < repeat; k++) - { - for (int j = 1; j < tmp[i].getLength(); ++j) - { - iter = tmp[i][j][1]; - switch_iter = switch_iter + iter; - alg_switches.push_back(Alg_switch(algorithm_id_map[tmp[i][j][0]], switch_iter)); - } - } - } - number_iterations = alg_switches[alg_switches.size()-1].iterations; - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'algorithm_sequence' parameter in configuration file."); - } - - try { - gc = cfg.lookup("gc"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'gcd' parameter in configuration file."); - } - - std::vector support_area; - try { - const Setting& root = cfg.getRoot(); - const Setting &tmp = root["support_area"]; - for (int i = 0; i < tmp.getLength(); ++i) - { - try { - //support_area[i] = Utils::GetDimension(tmp[i]); - support_area.push_back(tmp[i]); - } - catch ( const SettingTypeException &nfex) - { - float ftmp = tmp[i]; - //support_area[i] = Utils::GetDimension(int(ftmp * data_dim[i])); - support_area.push_back(int(ftmp * data_dim[i])); - } - } - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'support_area' parameter in configuration file."); - } - float support_threshold = 0; - try { - support_threshold = cfg.lookup("support_threshold"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'support_threshold' parameter in configuration file."); - } - bool support_threshold_adjust = true; - try { - support_threshold_adjust = cfg.lookup("support_threshold_adjust"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'support_threshold_adjust' parameter in configuration file."); - } - int support_sigma = 0; - try { - support_sigma = cfg.lookup("support_sigma"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'support_sigma' parameter in configuration file."); - } - std::vector support_triggers = ParseTriggers("support"); - int support_alg = -1; - try { - support_alg = algorithm_id_map[cfg.lookup("support_type")]; - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("'No support_type' parameter in configuration file.")).c_str()); - } - support_attr = new Support(data_dim, support_area, support_threshold, support_threshold_adjust, support_sigma, support_triggers, support_alg); - printf("created support\n"); - - int pcdi_alg = 0; - try { - pcdi_alg = algorithm_id_map[cfg.lookup("partial_coherence_type")]; - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("'No partial_coherence_type' parameter in configuration file.")).c_str()); - } - - if (pcdi_alg > 0) - { - std::vector roi; - try { - const Setting& root = cfg.getRoot(); - const Setting &tmp = root["partial_coherence_roi"]; - for (int i = 0; i < tmp.getLength(); ++i) - { - try { - roi.push_back(Utils::GetDimension(tmp[i])); - } - catch ( const SettingTypeException &nfex) - { - float ftmp = tmp[i]; - roi.push_back(Utils::GetDimension(int(ftmp * data_dim[i]))); - } - } - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'partial_coherence_kernel' parameter in configuration file. Setting the dimensions to roi"); - int * kernel = new int[3]; - kernel[0] = roi[0]; - kernel[1] = roi[1]; - kernel[2] = roi[2]; - } - int * kernel = new int[3]; - try { - const Setting& root = cfg.getRoot(); - const Setting &tmp = root["partial_coherence_kernel"]; - for (int i = 0; i < tmp.getLength(); ++i) - { - try { - kernel[i] = Utils::GetDimension(tmp[i]); - } - catch ( const SettingTypeException &nfex) - { - float ftmp = tmp[i]; - kernel[i] = Utils::GetDimension(int(ftmp * data_dim[i])); - } - } - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'partial_coherence_kernel' parameter in configuration file."); - } - - std::vector partial_coherence_trigger = ParseTriggers("partial_coherence"); - bool pcdi_normalize = false; - try { - pcdi_normalize = cfg.lookup("partial_coherence_normalize"); - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("'No partial_coherence_normalize' parameter in configuration file.")).c_str()); - } - bool pcdi_clip = false; - try { - pcdi_clip = cfg.lookup("partial_coherence_clip"); - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("'No partial_coherence_clip' parameter in configuration file.")).c_str()); - } - int pcdi_iter = 1; - try { - pcdi_iter = cfg.lookup("partial_coherence_iteration_num"); - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("'No partial_coherence_iteration_num' parameter in configuration file.")).c_str()); - } - if (partial_coherence_trigger.size() > 0) - { - partial_coherence = new PartialCoherence(roi, kernel, partial_coherence_trigger, pcdi_alg, pcdi_normalize, pcdi_iter, pcdi_clip); - } - } - - try - { - avg_iterations = cfg.lookup("avg_iterations"); - } - catch(const SettingNotFoundException &nfex) - { - printf("No 'avg_iterations' parameter in configuration file."); - } - - try - { - amp_threshold = cfg.lookup("amp_threshold"); - } - catch(const SettingNotFoundException &nfex) - { - printf("No 'amp_threshold' parameter in configuration file."); - } - - try { - amp_threshold_fill_zeros = cfg.lookup("amp_threshold_fill_zeros"); - } - catch (const SettingNotFoundException &nfex) - { - printf("No 'amp_threshold_fill_zeros' parameter in configuration file."); - } - - try { - phase_min = cfg.lookup("phase_min"); - } - catch (const SettingNotFoundException &nfex) - { - printf("No 'phase_min' parameter in configuration file."); - } - - try { - phase_max = cfg.lookup("phase_max"); - } - catch (const SettingNotFoundException &nfex) - { - printf("No 'phase_max' parameter in configuration file."); - } - - try - { - beta = cfg.lookup("beta"); - } - catch (const SettingNotFoundException &nfex) - { - printf("No 'beta' parameter in configuration file."); - } - - try - { - const char *reg_amp = cfg.lookup("regularized_amp"); - if (strcmp (reg_amp, "GAUSS") == 0) - { - regularized_amp = REGULARIZED_AMPLITUDE_GAUSS; - } - else if (strcmp (reg_amp, "POISSON") == 0) - { - regularized_amp = REGULARIZED_AMPLITUDE_POISSON; - } - else if (strcmp (reg_amp, "UNIFORM") == 0) - { - regularized_amp = REGULARIZED_AMPLITUDE_UNIFORM; - } - // else it is initialized - } - catch (const SettingNotFoundException &nfex) - { - printf("No 'regularized_amp' parameter in configuration file."); - } - twin = -1; - try { - twin = cfg.lookup("twin"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'twin' parameter in configuration file."); - } - - try { - matlab_order = cfg.lookup("matlab_order"); - } - catch ( const SettingNotFoundException &nfex) - { - printf("No 'matlab_order' parameter in configuration file., setting to true"); - } -// try { -// data_type = cfg.lookup("data_type"); -// } -// catch ( const SettingNotFoundException &nfex) -// { -// printf((std::string("'No data_type' parameter in configuration file. Setting to double.")).c_str()); -// data_type = "double"; -// } - -} - -void Params::BuildAlgorithmMap() -{ - // hardcoded - algorithm_id_map.insert(std::pair("ER", ALGORITHM_ER)); - algorithm_id_map.insert(std::pair("HIO", ALGORITHM_HIO)); - algorithm_id_map.insert(std::pair("LUCY", ALGORITHM_LUCY)); - algorithm_id_map.insert(std::pair("LUCY_PREV", ALGORITHM_LUCY_PREV)); - algorithm_id_map.insert(std::pair("GAUSS", ALGORITHM_GAUSS)); -} - -std::vector Params::ParseTriggers(std::string trigger_name) -{ - std::vector trigger_iterations; - const Setting& root = cfg.getRoot(); - std::vector triggers; - - try { - const Setting &tmp = root[(trigger_name + std::basic_string("_triggers")).c_str()]; - for (int i =0; i < tmp.getLength(); i++) - { - int start = tmp[i][0]; - int step = tmp[i][1]; - if (tmp[i].getLength() > 2) - { - end = tmp[i][2]; - end = std::min(end, number_iterations); - } - else - { - end = number_iterations; - } - triggers.push_back(Trigger_setting(start, step, end)); - } - } - catch ( const SettingNotFoundException &nfex) - { - printf((std::string("No ") + trigger_name + std::string("_triggers' parameter in configuration file.")).c_str()); - } - - for (int i = 0; i < triggers.size(); i++) - { - for (int j = triggers[i].start_iteration; j <= triggers[i].end_iteration; j += triggers[i].step_iteration) - { - trigger_iterations.push_back(j); - } - } - if (triggers.size() > 1) - { - std::sort(trigger_iterations.begin(), trigger_iterations.end()); - trigger_iterations.erase( unique(trigger_iterations.begin(), trigger_iterations.end()), trigger_iterations.end()); - } - - return trigger_iterations; -} - -//std::string Params::GetDataType() -//{ -// return data_type; -//} - -int Params::GetNumberIterations() -{ - return number_iterations; -} - -d_type Params::GetAmpThreshold() -{ - return amp_threshold; -} - -bool Params::IsAmpThresholdFillZeros() -{ - return amp_threshold_fill_zeros; -} - -d_type Params::GetPhaseMin() -{ - return phase_min; -} - -d_type Params::GetPhaseMax() -{ - return phase_max; -} - -float Params::GetBeta() -{ - return beta; -} - -int Params::GetAvgIterations() -{ - return avg_iterations; -} - -int Params::GetRegularizedAmp() -{ - return regularized_amp; -} - -int Params::GetTwin() -{ - return twin; -} - -Support * Params::GetSupport() -{ - return support_attr; -} - -PartialCoherence * Params::GetPartialCoherence() -{ - return partial_coherence; -} - -std::vector Params::GetAlgSwitches() -{ - return alg_switches; -} - -int Params::GetGC() -{ - return gc; -} - - diff --git a/src_cpp/state.cpp b/src_cpp/state.cpp deleted file mode 100644 index 4f15090..0000000 --- a/src_cpp/state.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// -// state.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#include "stdio.h" -#include "vector" -#include "map" -#include "state.hpp" -#include "parameters.hpp" -#include "support.hpp" -#include "algorithm.hpp" -#include "pcdi.hpp" -#include "arrayfire.h" - -using namespace af; - -// a reference to params object -Params *params; - -// current iteration -int current_iter = -1; -// number of configured iterations for reconstruction -int total_iter_num = 0; - -// The vector of errors indexed by iteration -std::vector errors; - -// current algorithm -Algorithm * current_alg = NULL; -// current index of index switches vector -int alg_switch_index = 0; - -// mapping of algorithm id to an Algorithm object -std::map algorithm_map; - -// a flag indicating whether to update support -bool update_support = false; -// current index in support_triggers vector -int support_triggers_index = 0; - -// partial coherence state -// a flag indicating whether to update partial coherence -bool run_convolution = false; -bool update_kernel = false; -// current index in support_partial_coherence vector -int partial_coherence_triggers_index = 0; - -bool averaging = false; - -bool apply_twin = false; - - -State::State(Params* parameters) -{ - params = parameters; -} - -void State::Init() -{ - total_iter_num = params->GetNumberIterations(); - // create algorithms that are used in algorithm sequence - // and load the objects into a map - for (int i = 0; i < params->GetAlgSwitches().size(); i++) - { - int alg_id = params->GetAlgSwitches()[i].algorithm_id; - if (algorithm_map[alg_id] == 0) - { - MapAlgorithmObject(alg_id); - } - } - current_alg = algorithm_map[params->GetAlgSwitches()[0].algorithm_id]; -} - -State::~State() -{ -} - -void State::MapAlgorithmObject(int alg_id) -{ - // TODO consider refactoring if there are many subclasses - // this method is called only during initialization, so it might be ok - if (alg_id == ALGORITHM_HIO) - { - algorithm_map[alg_id] = new Hio; - } - else if(alg_id == ALGORITHM_ER) - { - algorithm_map[alg_id] = new Er; - } -} - -int State::Next() -{ - if (current_iter++ == total_iter_num - 1) - { - return false; - } - // figure out current alg - if (params->GetAlgSwitches()[alg_switch_index].iterations == current_iter) - // switch to the next algorithm - { - alg_switch_index++; - current_alg = algorithm_map[params->GetAlgSwitches()[alg_switch_index].algorithm_id]; - } - - // check if update support this iteration - if ((params->GetSupport()->GetTriggers().size() > 0) && (params->GetSupport()->GetTriggers()[support_triggers_index] == current_iter)) - { - update_support = true; - support_triggers_index++; - } - else - { - update_support = false; - } - - // calculate if during the iteration should do averaging. - averaging = ( current_iter >= (total_iter_num - params->GetAvgIterations()) ); - - if (current_iter == params->GetTwin()) - { - apply_twin = true; - } - else - { - apply_twin = false; - } - - return true; -} - -Algorithm * State::GetCurrentAlg() -{ - return current_alg; -} - -void State::RecordError(d_type error) -{ - errors.push_back(error); - printf("iter, error %i %fl\n", current_iter, error); -} - -int State::GetCurrentIteration() -{ - return current_iter; -} - -bool State::IsUpdateSupport() -{ - return update_support; -} - -bool State::IsUpdatePartialCoherence() -{ - return update_kernel; -} - -bool State::IsApplyTwin() -{ - return apply_twin; -} - -bool State::IsAveragingIteration() -{ - return averaging; -} - -std::vector State::GetErrors() -{ - return errors; -} - - diff --git a/src_cpp/support.cpp b/src_cpp/support.cpp deleted file mode 100644 index cbe8c4a..0000000 --- a/src_cpp/support.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// -// support.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/15/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#include "stdio.h" -#include "support.hpp" -#include "parameters.hpp" -#include "util.hpp" - - -Support::Support(const dim4 data_dim, std::vector support_area, float th, bool threshold_adjust, int sgma, std::vector support_triggers, int alg) -{ - threshold = th; - threshold_adjust = threshold_adjust; - sigma = sgma; - triggers = support_triggers; - algorithm = alg; - af::array ones = constant(1, Utils::Int2Dim4(support_area), u32); - support_array = Utils::PadAround(ones, data_dim, 0); - - printf("support sum %i\n", sum(support_array)); - - if (alg == ALGORITHM_GAUSS) - { - int alpha = 1; - //distribution = InitDistribution(data_dim, sigma, alpha); - d_type *sigmas = new d_type[nD]; - for (int i=0; i(convag); - convag = convag/max_convag; - printf("convag sum max %f \n", sum(convag)); - support_array = (convag >= threshold); - -// if the support is too small, adjust threshold -// if (threshold_adjust) -// { -// dim4 dims = ds_image.dims(); -// float min_support_norm = .01 * int(dims[0])*int(dims[1])*int(dims[2]); -// printf("min support norm %f\n", min_support_norm); -// float temp_threshold = threshold; -// while (sum(pow(abs(support_array), 2)) < min_support_norm ) -// { -// temp_threshold = temp_threshold/10; -// support_array = (convag >= (temp_threshold * max_convag)); -// printf("in while loop support sum %f\n", sum(support_array)); -// } -// } - printf("support sum %f\n", sum(support_array)); -} - -std::vector Support::GetTriggers() -{ - return triggers; -} - -int Support::GetTriggerAlgorithm() -{ - return algorithm; -} - -int Support::GetSigma() -{ - return sigma; -} - -float Support::GetThreshold() -{ - return threshold; -} - -af::array Support::GetSupportArray(bool twin) -{ - if (twin) - { - dim4 dims = support_array.dims(); - af::array temp = constant(0, dims, u32); - temp( af::seq(0, dims[0]/2-1), af::seq(0, dims[1]/2-1), span, span) = 1; - return support_array * temp; - } - else - { - return support_array; - } -} - -af::array Support::GaussConvFft(af::array ds_image_abs) -{ - d_type image_sum = sum(ds_image_abs); - af::array shifted = Utils::ifftshift(ds_image_abs); - af::array rs_amplitudes = Utils::fft(shifted); - af::array rs_amplitudes_cent = Utils::ifftshift(rs_amplitudes); - - af::array amp_dist = rs_amplitudes_cent * distribution; - shifted = Utils::ifftshift(amp_dist); - af::array convag_compl = Utils::ifft(shifted); - af::array convag = (Utils::ifftshift(convag_compl)); - convag = real(convag); - convag(convag < 0) = 0; - d_type correction = image_sum/sum(convag); - convag *= correction; - return convag; -} - -//af::array Support::GaussConvFft(af::array ds_image_abs) -//{ -// -// d_type image_sum = sum(ds_image_abs); -// printf("abs image sum %f\n", image_sum); -// af::array rs_amplitudes = Utils::fft(ds_image_abs); -// dim4 dims = rs_amplitudes.dims(); -// af::shift(rs_amplitudes, dims[0]/2, dims[1]/2, dims[2]/2, dims[3]/2); -// af::array amp_distrib = Utils::ifft(rs_amplitudes * distribution); -// af::shift(amp_distrib, dims[0]/2, dims[1]/2, dims[2]/2, dims[3]/2); -// af::array convag = real(amp_distrib); -// convag(convag < 0) = 0; -// d_type correction = image_sum/sum(convag); -// convag *= correction; -// return convag; -//} - - diff --git a/src_cpp/util.cpp b/src_cpp/util.cpp deleted file mode 100644 index 898de57..0000000 --- a/src_cpp/util.cpp +++ /dev/null @@ -1,344 +0,0 @@ -#include "util.hpp" -#include "common.h" -#include -#include -#include "cstdio" - -using namespace af; - -int Utils::GetDimension(int dim) -{ - int new_dim = dim; - while (! IsDimCorrect(new_dim)) - { - new_dim++; - } - return new_dim; -} - -bool Utils::IsDimCorrect(int dim) -{ - int sub = dim; - while (sub % 2 == 0) - { - sub = sub/2; - } - while (sub % 3 == 0) - { - sub = sub/3; - } - while (sub % 5 == 0) - { - sub = sub/5; - } - if (sub == 1) - return true; - else - return false; -} - -af::dim4 Utils::Int2Dim4(std::vector dim) -{ - for (int i = dim.size(); i<4; i++) - { - dim.push_back(1); - } - return af::dim4(dim[0], dim[1], dim[2], dim[3]); -} - - -af::array Utils::CropCenter(af::array arr, af::dim4 roi) -{ - dim4 dims = arr.dims(); - int beginning[4]; //dim4 contains four dimensions - int ending[4]; - for (int i=0; i<4; i++) - { - beginning[i] = int((dims[i] - roi[i])/2); - ending[i] = beginning[i] + roi[i]; - } - return arr(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)); -} - -//af::array Utils::fftshift(af::array arr) -//{ -// return af::shift(arr, int(arr.dims()[0]/2-1), int(arr.dims()[1]/2-1), int(arr.dims()[2]/2-1), int(arr.dims()[3]/2)); -//} - -af::array Utils::fftshift(af::array arr) -{ - return af::shift(arr, ceil(arr.dims()[0]/2)-1, ceil(arr.dims()[1]/2)-1, ceil(arr.dims()[2]/2)-1, ceil(arr.dims()[3]/2)-1); -} - -af::array Utils::ifftshift(af::array arr) -{ - return af::shift(arr, ceil(arr.dims()[0]/2), ceil(arr.dims()[1]/2), ceil(arr.dims()[2]/2), ceil(arr.dims()[3]/2)); -} - -af::array Utils::fft(af::array arr) -{ - if (nD == 3) - { - return fft3(arr); - } - else if (nD == 2) - { - return fft2(arr); - } -} - -af::array Utils::ifft(af::array arr) -{ - - if (nD == 3) - { - return ifft3(arr); - } - else if (nD == 2) - { - return ifft2(arr); - } -} - -//af::array Utils::CropCenterZeroPad(af::array arr, int * roi) -//{ -// dim4 dims = arr.dims(); -// // Find the coordinates sub-array of the given dimensions in the center of array arr -// int x0 = int(dims[0]/2 - roi[0]/2); -// int x1 = x0 + roi[0]; -// int y0 = int(dims[1]/2 - roi[1]/2); -// int y1 = y0 + roi[1]; -// int z0 = int(dims[2]/2 - roi[2]/2); -// int z1 = z0 + roi[2]; -// -// af::array cropped = arr.copy() *0; -// // select the subarray -// cropped(seq(x0 + 1, x1), seq(y0 + 1, y1), seq(z0 + 1, z1)) = arr(seq(x0 + 1, x1), seq(y0 + 1, y1), seq(z0 + 1, z1)); -// return cropped; -// -//} - -//af::array Utils::CropRoi(af::array arr, int * roi) -//{ -// return arr(seq(0, roi[0]-1), seq(0, roi[1]-1), seq(0, roi[2]-1)); -//} - -//af::array Utils::CenterMax(af::array arr, int * kernel) -//{ -// printf("in CenterMax dims %i %i %i %i\n", arr.dims()[0], arr.dims()[1], arr.dims()[2], arr.dims()[0]*arr.dims()[1]*arr.dims()[2]); -// //find indexes of max -// double *arr_values = abs(arr).host(); -// std::vector v(arr_values, arr_values + arr.elements()); -// std::vector::iterator result = std::max_element(v.begin(), v.end()); -// int max_offset = result - v.begin(); -// printf("maximum value, %i %f\n", max_offset, v[max_offset]); -// delete [] arr_values; -// int x_max = max_offset % arr.dims()[0]; -// int y_max = max_offset / arr.dims()[0] % arr.dims()[1]; -// int z_max = max_offset/ (arr.dims()[0] * arr.dims()[1]); -// printf("max indexes %i %i %i\n", x_max, y_max, z_max); -// -// -// af::array shifted = af::shift(arr, arr.dims()[0]/2-x_max, arr.dims()[1]/2-y_max, arr.dims()[2]/2-z_max ); -// return shifted; -// -//} -// -void Utils::GetMaxIndices(af::array arr, int* indices) -{ - //find indexes of max - d_type *arr_values = abs(arr).host(); - std::vector v(arr_values, arr_values + arr.elements()); - std::vector::iterator result = std::max_element(v.begin(), v.end()); - int max_offset = result - v.begin(); - printf("maximum value, %i %f\n", max_offset, v[max_offset]); - delete [] arr_values; - indices[0] = max_offset % arr.dims()[0]; - indices[1] = max_offset / arr.dims()[0] % arr.dims()[1]; - indices[2] = max_offset/ (arr.dims()[0] * arr.dims()[1]); - printf("offset, ind1, ind2 ind3 %i %i %i %i\n", max_offset, indices[0], indices[1], indices[2]); -} - -//af::array Utils::CenterMax(af::array arr) -//{ -// printf("in CenterMax dims %i %i %i\n", arr.dims()[0], arr.dims()[1], arr.dims()[2]); -// //find indexes of max -// d_type *arr_values = abs(arr).host(); -// std::vector v(arr_values, arr_values + arr.elements()); -// std::vector::iterator result = std::max_element(v.begin(), v.end()); -// int max_offset = result - v.begin(); -// printf("maximum offset, value, %i %f\n", max_offset, v[max_offset]); -// delete [] arr_values; -// int x_max = max_offset % arr.dims()[0]; -// int y_max = max_offset / arr.dims()[0] % arr.dims()[1]; -// int z_max = max_offset/ (arr.dims()[0] * arr.dims()[1]); -// printf("CenterMax max indexes %i %i %i\n", x_max, y_max, z_max); -// -// return af::shift(arr, arr.dims()[0]/2 -x_max, arr.dims()[1]-y_max, arr.dims()[2]-z_max );; -//} - -//af::array Utils::ShiftMax(af::array arr, int * kernel) -//{ -// //find indexes of max -// d_type *arr_values = abs(arr).host(); -// std::vector v(arr_values, arr_values + arr.elements()); -// std::vector::iterator result = std::max_element(v.begin(), v.end()); -// int max_offset = result - v.begin(); -// printf("maximum value, %i %f\n", max_offset, v[max_offset]); -// delete [] arr_values; -// int x_max = max_offset % arr.dims()[0]; -// int y_max = max_offset / arr.dims()[0] % arr.dims()[1]; -// int z_max = max_offset/ (arr.dims()[0] * arr.dims()[1]); -// printf("ShiftMax max indexes %i %i %i\n", x_max, y_max, z_max); -// -// return arr; -//} - -af::array Utils::ReverseGaussDistribution(af::dim4 data_dim, d_type * sgma, int alpha) -{ - // calculate multipliers - int dimension = nD; //this should be changed to determine size from sgma - //initialize first element of the grid, assuming at least one dimension - d_type multiplier = - 0.5 * alpha/pow(sgma[0],2); - af::array exponent = pow(data_dim[0]/2 - abs(range(data_dim, 0)-(data_dim[0]-1)/2.0), 2) * multiplier; - af::array grid = exp(exponent); - - //add grid in other dimensions - for (int i = 1; i(grid); - grid = grid/grid_total; - return grid; -} - -af::array Utils::GaussDistribution(af::dim4 data_dim, d_type * sgma, int alpha) -{ - printf("in gauss distr\n"); - // calculate multipliers - int dimension = nD; //this should be changed to determine size from sgma -// af::array gridx = (range(data_dim, 1)-(data_dim[1]-1)/2.0); // the range method produces wrong data - d_type multiplier = - 0.5 * alpha/pow(sgma[1],2); - double val; - std::vector gridx_v; - for (int i = 0; i< data_dim[2]; i++) - { - for (int j=0; j< data_dim[1]; j++) - { - val = -(data_dim[1]-1)/2.0 + j; - for (int k=0; k< data_dim[0]; k++) - { - gridx_v.push_back(val); - } - } - } - af::array gridx(data_dim, &gridx_v[0]); - af::array exponent = pow( gridx ,2)* multiplier; - af::array gridxx = exp(exponent); - - multiplier = - 0.5 * alpha/pow(sgma[0],2); - std::vector gridy_v; - for (int i = 0; i< data_dim[2]; i++) - { - for (int j=0; j< data_dim[1]; j++) - { - for (int k=0; k< data_dim[0]; k++) - { - val = -(data_dim[0]-1)/2.0 + k; - gridy_v.push_back(val); - } - } - } - af::array gridy(data_dim, &gridy_v[0]); - exponent = pow( gridy ,2)* multiplier; - af::array gridyy = exp(exponent); - - multiplier = - 0.5 * alpha/pow(sgma[2],2); - std::vector gridz_v; - for (int i = 0; i< data_dim[2]; i++) - { - val = -(data_dim[2]-1)/2.0 + i; - for (int j=0; j< data_dim[1]; j++) - { - for (int k=0; k< data_dim[0]; k++) - { - gridz_v.push_back(val); - } - } - } - af::array gridz(data_dim, &gridz_v[0]); - exponent = pow( gridz ,2)* multiplier; - af::array gridzz = exp(exponent); - - af::array grid = gridxx * gridyy * gridzz; - d_type grid_total = sum(grid); - grid = grid/grid_total; - return grid; -} - -//af::array Utils::GaussDistribution(af::dim4 data_dim, d_type * sgma, int alpha) -//{ -// // calculate multipliers -// int dimension = nD; //this should be changed to determine size from sgma -// //initialize first element of the grid, assuming at least one dimension -// d_type multiplier = - 0.5 * alpha/pow(sgma[0],2); -// af::array exponent = pow( range(data_dim, 0)-(data_dim[0]-1)/2.0 ,2)* multiplier; -// af::array grid = exp(exponent); -// -// //add grid in other dimensions -// for (int i = 1; i(grid); -// grid = grid/grid_total; -// printf("grid sum, norm %f %f\n", sum(grid), sum(pow(grid,2))); -// return grid; -//} - -af::array Utils::PadAround(af::array arr, af::dim4 new_dims, d_type pad) -{ - //af::array padded = constant(pad, new_dims, (af_dtype) dtype_traits::ctype); - - af::array padded = constant(pad, new_dims, arr.type()); - int beginning[4]; //dim4 contains four dimensions - int ending[4]; - for (int i=0; i<4; i++) - { - beginning[i] = int(new_dims[i]/2 - arr.dims()[i]/2); - ending[i] = beginning[i] + arr.dims()[i]; - } - - padded(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)) = arr; - return padded; -} - -af::array Utils::PadAround(af::array arr, af::dim4 new_dims, int pad) -{ - af::array padded = constant(pad, new_dims, u32); - int beginning[4]; //dim4 contains four dimensions - int ending[4]; - for (int i=0; i<4; i++) - { - beginning[i] = int(new_dims[i]/2 - arr.dims()[i]/2); - ending[i] = beginning[i] + arr.dims()[i]; - } - - padded(seq(beginning[0], ending[0]-1), seq(beginning[1], ending[1]-1), seq(beginning[2], ending[2]-1), seq(beginning[3], ending[3]-1)) = arr; - return padded; -} - -af::array Utils::GetRatio(af::array divident, af::array divisor) -{ - af::array divisor_copy = divisor.copy(); - divisor_copy(divisor == 0) = 1; - return divident/divisor_copy; -} diff --git a/src_cpp/worker.cpp b/src_cpp/worker.cpp deleted file mode 100644 index 097cce9..0000000 --- a/src_cpp/worker.cpp +++ /dev/null @@ -1,266 +0,0 @@ -// -// worker.cpp -// ArrayFire-OpenCL -// -// Created by Barbara Frosik on 8/12/16. -// Copyright © 2016 ArrayFire. All rights reserved. -// - -#include "fstream" -#include "worker.hpp" -#include "cstdio" -#include "cstdlib" -#include "math.h" -#include "vector" -#include "map" -#include "parameters.hpp" -#include "support.hpp" -#include "pcdi.hpp" -#include "state.hpp" -#include "common.h" -#include "algorithm.hpp" -#include "util.hpp" - -af::array data; // this is abs -int num_points; -d_type norm_data; -int current_iteration; -af::array ds_image; -int aver_iter; -std::vector aver_v; -std::vector support_vector; -std::vector coherence_vector; - - -Reconstruction::Reconstruction(af::array image_data, af::array guess, const char* config_file) -{ - data = image_data; - ds_image = guess; - params = new Params(config_file, data.dims()); - state = new State(params); -} - -void Reconstruction::Init() -{ - // initialize other components - state->Init(); - support = params->GetSupport(); - partialCoherence = params->GetPartialCoherence(); - aver_iter = params->GetAvgIterations(); - if (partialCoherence != NULL) - { - partialCoherence->Init(data); - } - norm_data = GetNorm(data); - num_points = data.elements(); - // multiply the rs_amplitudes by max element of data array and the norm - d_type max_data = af::max(data); - ds_image *= max_data * GetNorm(ds_image); - -// the next two lines are for testing it sets initial guess to initial support -// af::array temp = support->GetSupportArray(); -// ds_image = complex(temp.as((af_dtype) dtype_traits::ctype), 0.0).as(c64); - - ds_image *= support->GetSupportArray(); - printf("initial image norm %f\n", GetNorm(ds_image)); - -} - - -void Reconstruction::Iterate() -{ - while (state->Next()) - { - if (state->IsUpdateSupport()) - { - support->Update(abs(ds_image).copy()); - } - - current_iteration = state->GetCurrentIteration(); - if (params->GetGC() && (current_iteration+1) % params->GetGC() == 0) - af::deviceGC(); - Algorithm * alg = state->GetCurrentAlg(); - alg->RunAlgorithm(this); - - Average(); - } - printf("final image\n"); - - if (aver_v.size() > 0) - { - printf("final averaging\n"); - af::array aver_a(ds_image.dims(), &aver_v[0]); - af::array ratio = Utils::GetRatio(aver_a, abs(ds_image)); - ds_image *= ratio/aver_iter; - } - ds_image *= support->GetSupportArray(); - VectorizeSupport(); - if (partialCoherence != NULL) - { - VectorizeCoherence(); - } -} - -af::array Reconstruction::ModulusProjection() -{ - printf("------------------current iteration %i -----------------\n", current_iteration); - af::array rs_amplitudes; - dim4 dims = data.dims(); - rs_amplitudes = Utils::ifft(ds_image)*num_points; - - - printf("data norm, ampl norm before ratio %fl %fl\n", GetNorm(data), GetNorm(rs_amplitudes)); - state->RecordError( GetNorm(abs(rs_amplitudes)(rs_amplitudes > 0)-data(rs_amplitudes > 0))/norm_data ); - - if ((partialCoherence == NULL) || (partialCoherence->GetTriggers().size() == 0)) - { - printf("applying ratio\n"); - //rs_amplitudes = data * exp(af::complex(0, af::arg(rs_amplitudes))); - af::array ratio = Utils::GetRatio(data, abs(rs_amplitudes)); - rs_amplitudes *= ratio; - } - else - { - if (current_iteration >= partialCoherence->GetTriggers()[0]) - { - printf("coherence using lucy\n"); - af::array abs_amplitudes = abs(rs_amplitudes).copy(); - af::array converged = partialCoherence->ApplyPartialCoherence(abs_amplitudes, current_iteration); - af::array ratio = Utils::GetRatio(data, abs(converged)); - printf("ratio norm %f\n", GetNorm(ratio)); - - rs_amplitudes *= ratio; - } - else - { - printf("applying ratio\n"); - //rs_amplitudes = data * exp(af::complex(0, af::arg(rs_amplitudes))); - af::array ratio = Utils::GetRatio(data, abs(rs_amplitudes)); - rs_amplitudes *= ratio; - } - printf("setting previous\n"); - partialCoherence->SetPrevious(abs(rs_amplitudes)); - } - printf("ampl norm after ratio %fl\n", GetNorm(rs_amplitudes)); - - if (params->GetGC() && current_iteration % params->GetGC() == 0) - af::deviceGC(); - - return Utils::fft(rs_amplitudes)/num_points; - -} - -void Reconstruction::ModulusConstrainEr(af::array ds_image_raw) -{ - printf("er\n"); - printf("image norm before support %fl\n", GetNorm(ds_image_raw)); - //ds_image = ds_image_raw * support->GetSupportArray(state->IsApplyTwin()); - af::array support_array = support->GetSupportArray(state->IsApplyTwin()); - ds_image = ds_image_raw * support_array; - - printf("image norm after support %fl\n", GetNorm(ds_image)); -} - -void Reconstruction::ModulusConstrainHio(af::array ds_image_raw) -{ - - printf("hio\n"); - printf("image norm before support %fl\n",GetNorm(ds_image_raw)); - //ds_image(support->GetSupportArray(state->IsApplyTwin()) == 0) = (ds_image - ds_image_raw * params->GetBeta())(support->GetSupportArray(state->IsApplyTwin()) == 0); - af::array support_array = support->GetSupportArray(state->IsApplyTwin()); - af::array adjusted_calc_image = ds_image_raw * params->GetBeta(); - af::array combined_image = ds_image - adjusted_calc_image; - ds_image = ds_image_raw; - ds_image(support_array == 0) = combined_image(support_array == 0); - printf("image norm after support %fl\n",GetNorm(ds_image)); -} - -void Reconstruction::Average() -{ - if (state->IsAveragingIteration()) - { - printf("average\n"); - // int aver_num = params->GetAvgIterations(); - af::array abs_image = abs(ds_image).copy(); - d_type *image_v = abs_image.host(); - std::vector v(image_v, image_v + ds_image.elements()); - if (aver_v.size() == 0) - { - printf("creating aver vector\n"); - for (int i = 0; i < v.size(); i++) - { - aver_v.push_back(v[i]); - } - } - else - { - printf("adding aver vector\n"); - for (int i = 0; i < v.size(); i++) - { - aver_v[i] += v[i]; - } - } - - delete [] image_v; - } -} - -double Reconstruction::GetNorm(af::array arr) -{ - return sum(pow(abs(arr), 2)); -} - -void Reconstruction::VectorizeSupport() -{ - af::array a = support->GetSupportArray().as(f32); - float *support_v = a.host(); - std::vector v(support_v, support_v + a.elements()); - support_vector = v; - delete [] support_v; - } - -void Reconstruction::VectorizeCoherence() -{ - // get the partial coherence as double, so it will work for float and double data types - af::array a = partialCoherence->GetKernelArray(); - d_type *coherence_v = a.host(); - std::vector v(coherence_v, coherence_v + a.elements()); - coherence_vector = v; - delete [] coherence_v; -} - -int Reconstruction::GetCurrentIteration() -{ - return state->GetCurrentIteration(); -} - -af::array Reconstruction::GetImage() -{ - return ds_image; -} - -std::vector Reconstruction::GetErrors() -{ - return state->GetErrors(); -} - -std::vector Reconstruction::GetSupportVector() -{ - return support_vector; -} - -std::vector Reconstruction::GetCoherenceVector() -{ - return coherence_vector; -} - -std::vector Reconstruction::GetCoherenceVectorR() -{ - return coherence_vector; -} - -std::vector Reconstruction::GetCoherenceVectorI() -{ - return coherence_vector; -} - diff --git a/src_py/controller/reconstruction.py b/src_py/controller/reconstruction.py deleted file mode 100755 index df0a02b..0000000 --- a/src_py/controller/reconstruction.py +++ /dev/null @@ -1,336 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# ######################################################################### -# Copyright (c) 2016, UChicago Argonne, LLC. All rights reserved. # -# # -# Copyright 2016. UChicago Argonne, LLC. This software was produced # -# under U.S. Government contract DE-AC02-06CH11357 for Argonne National # -# Laboratory (ANL), which is operated by UChicago Argonne, LLC for the # -# U.S. Department of Energy. The U.S. Government has rights to use, # -# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR # -# UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR # -# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is # -# modified to produce derivative works, such modified software should # -# be clearly marked, so as not to confuse it with the version available # -# from ANL. # -# # -# Additionally, redistribution and use in source and binary forms, with # -# or without modification, are permitted provided that the following # -# conditions are met: # -# # -# * Redistributions of source code must retain the above copyright # -# notice, this list of conditions and the following disclaimer. # -# # -# * Redistributions in binary form must reproduce the above copyright # -# notice, this list of conditions and the following disclaimer in # -# the documentation and/or other materials provided with the # -# distribution. # -# # -# * Neither the name of UChicago Argonne, LLC, Argonne National # -# Laboratory, ANL, the U.S. Government, nor the names of its # -# contributors may be used to endorse or promote products derived # -# from this software without specific prior written permission. # -# # -# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS # -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago # -# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # -# POSSIBILITY OF SUCH DAMAGE. # -# ######################################################################### - -""" -Please make sure the installation :ref:`pre-requisite-reference-label` are met. -This module controls the reconstruction process. The user has to provide parameters such as type of processor, data, and configuration. -The processor specifies which library will be used by CFM (Calc Fast Module) that performs the processor intensive calculations. The module -can be run on cpu, or gpu. Depending on the gpu hardware and library, one can use opencl or cuda library. -The module starts the data preparation routines, calls for reconstruction using the CFM, and prepares the reconstructed data for -visualization. -""" - -import numpy as np -import src_py.utilities.utils as ut -import src_py.utilities.CXDVizNX as cx -import pylibconfig2 as cfg -import os -import scipy.fftpack as sf -import scipy.io as sio -import src_py.cyth.bridge_cpu as bridge_cpu -import src_py.cyth.bridge_opencl as bridge_opencl -#import src_py.cyth.bridge_cuda as bridge_cuda -#import tifffile as tf - - -__author__ = "Barbara Frosik" -__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC." -__docformat__ = 'restructuredtext en' -__all__ = ['read_config', - 'prepare_data', - 'do_reconstruction', - 'reconstruction'] - -def read_config(config): - """ - This function gets configuration file. It checks if the file exists and parses it into a map. - - Parameters - ---------- - config : str - configuration file name, including path - - Returns - ------- - config_map : dict - a map containing parsed configuration, None if the given file does not exist - """ - - if os.path.isfile(config): - with open(config, 'r') as f: - config_map = cfg.Config(f.read()) - return config_map; - else: - return None - -def prepare_data(config_map, data): - """ - This function prepares raw data for reconstruction. It uses configured parameters. The preparation consists of the following steps: - 1. clearing the noise - the values below an amplitude threshold are set to zero - 2. removing the "aliens" - aliens are areas that are effect of interference. The area is manually set in a configuration file - after inspecting the data. - 3. binning - adding amplitudes of several consecutive points. Binning can be done in any dimension. - 4. amplitudes are set to sqrt - 5. centering - finding the greatest amplitude and locating it at a center of new array. Typically several new rows/columns/slices - are added. These are filled with zeros. When changing the dimension the code finds the smallest possible dimension that is - supported by opencl library (multiplier of 2, 3, and 5). - 6. shift in place - shift the zero-frequency component to the center of the spectrum - - Parameters - ---------- - config_map : dict - configuration map - - data : array - a 3D np array containing experiment data - - Returns - ------- - data : array - a 3D np array containing data after the preprocessing - """ - - # zero out the noise - data = np.where(data < config_map.amp_threshold, 0, data) - print (data.shape) - - # zero out the aliens - try: - aliens = config_map.aliens - for alien in aliens: - data[alien[0]:alien[3], alien[1]:alien[4], alien[2]:alien[5]]=0 - except AttributeError: - pass - - # do binning - try: - binsizes = config_map.binning - data = ut.binning(data, binsizes) - except AttributeError: - pass - - # square root data - data = np.sqrt(data) - - # get centered array - try: - center_shift = tuple(config_map.center_shift) - except AttributeError: - center_shift = (0,0,0) - - data = ut.get_centered(data, center_shift) - - # zero pad array - try: - pad = tuple(config_map.zero_pad) - except AttributeError: - center_shift = (0,0,0) - data = ut.zero_pad(data, pad) - - # shift data - data=sf.fftshift(data) - return data - -def fast_module_reconstruction(proc, data, conf): - """ - This function calls a bridge method corresponding to the requested processor type. The bridge method is an access to the CFM - (Calc Fast Module). When reconstruction is completed the function retrieves results from the CFM. - - Parameters - ---------- - proc : str - a string indicating the processor type - - data : array - a 3D np array containing pre-processed experiment data - - conf : dict - configuration map - - Returns - ------- - image_r : array - a 3D np real part array containing reconstructed image - - image_i : array - a 3D np imaginary part array containing reconstructed image - - er : array - a vector containing mean error for each iteration - """ - if proc == 'cpu': - bridge = bridge_cpu - elif proc == 'opencl': - bridge = bridge_opencl - # elif proc == 'cuda': - # bridge = bridge_cuda - - data = np.swapaxes(data,1,2) - - dims = data.shape - dims1 = (dims[2], dims[1], dims[0]) - print 'data norm in reconstruction', sum(sum(sum(abs(data)**2))) - fast_module = bridge.PyBridge() - - data_l = data.flatten().tolist() - fast_module.start_calc(data_l, dims1, conf) - er = fast_module.get_errors() - image_r = np.asarray(fast_module.get_image_r()) - image_i = np.asarray(fast_module.get_image_i()) - image = image_r + 1j*image_i - # normalize image - mx = max(np.absolute(image).ravel().tolist()) - image = image/mx - support = np.asarray(fast_module.get_support()) - coherence = np.asarray(fast_module.get_coherence()) - - image = np.reshape(image, dims) - support = np.reshape(support, dims) - - image = np.swapaxes(image, 2,0) - support = np.swapaxes(support, 2,0) - image = np.swapaxes(image, 1, 0) - support = np.swapaxes(support, 1, 0) - - if coherence.shape[0] > 1: - coh_size = int(round(coherence.shape[0] ** (1. / 3.))) - coh_dims = (coh_size, coh_size, coh_size,) - coherence = np.reshape(coherence, coh_dims) - coherence = np.swapaxes(coherence, 2, 0) - coherence = np.swapaxes(coherence, 1, 0) - else: - coherence = None - - return image, support, coherence, er - - -def write_simple(arr, filename): - from tvtk.api import tvtk, write_data - - id=tvtk.ImageData() - id.point_data.scalars=abs(arr.ravel(order='F')) - id.dimensions=arr.shape - write_data(id, filename) - -def reconstruction(proc, filename, conf): - """ - This function is called by the user. It checks whether the data is valid and configuration file exists. - It calls function to pre-process the data, and then to run reconstruction. - The reconstruction results, image and errors are returned. - - Parameters - ---------- - proc : str - a string indicating the processor type - - filename : str - name of a file containing experiment data - - conf : str - configuration file name - - Returns - ------- - image : array - a 3D np array containing reconstructed image - - er : array - a vector containing mean error for each iteration - """ - - data = ut.get_array_from_tif(filename) - if len(data.shape) > 3: - print ("this program supports 3d images only") - return - - config_map = read_config(conf) - if config_map is None: - print ("can't read configuration file") - return None, None - - data = prepare_data(config_map, data) - # save prepared data in .mat file if configured - try: - save_data = config_map.save_data - if save_data: - data_dict = {} - data_dict['data'] = data - sio.savemat('data.mat', data_dict) - except AttributeError: - pass - - image, support, coherence, errors = fast_module_reconstruction(proc, data, conf) - - # save image and support in .mat files if configured - try: - save_results = config_map.save_data - if save_results: - image_dict = {} - support_dict = {} - image_dict['image'] = image - sio.savemat('image.mat', image_dict) - support_dict['support'] = support - sio.savemat('support.mat', support_dict) - except AttributeError: - pass - - - try: - res_dir = config_map.res_dir - if not res_dir.endswith('/'): - res_dir = res_dir + '/' - if not os.path.exists(res_dir): - os.makedirs(res_dir) - except AttributeError: - res_dir = '' - write_simple(image, res_dir + "simple_amp_ph.vtk") - write_simple(support, res_dir + "simple_support.vtk") - - cx.save_CX(conf, image, support, res_dir + 'cx') - - if coherence is not None: - write_simple(coherence, res_dir + "simple_coh.vtk") - - # plot error - - - print 'image, support shape', image.shape, support.shape - - - - diff --git a/src_py/cyth/bridge_cpu.pyx b/src_py/cyth/bridge_cpu.pyx deleted file mode 100644 index 9377b6a..0000000 --- a/src_py/cyth/bridge_cpu.pyx +++ /dev/null @@ -1,47 +0,0 @@ -# distutils: language = c++ -# distutils: include_dirs = ['include', 'AF_DIR/include', 'LC_DIR/lib',] -# distutils: sources = ['src_cpp/algorithm.cpp', 'src_cpp/bridge.cpp', 'src_cpp/manager.cpp', 'src_cpp/parameters.cpp', 'src_cpp/pcdi.cpp', 'src_cpp/state.cpp', 'src_cpp/support.cpp', 'src_cpp/util.cpp', 'src_cpp/worker.cpp'] -# distutils: libraries = ['afcpu', 'config++',] -# distutils: library_dirs = ['AF_DIR/lib', 'LC_DIR/lib/.libs', ] - -from libcpp.vector cimport vector -from libcpp.string cimport string - - -cdef extern from "../include/bridge.hpp": - cdef cppclass Bridge: - Bridge() except + - void StartCalcWithGuess(vector[float], vector[float], vector[float], vector[int], string) - void StartCalc(vector[float], vector[int], string) - void StartCalcMultiple(vector[float], vector[int], string, int) - vector[def_type] GetImageR() - vector[def_type] GetImageI() - vector[def_type] GetErrors() - vector[float] GetSupportV() - vector[def_type] GetCoherenceV() - - -cdef class PyBridge: - cdef Bridge *thisptr - def __cinit__(self): - self.thisptr = new Bridge() - def __dealloc__(self): - del self.thisptr - def start_calc_with_guess(self, data_r, guess_r, guess_i, dims, config): - self.thisptr.StartCalcWithGuess(data_r, guess_r, guess_i, dims, config.encode()) - def start_calc(self, data_r, dims, config): - self.thisptr.StartCalc(data_r, dims, config.encode()) - def start_calc_multiple(self, data_r, dims, config, no_threads): - self.thisptr.StartCalcMultiple(data_r, dims, config, no_threads) - def get_image_r(self): - return self.thisptr.GetImageR() - def get_image_i(self): - return self.thisptr.GetImageI() - def get_errors(self): - return self.thisptr.GetErrors() - def get_support(self): - return self.thisptr.GetSupportV() - def get_coherence(self): - return self.thisptr.GetCoherenceV() - - diff --git a/src_py/cyth/bridge_cuda.pyx b/src_py/cyth/bridge_cuda.pyx deleted file mode 100755 index d3ad9fd..0000000 --- a/src_py/cyth/bridge_cuda.pyx +++ /dev/null @@ -1,45 +0,0 @@ -# distutils: language = c++ -# distutils: include_dirs = ['include', 'AF_DIR/include', 'LC_DIR/lib',] -# distutils: sources = ['src_cpp/algorithm.cpp', 'src_cpp/bridge.cpp', 'src_cpp/manager.cpp', 'src_cpp/parameters.cpp', 'src_cpp/pcdi.cpp', 'src_cpp/state.cpp', 'src_cpp/support.cpp', 'src_cpp/util.cpp', 'src_cpp/worker.cpp'] -# distutils: libraries = ['afcuda', 'config++',] -# distutils: library_dirs = ['AF_DIR/lib', 'LC_DIR/lib/.libs', ] - -from libcpp.vector cimport vector -from libcpp.string cimport string - -cdef extern from "../include/bridge.hpp": - cdef cppclass Bridge: - Bridge() except + - void StartCalcWithGuess(vector[float], vector[float], vector[float], vector[int], string) - void StartCalc(vector[float], vector[int], string) - void StartCalcMultiple(vector[float], vector[int], string, int) - vector[def_type] GetImageR() - vector[def_type] GetImageI() - vector[def_type] GetErrors() - vector[float] GetSupportV() - vector[def_type] GetCoherenceV() - - -cdef class PyBridge: - cdef Bridge *thisptr - def __cinit__(self): - self.thisptr = new Bridge() - def __dealloc__(self): - del self.thisptr - def start_calc_with_guess(self, data_r, guess_r, guess_i, dims, config): - self.thisptr.StartCalcWithGuess(data_r, guess_r, guess_i, dims, config.encode()) - def start_calc(self, data_r, dims, config): - self.thisptr.StartCalc(data_r, dims, config.encode()) - def start_calc_multiple(self, data_r, dims, config, no_threads): - self.thisptr.StartCalcMultiple(data_r, dims, config, no_threads) - def get_image_r(self): - return self.thisptr.GetImageR() - def get_image_i(self): - return self.thisptr.GetImageI() - def get_errors(self): - return self.thisptr.GetErrors() - def get_support(self): - return self.thisptr.GetSupportV() - def get_coherence(self): - return self.thisptr.GetCoherenceV() - diff --git a/src_py/cyth/bridge_opencl.pyx b/src_py/cyth/bridge_opencl.pyx deleted file mode 100755 index 644c275..0000000 --- a/src_py/cyth/bridge_opencl.pyx +++ /dev/null @@ -1,45 +0,0 @@ -# distutils: language = c++ -# distutils: include_dirs = ['include', 'AF_DIR/include', 'LC_DIR/lib',] -# distutils: sources = ['src_cpp/algorithm.cpp', 'src_cpp/bridge.cpp', 'src_cpp/manager.cpp', 'src_cpp/parameters.cpp', 'src_cpp/pcdi.cpp', 'src_cpp/state.cpp', 'src_cpp/support.cpp', 'src_cpp/util.cpp', 'src_cpp/worker.cpp'] -# distutils: libraries = ['afopencl', 'config++',] -# distutils: library_dirs = ['AF_DIR/lib', 'LC_DIR/lib/.libs',] - -from libcpp.vector cimport vector -from libcpp.string cimport string - -cdef extern from "../include/bridge.hpp": - cdef cppclass Bridge: - Bridge() except + - void StartCalcWithGuess(vector[float], vector[float], vector[float], vector[int], string) - void StartCalc(vector[float], vector[int], string) - void StartCalcMultiple(vector[float], vector[int], string, int) - vector[def_type] GetImageR() - vector[def_type] GetImageI() - vector[def_type] GetErrors() - vector[float] GetSupportV() - vector[def_type] GetCoherenceV() - - -cdef class PyBridge: - cdef Bridge *thisptr - def __cinit__(self): - self.thisptr = new Bridge() - def __dealloc__(self): - del self.thisptr - def start_calc_with_guess(self, data_r, guess_r, guess_i, dims, config): - self.thisptr.StartCalcWithGuess(data_r, guess_r, guess_i, dims, config.encode()) - def start_calc(self, data_r, dims, config): - self.thisptr.StartCalc(data_r, dims, config.encode()) - def start_calc_multiple(self, data_r, dims, config, no_threads): - self.thisptr.StartCalcMultiple(data_r, dims, config, no_threads) - def get_image_r(self): - return self.thisptr.GetImageR() - def get_image_i(self): - return self.thisptr.GetImageI() - def get_errors(self): - return self.thisptr.GetErrors() - def get_support(self): - return self.thisptr.GetSupportV() - def get_coherence(self): - return self.thisptr.GetCoherenceV() - diff --git a/src_py/utilities/CXDVizNX.py b/src_py/utilities/CXDVizNX.py deleted file mode 100644 index 9e5c90d..0000000 --- a/src_py/utilities/CXDVizNX.py +++ /dev/null @@ -1,327 +0,0 @@ -import pylibconfig2 as cfg -import os -import traits.api as tr -from tvtk.api import tvtk -import numpy as np -import math as m -import utils as ut - - -class DispalyParams: - """ - This class encapsulates parameters defining image display. The parameters are read from config file on - construction - """ - - def __init__(self, config): - """ - The constructor gets config file and fills out the class members. - - Parameters - ---------- - conf : str - configuration file name - - Returns - ------- - none - """ - if os.path.isfile(config): - with open(config, 'r') as f: - config_map = cfg.Config(f.read()) - deg2rad = np.pi / 180.0 - try: - self.lamda = config_map.lamda - except AttributeError: - print ('lamda not defined') - try: - self.delta = config_map.delta * deg2rad - except AttributeError: - print ('delta not defined') - try: - self.gamma = config_map.gamma * deg2rad - except AttributeError: - print ('gamma not defined') - try: - self.arm = config_map.arm - except AttributeError: - print ('arm not defined') - try: - self.dth = config_map.dth - except AttributeError: - print ('dth not defined') - try: - pixel = config_map.pixel - self.dpx = pixel[0] / self.arm - self.dpy = pixel[1] / self.arm - except AttributeError: - print ('pixel not defined') - try: - self.save_two_files = config_map.save_two_files - except AttributeError: - print ('save_two_files not defined') - try: - self.crop = config_map.crop - except AttributeError: - self.crop = None - print ('crop not defined') - -class CXDViz(tr.HasTraits): - coords = tr.Array() - arr = tr.Array() - - cropx = tr.Int() - cropy = tr.Int() - cropz = tr.Int() - - def __init__(self): - self.imd = tvtk.ImageData() - self.sg = tvtk.StructuredGrid() - pass - - def set_geometry(self, params, shape): - lam = params.lamda - tth = params.delta - gam = params.gamma - dpx = params.dpx - dpy = params.dpy - dth = params.dth - dx = 1.0 / shape[0] - dy = 1.0 / shape[1] - dz = 1.0 / shape[2] - dQdpx = np.zeros(3) - dQdpy = np.zeros(3) - dQdth = np.zeros(3) - Astar = np.zeros(3) - Bstar = np.zeros(3) - Cstar = np.zeros(3) - - dQdpx[0] = -m.cos(tth) * m.cos(gam) - dQdpx[1] = 0.0 - dQdpx[2] = +m.sin(tth) * m.cos(gam) - - dQdpy[0] = m.sin(tth) * m.sin(gam) - dQdpy[1] = -m.cos(gam) - dQdpy[2] = m.cos(tth) * m.sin(gam) - - dQdth[0] = -m.cos(tth) * m.cos(gam) + 1.0 - dQdth[1] = 0.0 - dQdth[2] = m.sin(tth) * m.cos(gam) - - Astar[0] = 2 * m.pi / lam * dpx * dQdpx[0] - Astar[1] = 2 * m.pi / lam * dpx * dQdpx[1] - Astar[2] = 2 * m.pi / lam * dpx * dQdpx[2] - - Bstar[0] = (2 * m.pi / lam) * dpy * dQdpy[0] - Bstar[1] = (2 * m.pi / lam) * dpy * dQdpy[1] - Bstar[2] = (2 * m.pi / lam) * dpy * dQdpy[2] - - Cstar[0] = (2 * m.pi / lam) * dth * dQdth[0] - Cstar[1] = (2 * m.pi / lam) * dth * dQdth[1] - Cstar[2] = (2 * m.pi / lam) * dth * dQdth[2] - - denom = np.dot(Astar, np.cross(Bstar, Cstar)) - A = 2 * m.pi * np.cross(Bstar, Cstar) / denom - B = 2 * m.pi * np.cross(Cstar, Astar) / denom - C = 2 * m.pi * np.cross(Astar, Bstar) / denom - - self.T = np.zeros(9) - self.T.shape = (3, 3) - space = 'direct' - if space == 'recip': - self.T[:, 0] = Astar - self.T[:, 1] = Bstar - self.T[:, 2] = Cstar - self.dx = 1.0 - self.dy = 1.0 - self.dz = 1.0 - elif space == 'direct': - self.T = np.array((A, B, C)) - self.dx = dx - self.dy = dy - self.dz = dz - else: - pass - - def update_coords(self): - dims = list(self.arr[self.cropobj].shape) - - r = np.mgrid[(dims[0] - 1) * self.dx:-self.dx:-self.dx, \ - 0:dims[1] * self.dy:self.dy, 0:dims[2] * self.dz:self.dz] - - r.shape = 3, dims[0] * dims[1] * dims[2] - r = r.transpose() - - print r.shape - print self.T.shape - - self.coords = np.dot(r, self.T) - - def set_array(self, array, logentry=None): - self.arr = array - if len(self.arr.shape) < 3: - newdims = list(self.arr.shape) - for i in range(3 - len(newdims)): - newdims.append(1) - self.arr.shape = tuple(newdims) - - def set_crop(self, cropx, cropy, cropz): - dims = list(self.arr.shape) - if len(dims) == 2: - dims.append(1) - - if dims[0] > cropx and cropx > 0: - self.cropx = cropx - else: - self.cropx = dims[0] - - if dims[1] > cropy and cropy > 0: - self.cropy = cropy - else: - self.cropy = dims[1] - - if dims[2] > cropz and cropz > 0: - self.cropz = cropz - else: - self.cropz = dims[2] - - start1 = dims[0] / 2 - self.cropx / 2 - end1 = dims[0] / 2 + self.cropx / 2 - if start1 == end1: - end1 = end1 + 1 - start2 = dims[1] / 2 - self.cropy / 2 - end2 = dims[1] / 2 + self.cropy / 2 - if start2 == end2: - end2 = end2 + 1 - start3 = dims[2] / 2 - self.cropz / 2 - end3 = dims[2] / 2 + self.cropz / 2 - if start3 == end3: - end3 = end3 + 1 - - self.cropobj = (slice(start1, end1, None), slice(start2, end2, None), - slice(start3, end3, None)) - - def get_structured_grid(self, **args): - self.update_coords() - dims = list(self.arr[self.cropobj].shape) - self.sg.points = self.coords - if args.has_key("mode"): - if args["mode"] == "Phase": - arr1 = self.arr[self.cropobj].ravel() - arr = (np.arctan2(arr1.imag, arr1.real)) - else: - arr = np.abs(self.arr[self.cropobj].ravel()) - else: - arr = self.arr[self.cropobj].ravel() - if (arr.dtype == np.complex128 or arr.dtype == np.complex64): - self.sg.point_data.scalars = np.abs(arr) - self.sg.point_data.scalars.name = "Amp" - ph = tvtk.DoubleArray() - ph.from_array(np.arctan2(arr.imag, arr.real)) - ph.name = "Phase" - self.sg.point_data.add_array(ph) - else: - self.sg.point_data.scalars = arr - self.sg.dimensions = (dims[2], dims[1], dims[0]) - self.sg.extent = 0, dims[2] - 1, 0, dims[1] - 1, 0, dims[0] - 1 - return self.sg - - def get_image_data(self, **args): - self.set_crop(self.cropx, self.cropy, self.cropz) - dims = list(self.arr[self.cropobj].shape) - if len(dims) == 2: - dims.append(1) - self.imd.dimensions = tuple(dims) - self.imd.extent = 0, dims[2] - 1, 0, dims[1] - 1, 0, dims[0] - 1 - self.imd.point_data.scalars = self.arr[self.cropobj].ravel() - return self.imd - - def write_structured_grid(self, filename, **args): - print 'in WriteStructuredGrid' - sgwriter = tvtk.StructuredGridWriter() - sgwriter.file_type = 'binary' - if filename.endswith(".vtk"): - sgwriter.file_name = filename - else: - sgwriter.file_name = filename + '.vtk' - - sgwriter.set_input_data(self.get_structured_grid()) - print sgwriter.file_name - sgwriter.write() - -def shift(arr, s0, s1, s2): - shifted = np.roll(arr, s0, 0) - shifted = np.roll(shifted, s1, 1) - return np.roll(shifted, s2, 2) - - -def sub_pixel_shift(arr, shift_ind): - buf = np.fft.fftn(arr) - dims = buf.shape - x = np.fft.ifftshift(np.arange(-(dims[0] / 2), dims[0] / 2)) - y = np.fft.ifftshift(np.arange(-(dims[1] / 2), dims[1] / 2)) - z = np.fft.ifftshift(np.arange(-(dims[2] / 2), dims[2] / 2)) - gx, gy, gz = np.meshgrid(x, y, z) - - grid_shift = - gx * shift_ind[0] / dims[0] - gy * shift_ind[1] / dims[1] - gy * shift_ind[2] / dims[2] - g = buf * np.exp(1j * 2 * np.pi * grid_shift) - return np.fft.ifftn(g) - - -def center_of_mass(arr): - tot = np.sum(arr) - dims = arr.shape - xyz = [] - griddims = [] - for d in dims: - griddims.append(slice(0, d)) - grid = np.ogrid[griddims] - for g in grid: - xyz.append(np.sum(arr * g) / tot) - com = np.asarray(xyz) - com = np.ma.round(com).astype(np.int) - return list(com) - - -def remove_ramp(arr): - # pad zeros around arr, to the size of 3 times (ups = 3) of arr size - padded = ut.zero_pad(arr, arr.shape) - data = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(padded))) - com = center_of_mass(np.power(np.absolute(data), 2)) - .5 - sub_pixel_shifted = sub_pixel_shift(data, (com[1], com[0], com[2])) - ramp_removed_padded = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(sub_pixel_shifted))) - ramp_removed = ut.crop_center(ramp_removed_padded, arr.shape) - - return ramp_removed - - -def center(image, support): - dims = image.shape - com = center_of_mass(np.absolute(image) * support) - # place center of mass image*support in the center - image = shift(image, dims[0] / 2 - com[0], dims[1] / 2 - com[1], dims[2] / 2 - com[2]) - support = shift(support, dims[0] / 2 - com[0], dims[1] / 2 - com[1], dims[2] / 2 - com[2]) - - # set com phase to zero, use as a reference - phi0 = m.atan2(image.imag[dims[0]/2, dims[1]/2, dims[2]/2], image.real[dims[0]/2, dims[1]/2, dims[2]/2]) - print 'phi0', phi0 - image = image * np.exp(-1j * phi0) - return image, support - - -def save_CX(conf, image, support, filename): - image, support = center(image, support) - # image = remove_ramp(image) - params = DispalyParams(conf) - viz = CXDViz() - viz.set_array(image) - viz.set_geometry(params, image.shape) - if params.crop is None: - crop = image.shape - else: - crop = params.crop - viz.set_crop(crop[0], crop[1], crop[2]) # save image - viz.write_structured_grid(filename + '_img') - viz.set_array(support) - viz.write_structured_grid(filename + '_support') - diff --git a/src_py/utilities/utils.py b/src_py/utilities/utils.py deleted file mode 100644 index 1a6fc2a..0000000 --- a/src_py/utilities/utils.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# ######################################################################### -# Copyright (c) 2016, UChicago Argonne, LLC. All rights reserved. # -# # -# Copyright 2016. UChicago Argonne, LLC. This software was produced # -# under U.S. Government contract DE-AC02-06CH11357 for Argonne National # -# Laboratory (ANL), which is operated by UChicago Argonne, LLC for the # -# U.S. Department of Energy. The U.S. Government has rights to use, # -# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR # -# UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR # -# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is # -# modified to produce derivative works, such modified software should # -# be clearly marked, so as not to confuse it with the version available # -# from ANL. # -# # -# Additionally, redistribution and use in source and binary forms, with # -# or without modification, are permitted provided that the following # -# conditions are met: # -# # -# * Redistributions of source code must retain the above copyright # -# notice, this list of conditions and the following disclaimer. # -# # -# * Redistributions in binary form must reproduce the above copyright # -# notice, this list of conditions and the following disclaimer in # -# the documentation and/or other materials provided with the # -# distribution. # -# # -# * Neither the name of UChicago Argonne, LLC, Argonne National # -# Laboratory, ANL, the U.S. Government, nor the names of its # -# contributors may be used to endorse or promote products derived # -# from this software without specific prior written permission. # -# # -# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS # -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago # -# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # -# POSSIBILITY OF SUCH DAMAGE. # -# ######################################################################### - -""" -Please make sure the installation :ref:`pre-requisite-reference-label` are met. -This module is a suite of utility mehods. -""" - -import tifffile as tf -import numpy as np - - -def get_array_from_tif(filename): - """ - This method reads tif type file containing experiment data and returns the data as array. - - Parameters - ---------- - filename : str - a filename containing the experiment data - - Returns - ------- - data : array - an array containing the experiment data - """ - - return tf.imread(filename) - - -def get_opencl_dim(dim, step): - """ - This function calculates the dimension supported by opencl library (i.e. is multiplier of 2,3, or 5) and is closest to the - given starting dimension, and spaced by the given step. - If the dimension is not supported the function adds step value and verifies the new dimension. It iterates until it finds - supported value. - - Parameters - ---------- - dim : int - a dimension that needs to be tranformed to one that is supported by the opencl library, if it is not already - - step : int - a delta to increase the dimension - - Returns - ------- - dim : int - a dimension that is supported by the opencl library, and closest to the original dimension by n*step - """ - - def is_correct(x): - sub = x - while sub%2 == 0: - sub = sub/2 - while sub%3 == 0: - sub = sub/3 - while sub%5 == 0: - sub = sub/5 - return sub == 1 - - new_dim = dim - while not is_correct(new_dim): - new_dim += step - return new_dim - - -def binning(array, binsizes): - """ - This function does the binning of the array. The array is binned in each dimension by the corresponding binsizes elements. - If binsizes list is shorter than the array dimensions, the remaining dimensions are not binned. The elements in - a bucket are summed. - - Parameters - ---------- - array : array - the original array to be binned - - binsizes : list - a list defining binning buckets for corresponding dimensions - - Returns - ------- - array : array - the binned array - """ - - data_dims = array.shape - # trim array - for ax in range(len(binsizes)): - cut_slices = range(data_dims[ax] - data_dims[ax] % binsizes[ax], data_dims[ax]) - array = np.delete(array, cut_slices, ax) - - binned_array = array - new_shape = list(array.shape) - - for ax in range(len(binsizes)): - if binsizes[ax] > 1: - new_shape[ax] = binsizes[ax] - new_shape.insert(ax, array.shape[ax] / binsizes[ax]) - binned_array = np.reshape(binned_array, tuple(new_shape)) - binned_array = np.sum(binned_array, axis=ax+1) - new_shape = list(binned_array.shape) - return binned_array - - -def get_centered(array, center_shift): - """ - This function finds a greatest value in the array, and puts it in a center of a new array. The extra elements in the new - array are set to 0. - - Parameters - ---------- - array : array - the original array to be centered - - center_shift : list - a list defining shift of the center - - Returns - ------- - array : array - the centered array - """ - max_coordinates = list(np.unravel_index(np.argmax(array), array.shape)) - max_coordinates = np.add(max_coordinates, center_shift) - shape = array.shape - new_shape = [] - shift = [] - # find new shape that can fit the array with max_coordinate in center - for ax in range(len(shape)): - if max_coordinates[ax] <= int(shape[ax]/2): - new_shape.append(2*(shape[ax] - max_coordinates[ax])) - else: - new_shape.append(2*max_coordinates[ax]) - shift.append(int(new_shape[ax]/2) - max_coordinates[ax]) - - centered = np.zeros(tuple(new_shape), dtype=array.dtype) - - # this supports 3D arrays - centered[shift[0]:shift[0]+shape[0], shift[1]:shift[1]+shape[1], shift[2]:shift[2]+shape[2]] = array - - return centered - -def zero_pad(array, pad): - """ - This function adds to each dimension of the array elements defined by pad. The elements are added to the - beginning of array and end by the same number, so the original array is centered. The dimensions of the new array are - supported by the opencl library. - - - Parameters - ---------- - array : array - the array to pad - - pad : list - list of three pad values, for each dimension - - Returns - ------- - array : array - the padded array - """ - # calculate the opencl dimensions - dims = array.shape - x_dim = get_opencl_dim(dims[0] + 2*pad[0], 2) - x_pad = (x_dim - dims[0])/2 - y_dim = get_opencl_dim(dims[1] + 2*pad[1], 2) - y_pad = (y_dim - dims[1])/2 - z_dim = get_opencl_dim(dims[2] + 2*pad[2], 2) - z_pad = (z_dim - dims[2])/2 - return np.lib.pad(array, ((x_pad, x_pad),(y_pad, y_pad), (z_pad, z_pad)), 'constant', constant_values=((0.0,0.0),(0.0,0.0),(0.0,0.0))).copy() - -def crop_center(arr, new_size): - size = arr.shape - return arr[ (size[0]-new_size[0])/2 : (size[0]-new_size[0])/2 + new_size[0], (size[1]-new_size[1])/2 : (size[1]-new_size[1])/2 + new_size[1], (size[2]-new_size[2])/2 : (size[2]-new_size[2])/2 + new_size[2]] - -def flip(m, axis): - """ - Copied from numpy 1.12.0. - - """ - if not hasattr(m, 'ndim'): - m = np.asarray(m) - indexer = [slice(None)] * m.ndim - try: - indexer[axis] = slice(None, None, -1) - except IndexError: - raise ValueError("axis=%i is invalid for the %i-dimensional input array" - % (axis, m.ndim)) - return m[tuple(indexer)] - - - - - diff --git a/src_py/utilities/utils_post.py b/src_py/utilities/utils_post.py deleted file mode 100644 index 2571555..0000000 --- a/src_py/utilities/utils_post.py +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# ######################################################################### -# Copyright (c) 2016, UChicago Argonne, LLC. All rights reserved. # -# # -# Copyright 2016. UChicago Argonne, LLC. This software was produced # -# under U.S. Government contract DE-AC02-06CH11357 for Argonne National # -# Laboratory (ANL), which is operated by UChicago Argonne, LLC for the # -# U.S. Department of Energy. The U.S. Government has rights to use, # -# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR # -# UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR # -# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is # -# modified to produce derivative works, such modified software should # -# be clearly marked, so as not to confuse it with the version available # -# from ANL. # -# # -# Additionally, redistribution and use in source and binary forms, with # -# or without modification, are permitted provided that the following # -# conditions are met: # -# # -# * Redistributions of source code must retain the above copyright # -# notice, this list of conditions and the following disclaimer. # -# # -# * Redistributions in binary form must reproduce the above copyright # -# notice, this list of conditions and the following disclaimer in # -# the documentation and/or other materials provided with the # -# distribution. # -# # -# * Neither the name of UChicago Argonne, LLC, Argonne National # -# Laboratory, ANL, the U.S. Government, nor the names of its # -# contributors may be used to endorse or promote products derived # -# from this software without specific prior written permission. # -# # -# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS # -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago # -# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # -# POSSIBILITY OF SUCH DAMAGE. # -# ######################################################################### - -""" -This module is a suite of utility mehods. -""" - -import pylibconfig2 as cfg -import os -from tvtk.api import tvtk -import numpy as np -import math as m -from traits.api import * - -class DispalyParams: - """ - This class encapsulates parameters defining image display. The parameters are read from config file on - construction - """ - - def __init__(self, config): - """ - The constructor gets config file and fills out the class members. - Parameters - ---------- - conf : str - configuration file name - Returns - ------- - none - """ - - if os.path.isfile(config): - with open(config, 'r') as f: - config_map = cfg.Config(f.read()) - deg2rad = np.pi / 180.0 - try: - self.lamda = config_map.lamda - except AttributeError: - print ('lamda not defined') - try: - self.delta = config_map.delta * deg2rad - except AttributeError: - print ('delta not defined') - try: - self.gamma = config_map.gamma * deg2rad - except AttributeError: - print ('gamma not defined') - try: - self.arm = config_map.arm - except AttributeError: - print ('arm not defined') - try: - self.dth = config_map.dth * deg2rad - except AttributeError: - print ('dth not defined') - try: - pixel = config_map.pixel - self.dpx = pixel[0] / self.arm - self.dpy = pixel[1] / self.arm - except AttributeError: - print ('pixel not defined') - try: - self.save_two_files = config_map.save_two_files - except AttributeError: - print ('save_two_files not defined') - try: - self.crop = config_map.crop - except AttributeError: - self.crop = None - print ('crop not defined') - - def get_geometry(self): - """ - The class method calculates geometry of display based on the configuration parameters. - It returns calculated geometry as array. - Parameters - ---------- - none - Returns - ------- - numpy.array - an array defining geometry - """ - dQdpx = np.zeros(3) - dQdpy = np.zeros(3) - dQdth = np.zeros(3) - Astar = np.zeros(3) - Bstar = np.zeros(3) - Cstar = np.zeros(3) - - dQdpx[0] = -m.cos(self.delta) * m.cos(self.gamma) - dQdpx[1] = 0.0 - dQdpx[2] = +m.sin(self.delta) * m.cos(self.gamma) - - dQdpy[0] = m.sin(self.delta) * m.sin(self.gamma) - dQdpy[1] = -m.cos(self.gamma) - dQdpy[2] = m.cos(self.delta) * m.sin(self.gamma) - - dQdth[0] = -m.cos(self.delta) * m.cos(self.gamma) + 1.0 - dQdth[1] = 0.0 - dQdth[2] = m.sin(self.delta) * m.cos(self.gamma) - - Astar[0] = 2 * m.pi / self.lamda * self.dpx * dQdpx[0] - Astar[1] = 2 * m.pi / self.lamda * self.dpx * dQdpx[1] - Astar[2] = 2 * m.pi / self.lamda * self.dpx * dQdpx[2] - - Bstar[0] = (2 * m.pi / self.lamda) * self.dpy * dQdpy[0] - Bstar[1] = (2 * m.pi / self.lamda) * self.dpy * dQdpy[1] - Bstar[2] = (2 * m.pi / self.lamda) * self.dpy * dQdpy[2] - - Cstar[0] = (2 * m.pi / self.lamda) * self.dth * dQdth[0] - Cstar[1] = (2 * m.pi / self.lamda) * self.dth * dQdth[1] - Cstar[2] = (2 * m.pi / self.lamda) * self.dth * dQdth[2] - - denom = np.dot(Astar, np.cross(Bstar, Cstar)) - A = 2 * m.pi * np.cross(Bstar, Cstar) / denom - B = 2 * m.pi * np.cross(Cstar, Astar) / denom - C = 2 * m.pi * np.cross(Astar, Bstar) / denom - - return np.array((A, B, C)) - - -def crop_array_center(ar, crop): - """ - This method returns cropped array from the given array ar. The crop is taken from the center of the array - and the size is defined by crop. If any of the crop value is either equal zero or less, or is greater than - the corresponding array dimension, the crop value defaults to that dimension. - Parameters - ---------- - ar : numpy.array - array to be cropped - crop : tuple - the dimension of new cropped array - Returns - ------- - cropped array - """ - - dims = list(ar.shape) - if len(dims) == 2: - dims.append(1) - cropx = crop[0] - cropy = crop[1] - cropz = crop[2] - if dims[0] > cropx and cropx > 0: - x = cropx - else: - x = dims[0] - startx = dims[0] / 2 - x / 2 - endx = dims[0] / 2 + x / 2 - if startx == endx: - endx += 1 - - if dims[1] > cropy and cropy > 0: - y = cropy - else: - y = dims[1] - starty = dims[1] / 2 - y / 2 - endy = dims[1] / 2 + y / 2 - if starty == endy: - endy += 1 - - if dims[2] > cropz and cropz > 0: - z = cropz - else: - z = dims[2] - startz = dims[2] / 2 - z / 2 - endz = dims[2] / 2 + z / 2 - if startz == endz: - endz += 1 - - return ar[startx:endx, starty:endy, startz:endz] - - -def get_coords(dims, geometry): - """ - This method calculates grid based on array dimension, configuration parameters coordinates, and - geometry. - Parameters - ---------- - dims : tuple - displayed array dimension - dx : int - the configured x coordinate - dy : int - the configured y coordinate - dz : int - the configured z coordinate - geometry : array - an array defining geometry, calculated from configuration parameters - Returns - ------- - grid - """ - dx = 1.0 / dims[1] - dy = 1.0 / dims[0] - dz = 1.0 / dims[2] - - r = np.mgrid[(dims[1] - 1) * dx:-dx:-dx, 0:dims[0] * dy:dy, 0:dims[2] * dz:dz] - r.shape = 3, dims[0] * dims[1] * dims[2] - r = r.transpose() - return np.dot(r, geometry) - - -def write_array(sg, filename): - """ - This method writes initialized StructuredGrid object into given file. - Parameters - ---------- - sg : StructuredGrid - an instance of vtkt.StructuredGrid initialized with data - filename : str - name of the file the image will be written to - Returns - ------- - none - """ - - writer = tvtk.StructuredGridWriter() - writer.file_type = 'binary' - writer.file_name = filename - writer.set_input_data(sg) - writer.write() - - -def write_to_vtk(conf, ar, filename): - """ - This function writes a numpy array into vtk format file. - It reads configuration into DispalyParams object. - The given array is cropped to the configured size if the crop parameter is defined in configuration file. - Based on configuration parameters the grid needed for the tvtk StructuredGrid is calculated. - Other StructuredGrid attributes are set. - The initialized StructuredGrid object is then written into vtk type file. If configuration parameter - save_two_files is set to True, one file will be saved for amplitudes with "_Amp.vtk" ending, and one for - phases with "_Phase.vtk" ending. Otherwise, a single file will be saved that contains double array - with amplitudes a nd phases. - Parameters - ---------- - conf : str - configuration file name - - ar : numpy array - a complex array thatwill be saved - filename : str - a prefix to an output filename - Returns - ------- - none - """ - params = DispalyParams(conf) - - if params.crop is None: - dims = ar.shape - arr_cropped = ar.ravel() - else: - dims = params.crop - arr_cropped = crop_array_center(ar, params.crop).ravel() - - amps = np.abs(arr_cropped) - phases = np.arctan2(arr_cropped.imag, arr_cropped.real) - - geometry = params.get_geometry() - coordinates = get_coords(dims, geometry) - - sg = tvtk.StructuredGrid() - sg.points = coordinates - sg.dimensions = (dims[2], dims[1], dims[0]) - sg.extent = 0, dims[2] - 1, 0, dims[1] - 1, 0, dims[0] - 1 - if params.save_two_files: - sg.point_data.scalars = amps - sg.point_data.scalars.name = "Amp" - write_array(sg, filename + "_Amp.vtk") - - sg.point_data.scalars = phases - sg.point_data.scalars.name = "Phase" - write_array(sg, filename + "_Phase.vtk") - else: - sg.point_data.scalars = amps - sg.point_data.scalars.name = "Amp" - ph = tvtk.FloatArray() - ph.from_array(phases) - ph.name = "Phase" - sg.point_data.add_array(ph) - - - write_array(sg, filename + ".vtk") - diff --git a/test.py b/test.py deleted file mode 100755 index a8ddeff..0000000 --- a/test.py +++ /dev/null @@ -1,26 +0,0 @@ -import src_py.controller.reconstruction as rec -import sys -import os -import argparse -from os.path import expanduser - -def main(arg): - - parser = argparse.ArgumentParser() - parser.add_argument("proc", help="the processor the code will run on, can be 'cpu', 'opencl', or 'cuda'.") - parser.add_argument("fname", help="data file filename. It is assumed the file is in tif format.") - parser.add_argument("conf", help="configuration file.") - - args = parser.parse_args() - proc = args.proc - fname = args.fname - conf = args.conf - - rec.reconstruction(proc, fname, conf) - - -if __name__ == "__main__": - main(sys.argv[1:]) - -#python test.py 'opencl' '/home/phoebus/BFROSIK/CDI/S149/Staff14-3_S0149.tif' 'config.test' -