Update comparison_table.md for ndi.cloud.sync progress

comparison_table.md

@@ -135,32 +135,32 @@
 | ndi.cloud.internal.getActiveToken | No | No |
 | ndi.cloud.internal.getCloudDatasetIdForLocalDataset | No | No |
 | ndi.cloud.internal.getTokenExpiration | No | No |
-| ndi.cloud.internal.getUploadedDocumentIds | No | No |
-| ndi.cloud.internal.getUploadedFileIds | No | No |
-| ndi.cloud.internal.getWeboptionsWithAuthHeader | No | No |
-| ndi.cloud.sync.enum.SyncMode | No | No |
-| ndi.cloud.sync.internal.index.createSyncIndexStruct | No | No |
-| ndi.cloud.sync.internal.index.getIndexFilepath | No | No |
-| ndi.cloud.sync.internal.index.readSyncIndex | No | No |
-| ndi.cloud.sync.internal.index.updateSyncIndex | No | No |
-| ndi.cloud.sync.internal.index.writeSyncIndex | No | No |
-| ndi.cloud.sync.internal.Constants | No | No |
-| ndi.cloud.sync.internal.datasetSessionIdFromDocs | No | No |
-| ndi.cloud.sync.internal.deleteLocalDocuments | No | No |
-| ndi.cloud.sync.internal.deleteRemoteDocuments | No | No |
-| ndi.cloud.sync.internal.downloadNdiDocuments | No | No |
+| ndi.cloud.internal.getUploadedDocumentIds | Yes | Yes |
+| ndi.cloud.internal.getUploadedFileIds | Yes | Yes |
+| ndi.cloud.internal.getWeboptionsWithAuthHeader | Yes | Yes |
+| ndi.cloud.sync.enum.SyncMode | Yes | Yes |
+| ndi.cloud.sync.internal.index.createSyncIndexStruct | Yes | Yes |
+| ndi.cloud.sync.internal.index.getIndexFilepath | Yes | Yes |
+| ndi.cloud.sync.internal.index.readSyncIndex | Yes | Yes |
+| ndi.cloud.sync.internal.index.updateSyncIndex | Yes | Yes |
+| ndi.cloud.sync.internal.index.writeSyncIndex | Yes | Yes |
+| ndi.cloud.sync.internal.Constants | Yes | Yes |
+| ndi.cloud.sync.internal.datasetSessionIdFromDocs | Yes | Yes |
+| ndi.cloud.sync.internal.deleteLocalDocuments | Yes | Yes |
+| ndi.cloud.sync.internal.deleteRemoteDocuments | Yes | Yes |
+| ndi.cloud.sync.internal.downloadNdiDocuments | Yes | Yes |
 | ndi.cloud.sync.internal.filesNotYetUploaded | No | No |
-| ndi.cloud.sync.internal.getFileUidsFromDocuments | No | No |
-| ndi.cloud.sync.internal.listLocalDocuments | No | No |
-| ndi.cloud.sync.internal.listRemoteDocumentIds | No | No |
-| ndi.cloud.sync.internal.updateFileInfoForLocalFiles | No | No |
-| ndi.cloud.sync.internal.updateFileInfoForRemoteFiles | No | No |
-| ndi.cloud.sync.internal.uploadFilesForDatasetDocuments | No | No |
-| ndi.cloud.sync.SyncOptions | No | No |
-| ndi.cloud.sync.downloadNew | No | No |
-| ndi.cloud.sync.mirrorFromRemote | No | No |
-| ndi.cloud.sync.mirrorToRemote | No | No |
-| ndi.cloud.sync.twoWaySync | No | No |
+| ndi.cloud.sync.internal.getFileUidsFromDocuments | Yes | Yes |
+| ndi.cloud.sync.internal.listLocalDocuments | Yes | Yes |
+| ndi.cloud.sync.internal.listRemoteDocumentIds | Yes | Yes |
+| ndi.cloud.sync.internal.updateFileInfoForLocalFiles | Yes | Yes |
+| ndi.cloud.sync.internal.updateFileInfoForRemoteFiles | Yes | Yes |
+| ndi.cloud.sync.internal.uploadFilesForDatasetDocuments | Yes | Yes |
+| ndi.cloud.sync.SyncOptions | Yes | Yes |
+| ndi.cloud.sync.downloadNew | Yes | Yes |
+| ndi.cloud.sync.mirrorFromRemote | Yes | Yes |
+| ndi.cloud.sync.mirrorToRemote | Yes | Yes |
+| ndi.cloud.sync.twoWaySync | Yes | Yes |
 | ndi.cloud.sync.uploadNew | No | No |
 | ndi.cloud.sync.validate | No | No |
 | ndi.cloud.ui.dialog.selectCloudDataset | No | No |

src/ndi/cloud/api/files/__init__.py

@@ -0,0 +1,2 @@
+from .get_file_details import get_file_details
+from .get_file_upload_url import get_file_upload_url

src/ndi/cloud/api/files/get_file_upload_url.py

@@ -0,0 +1,16 @@
+from ..implementation.files.get_file_upload_url import GetFileUploadURL as GetFileUploadURLImpl
+
+def get_file_upload_url(dataset_id, file_uid, organization_id=None):
+    """
+    Gets the upload URL for a file in a dataset.
+
+    Args:
+        dataset_id (str): The ID of the dataset.
+        file_uid (str): The UID of the file.
+        organization_id (str, optional): The ID of the organization.
+
+    Returns:
+        tuple: (success, answer, response, url)
+    """
+    api_call = GetFileUploadURLImpl(dataset_id, file_uid, organization_id)
+    return api_call.execute()

src/ndi/cloud/api/implementation/files/get_file_upload_url.py

@@ -0,0 +1,60 @@
+from ...call import Call
+from ... import url
+from ....authenticate import authenticate
+import requests
+import json
+
+class GetFileUploadURL(Call):
+    """
+    Implementation class for getting a file upload URL.
+    """
+
+    def __init__(self, dataset_id, file_uid, organization_id=None):
+        """
+        Creates a new GetFileUploadURL API call object.
+
+        Args:
+            dataset_id (str): The ID of the dataset.
+            file_uid (str): The UID of the file.
+            organization_id (str, optional): The ID of the organization. If not provided, it will be retrieved from the environment.
+        """
+        self.dataset_id = dataset_id
+        self.file_uid = file_uid
+        self.organization_id = organization_id
+        self.endpoint_name = 'get_file_upload_url'
+
+    def execute(self):
+        """
+        Performs the API call.
+        """
+        token = authenticate()
+
+        # Pass organization_id if available, otherwise url.get_url will try env var
+        kwargs = {
+            'dataset_id': self.dataset_id,
+            'file_uid': self.file_uid
+        }
+        if self.organization_id:
+            kwargs['organization_id'] = self.organization_id
+
+        try:
+            api_url = url.get_url(self.endpoint_name, **kwargs)
+        except ValueError as e:
+            # Likely missing organizationId
+            return False, str(e), None, None
+
+        headers = {
+            'Accept': 'application/json',
+            'Authorization': f'Bearer {token}'
+        }
+
+        response = requests.get(api_url, headers=headers)
+
+        if response.status_code == 200:
+            return True, response.json(), response, api_url
+        else:
+            try:
+                answer = response.json()
+            except json.JSONDecodeError:
+                answer = response.text
+            return False, answer, response, api_url

src/ndi/cloud/api/url.py

@@ -30,7 +30,7 @@ def get_url(endpoint_name, **kwargs):
         'uid': kwargs.get('file_uid'),
         'datasetId': kwargs.get('dataset_id'),
         'documentId': kwargs.get('document_id'),
-        'organizationId': kwargs.get('organization_id'),
+        'organizationId': kwargs.get('organization_id') or os.environ.get('NDI_CLOUD_ORGANIZATION_ID'),
         'userId': kwargs.get('user_id'),
         'pageSize': kwargs.get('page_size', 20),
         'page': kwargs.get('page', 1)

src/ndi/cloud/internal/__init__.py

@@ -0,0 +1,4 @@
+from .get_cloud_dataset_id_for_local_dataset import get_cloud_dataset_id_for_local_dataset
+from .get_weboptions_with_auth_header import get_weboptions_with_auth_header
+from .get_uploaded_document_ids import get_uploaded_document_ids
+from .get_uploaded_file_ids import get_uploaded_file_ids

src/ndi/cloud/internal/get_uploaded_document_ids.py

@@ -0,0 +1,34 @@
+def get_uploaded_document_ids(dataset_id, verbose=False):
+    """
+    Returns a dictionary of uploaded document IDs from the cloud.
+
+    Args:
+        dataset_id (str): The cloud dataset ID.
+        verbose (bool): Whether to print progress messages.
+
+    Returns:
+        dict: A dictionary with keys 'ndiId' and 'apiId', containing lists of NDI and API document IDs.
+    """
+    from ..api.documents import list_dataset_documents_all
+
+    if verbose:
+        print(f'Fetching complete remote document list for dataset {dataset_id}...')
+
+    success, all_documents, _, _ = list_dataset_documents_all(dataset_id)
+
+    if not success:
+        error_msg = all_documents.get('message', 'Unknown error') if isinstance(all_documents, dict) else all_documents
+        raise RuntimeError(f"Failed to list remote documents: {error_msg}")
+
+    if not all_documents:
+        if verbose:
+            print('No remote documents found.')
+        return {'ndiId': [], 'apiId': []}
+
+    all_ndi_ids = [doc.get('ndiId') for doc in all_documents]
+    all_api_ids = [doc.get('id') for doc in all_documents]
+
+    if verbose:
+        print(f'Total remote documents processed: {len(all_ndi_ids)}.')
+
+    return {'ndiId': all_ndi_ids, 'apiId': all_api_ids}

src/ndi/cloud/internal/get_uploaded_file_ids.py

@@ -0,0 +1,40 @@
+def get_uploaded_file_ids(dataset_id, verbose=False):
+    """
+    Returns a list of uploaded file IDs from the cloud.
+
+    Args:
+        dataset_id (str): The cloud dataset ID.
+        verbose (bool): Whether to print progress messages.
+
+    Returns:
+        list: A list of unique file UIDs.
+    """
+    from ..api.documents import list_dataset_documents_all
+
+    if verbose:
+        print(f'Listing uploaded files for dataset {dataset_id}...')
+
+    success, all_documents, _, _ = list_dataset_documents_all(dataset_id)
+
+    if not success:
+        error_msg = all_documents.get('message', 'Unknown error') if isinstance(all_documents, dict) else all_documents
+        raise RuntimeError(f"Failed to list remote documents for files: {error_msg}")
+
+    file_uids = set()
+    if all_documents:
+        for document in all_documents:
+            # Assuming document structure from cloud: properties are usually top-level or under 'document_properties' depending on API
+            # Cloud usually returns the document object itself as dict
+            # Check if 'files' property exists
+            file_info = document.get('files', {}).get('file_info', [])
+            for info in file_info:
+                locations = info.get('locations', [])
+                for location in locations:
+                    uid = location.get('uid')
+                    if uid:
+                        file_uids.add(uid)
+
+    if verbose:
+        print(f'Found {len(file_uids)} unique file UIDs.')
+
+    return list(file_uids)

src/ndi/cloud/internal/get_weboptions_with_auth_header.py

@@ -0,0 +1,27 @@
+def get_weboptions_with_auth_header(token=None):
+    """
+    Returns the web options (headers) with the authentication header.
+
+    Args:
+        token (str, optional): The authentication token. If not provided, it will be retrieved using authenticate().
+
+    Returns:
+        dict: A dictionary containing the headers.
+    """
+    if token is None:
+        from ..authenticate import authenticate
+        token = authenticate()
+
+    if not token:
+        # Should we raise error?
+        # Matlab might return empty or error.
+        # For now, return empty headers or raise error?
+        # authenticate() prints error if interactive login fails.
+        pass
+
+    headers = {
+        'Authorization': f'Bearer {token}',
+        'Content-Type': 'application/json'
+    }
+
+    return headers

src/ndi/cloud/sync/__init__.py

@@ -1,3 +1,6 @@
 from .download_new_impl import download_new
 from .sync_options import SyncOptions
 from .enum.sync_mode import SyncMode
+from .mirror_from_remote import mirror_from_remote
+from .mirror_to_remote import mirror_to_remote
+from .two_way_sync import two_way_sync

src/ndi/cloud/sync/internal/__init__.py

@@ -0,0 +1,8 @@
+from .dataset_session_id_from_docs import dataset_session_id_from_docs
+from .delete_local_documents import delete_local_documents
+from .delete_remote_documents import delete_remote_documents
+from .download_ndi_documents import download_ndi_documents
+from .file_utils import get_file_uids_from_documents, update_file_info_for_local_files, update_file_info_for_remote_files
+from .upload_files_for_dataset_documents import upload_files_for_dataset_documents
+from .constants import Constants
+from .document_utils import list_local_documents, list_remote_document_ids

src/ndi/cloud/sync/internal/dataset_session_id_from_docs.py

@@ -0,0 +1,34 @@
+def dataset_session_id_from_docs(documents):
+    """
+    Extracts the session ID from a list of documents.
+
+    Args:
+        documents (list): A list of NDI documents (objects or dicts).
+
+    Returns:
+        str: The session ID, or None if not found.
+    """
+    for doc in documents:
+        # Check if doc is dict or object
+        if isinstance(doc, dict):
+            props = doc
+        elif hasattr(doc, 'document_properties'):
+            props = doc.document_properties
+        else:
+            continue
+
+        # Look for 'ndi_session_id' or similar
+        # Based on naming convention, session ID might be in 'session' -> 'id'
+        if 'session' in props:
+            if isinstance(props['session'], dict) and 'id' in props['session']:
+                return props['session']['id']
+            # Sometimes it might be directly 'session_id' in base?
+
+        if 'base' in props and 'session_id' in props['base']:
+             return props['base']['session_id']
+
+        # Or maybe the document itself IS a session document?
+        if 'ndi_document' in props and props['ndi_document'].get('type') == 'ndi_session':
+             return props.get('id') or props.get('base', {}).get('id')
+
+    return None

src/ndi/cloud/sync/internal/delete_local_documents.py

@@ -0,0 +1,23 @@
+def delete_local_documents(ndi_dataset, document_ids):
+    """
+    Deletes documents from the local dataset.
+
+    Args:
+        ndi_dataset (ndi.dataset.Dataset): The local dataset object.
+        document_ids (list): A list of document IDs to delete.
+    """
+    if not document_ids:
+        return
+
+    # Check if ndi_dataset has database_rm method
+    if hasattr(ndi_dataset, 'database_rm'):
+        for doc_id in document_ids:
+            try:
+                ndi_dataset.database_rm(doc_id)
+            except Exception as e:
+                print(f"Warning: Failed to delete local document {doc_id}: {e}")
+    else:
+        # Fallback or error if method is missing?
+        # Maybe access database directly if available?
+        # But database_rm is the public interface.
+        print("Warning: database_rm method not found on dataset object. Cannot delete local documents.")

src/ndi/cloud/sync/internal/delete_remote_documents.py

@@ -0,0 +1,41 @@
+def delete_remote_documents(cloud_dataset_id, document_ids, verbose=False):
+    """
+    Deletes documents from the remote cloud dataset.
+
+    Args:
+        cloud_dataset_id (str): The cloud dataset ID.
+        document_ids (list): A list of cloud document IDs to delete.
+        verbose (bool): Whether to print progress messages.
+
+    Returns:
+        tuple: (success, deleted_ids)
+    """
+    from ...api.documents import delete_document
+
+    if not document_ids:
+        return True, []
+
+    if verbose:
+        print(f'Deleting {len(document_ids)} documents from remote dataset {cloud_dataset_id}...')
+
+    success_all = True
+    deleted_ids = []
+
+    for doc_id in document_ids:
+        try:
+            success, _, _, _ = delete_document(cloud_dataset_id, doc_id)
+            if success:
+                deleted_ids.append(doc_id)
+            else:
+                success_all = False
+                if verbose:
+                    print(f'Failed to delete remote document: {doc_id}')
+        except Exception as e:
+            success_all = False
+            if verbose:
+                print(f'Error deleting remote document {doc_id}: {e}')
+
+    if verbose:
+        print(f'Deleted {len(deleted_ids)} documents.')
+
+    return success_all, deleted_ids