Fix Python 3 bin scripts and added CLI test coverage

README.md

@@ -56,6 +56,9 @@ Features
 
 * Client, proxy and server classes implemented for some of the protocols.
 
+* Command-line tools for offline work with SAP archive, PSE, Credv2, and SSFS
+  files.
+
 * Example scripts to illustrate the use of the different modules and protocols.
 
 

bin/pysapcar

@@ -125,13 +125,14 @@ class PySAPCAR(object):
         # Opens the input/output file
         self.archive_fd = None
         if options.filename:
+            file_mode = {"r": "rb", "w": "wb", "r+": "r+b"}[self.mode]
             try:
-                self.archive_fd = open(options.filename, self.mode)
+                self.archive_fd = open(options.filename, file_mode)
             except IOError as e:
                 self.logger.error("pysapcar: error opening '%s' (%s)" % (options.filename, e.strerror))
                 return
         else:
-            self.archive_fd = stdin
+            self.archive_fd = stdin.buffer if hasattr(stdin, "buffer") else stdin
 
         # Execute the action
         try:
@@ -152,10 +153,18 @@ class PySAPCAR(object):
             sapcar = SAPCARArchive(self.archive_fd, mode=self.mode)
             self.logger.info("pysapcar: Processing archive '%s' (version %s)", self.archive_fd.name, sapcar.version)
         except Exception as e:
-            self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, e.message)
+            self.logger.error("pysapcar: Error processing archive '%s' (%s)", self.archive_fd.name, str(e))
             return None
         return sapcar
 
+    @staticmethod
+    def _to_archive_filename(filename):
+        return filename.encode() if isinstance(filename, str) else filename
+
+    @staticmethod
+    def _to_path(filename):
+        return filename.decode() if isinstance(filename, bytes) else filename
+
     @staticmethod
     def target_files(filenames, target_filenames=None):
         """Generates the list of files to work on. It calculates
@@ -169,7 +178,8 @@ class PySAPCAR(object):
         """
         files = set(filenames)
         if target_filenames:
-            files = files.intersection(set(target_filenames))
+            target_filenames = set(PySAPCAR._to_archive_filename(filename) for filename in target_filenames)
+            files = files.intersection(target_filenames)
 
         for filename in files:
             yield filename
@@ -237,7 +247,7 @@ class PySAPCAR(object):
         for filename in self.target_files(sapcar.files_names, args):
             flag = CONTINUE
             fil = sapcar.files[filename]
-            filename = path.normpath(filename.replace("\x00", ""))  # Take out null bytes if found
+            filename = path.normpath(self._to_path(filename).replace("\x00", ""))  # Take out null bytes if found
             if options.outdir:
                 # Have to strip directory separator from the beginning of the file name, because path.join disregards
                 # all previous components if any of the following components is an absolute path
@@ -266,7 +276,7 @@ class PySAPCAR(object):
                 try:
                     data = fil.open(enforce_checksum=options.enforce_checksum).read()
                 except (SAPCARInvalidFileException, DecompressError) as e:
-                    self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, e.message)
+                    self.logger.error("pysapcar: Invalid SAP CAR file '%s' (%s)", self.archive_fd.name, str(e))
                     if options.break_on_error:
                         flag = STOP
                     else:

bin/pysapgenpse

@@ -140,11 +140,11 @@ class PySAPGenPSE(object):
         """
 
         try:
-            with open(filename) as f:
+            with open(filename, "rb") as f:
                 obj = cls(f.read())
             self.logger.info("pysapgenpse: Reading {} file '{}'".format(type, filename))
-        except IOError as e:
-            self.logger.error("pysapgenpse: Error reading {} file '{}' ({})".format(type, filename, e.message))
+        except Exception as e:
+            self.logger.error("pysapgenpse: Error reading {} file '{}' ({})".format(type, filename, str(e)))
             return None
 
         return obj
@@ -170,8 +170,11 @@ class PySAPGenPSE(object):
             self.logger.error("pysapgenpse: Unable to read certificates in PSE file {}\n".format(options.filename))
             return
 
-        if pse is None or not pse.enc_cont:
+        if pse is None:
+            return
+        if not pse.enc_cont:
             self.logger.error("pysapgenpse: No encrypted content found in file {}".format(options.filename))
+            return
 
         plain = pse.decrypt(options.pin)
         self.logger.info("Decrypted PSE, {} bytes".format(len(plain)))
@@ -205,7 +208,9 @@ class PySAPGenPSE(object):
             return
 
         # Validate that there are credentials there
-        if not (cred_v2 and cred_v2.creds):
+        if cred_v2 is None:
+            return
+        if not cred_v2.creds:
             self.logger.error("pysapgenpse: No credentials found in file {}\n".format(options.filename))
             return
 
@@ -259,7 +264,7 @@ class PySAPGenPSE(object):
                 pin = plain.decrypt_provider(cred)
             except Exception as e:
                 self.logger.error("pysapgenpse: Unable to decrypt using the provider {} ({}), writing plain blob".format(
-                    plain.option1.val, e.message))
+                    plain.option1.val, str(e)))
                 pin = plain.pin.val
         else:
             pin = plain.pin.val
@@ -283,6 +288,8 @@ class PySAPGenPSE(object):
         :type output: string
         """
         if output_filename:
+            if isinstance(output, str):
+                output = output.encode()
             with open(output_filename, "wb") as output_file:
                 output_file.write(output)
             self.logger.info("pysapgenpse: Output written to file '{}'".format(output_filename))

bin/pysaphdbuserstore

@@ -79,7 +79,7 @@ class PySAPHDBUserStore(object):
         parser.add_argument_group("List command options")
 
         get = parser.add_argument_group("Get command options")
-        get.add_argument("--decrypt", dest="decrypt", action="store_false", help="Whether to try to decrypt the value")
+        get.add_argument("--decrypt", dest="decrypt", action="store_true", help="Whether to try to decrypt the value")
 
         misc = parser.add_argument_group("Misc options")
         misc.add_argument("--deleted", dest="deleted", action="store_true", help="Show deleted records")
@@ -139,59 +139,72 @@ class PySAPHDBUserStore(object):
         """
 
         try:
-            with open(filename) as f:
+            with open(filename, "rb") as f:
                 obj = cls(f.read())
             self.logger.info("pysaphdbuserstore: Reading {} file '{}'".format(type, filename))
-        except IOError as e:
-            self.logger.error("pysaphdbuserstore: Error reading {} file '{}' ({})".format(type, filename, e.message))
+        except Exception as e:
+            self.logger.error("pysaphdbuserstore: Error reading {} file '{}' ({})".format(type, filename, str(e)))
             return None
 
         return obj
 
+    @staticmethod
+    def _to_text(value):
+        """Convert bytes values to text for command-line display."""
+        if isinstance(value, bytes):
+            return value.decode("utf-8", errors="replace")
+        return value
+
     def list(self, options, args):
         """List records in a SSFS Data file
         """
 
         # Parse the data file
-        try:
-            ssfs_data = self.open_file(options.data_filename, SAPSSFSData, "SSFS Data")
-        except Exception:
+        ssfs_data = self.open_file(options.data_filename, SAPSSFSData, "SSFS Data")
+        if ssfs_data is None:
             self.logger.error("pysaphdbuserstore: Unable to read data in file {}\n".format(options.data_filename))
+            return
 
         for ssfs_record in ssfs_data.records:
             if options.deleted or not ssfs_record.deleted:
                 self.logger.info("%s\t%s\t%s",
-                                 ssfs_record.key_name.rstrip(" "),
+                                 self._to_text(ssfs_record.key_name.rstrip(b" ")),
                                  "Plaintext" if ssfs_record.is_stored_as_plaintext else "Encrypted",
                                  ssfs_record.timestamp)
 
     def get(self, options, args):
         """Get a record value in a SSFS Data file
         """
 
-        # Parse the key file
-        try:
+        ssfs_key = None
+        if options.decrypt:
             ssfs_key = self.open_file(options.key_filename, SAPSSFSKey, "SSFS Key")
-        except Exception:
-            self.logger.error("pysaphdbuserstore: Unable to read key in file {}\n".format(options.key_filename))
+            if ssfs_key is None:
+                self.logger.error("pysaphdbuserstore: Unable to read key in file {}\n".format(options.key_filename))
+                return
 
         # Parse the data file
-        try:
-            ssfs_data = self.open_file(options.data_filename, SAPSSFSData, "SSFS Data")
-        except Exception:
+        ssfs_data = self.open_file(options.data_filename, SAPSSFSData, "SSFS Data")
+        if ssfs_data is None:
             self.logger.error("pysaphdbuserstore: Unable to read data in file {}\n".format(options.data_filename))
+            return
+
+        if not args:
+            self.logger.error("pysaphdbuserstore: No record key specified")
+            return
 
         for arg in args:
             if ssfs_data.has_record(arg):
                 for ssfs_record in ssfs_data.get_records(arg):
                     if options.deleted or not ssfs_record.deleted:
                         self.logger.info("Is Deleted   : %s", ssfs_record.deleted)
                         self.logger.info("Is Valid     : %s", ssfs_record.valid)
-                        self.logger.info("Record Key   : %s", ssfs_record.key_name)
+                        self.logger.info("Record Key   : %s", self._to_text(ssfs_record.key_name.rstrip(b" ")))
                         self.logger.info("Time Stamp   : %s", ssfs_record.timestamp)
-                        self.logger.info("Host Name    : %s", ssfs_record.host)
-                        self.logger.info("OS-User      : %s", ssfs_record.user)
-                        self.logger.info("Record Value : %s", ssfs_record.get_plain_data(ssfs_key))
+                        self.logger.info("Host Name    : %s", self._to_text(ssfs_record.host.rstrip(b" ")))
+                        self.logger.info("OS-User      : %s", self._to_text(ssfs_record.user.rstrip(b" ")))
+                        value = ssfs_record.get_plain_data(ssfs_key) if options.decrypt else ssfs_record.data
+                        self.logger.info("Record Value : %s", value)
             else:
                 self.logger.info("Record with key %s not found in data file.", arg)
 

docs/dev/testing.rst

@@ -19,6 +19,12 @@ Integration tests
   They are marked with ``integration`` and are skipped automatically when the
   environment cannot bind sockets.
 
+Bin-script tests
+  Subprocess tests for installed command-line tools under ``bin/``. They are
+  marked with ``bin_script`` and run separately from core library tests so CLI
+  process handling, packaging assumptions, and stdout/stderr behavior do not
+  affect the module-level unit suite.
+
 Core test suites
 ----------------
 
@@ -36,6 +42,9 @@ The current test coverage is centered on these modules:
 * ``tests/saprfc_test.py`` covers RFC packet variants and field handling.
 * ``tests/sapcar_test.py``, ``tests/sapcredv2_test.py``, ``tests/sappse_test.py``,
   and ``tests/sapssfs_test.py`` cover the file-format and crypto-oriented paths.
+* ``tests/pysapcar_script_test.py``, ``tests/pysapgenpse_script_test.py``, and
+  ``tests/pysaphdbuserstore_script_test.py`` cover the command-line tools with
+  ``bin_script``-marked subprocess tests.
 * ``tests/sapdiag_test.py``, ``tests/sapni_test.py``, ``tests/saprouter_test.py``,
   and ``tests/saphdb_test.py`` cover protocol packet handling, with the socket
   heavy cases marked as integration.
@@ -66,6 +75,10 @@ Run the full unit suite with tox::
 
    $ python3 -m tox -e unit
 
+Run the bin-script suite separately::
+
+   $ python3 -m tox -e bin-scripts
+
 Run the integration suite separately::
 
    $ python3 -m tox -e integration
@@ -87,6 +100,7 @@ Good additions usually follow these rules:
 * deterministic inputs and outputs;
 * negative-path coverage for invalid versions, malformed fields, and error
   handling;
+* ``bin_script`` markers for subprocess tests of installed command-line tools;
 * integration markers only for code that must bind sockets or talk to a live
   service.
 

docs/examples/index.rst

@@ -10,6 +10,10 @@ installation, and ``router_password_check`` requires ``fau_timer`` from
 ``mona-timing-lib`` because those packages are not covered by
 ``requirements-examples.txt``.
 
+Installed utilities such as ``pysapcar``, ``pysapgenpse``, and
+``pysaphdbuserstore`` are documented separately in
+:doc:`../tools/index`.
+
 For practical offline workflows, see the file format notebooks for
 :doc:`SAPCAR archive inspection and extraction <../fileformats/SAPCAR>`,
 :doc:`SAP Credv2 parsing <../fileformats/SAPCredv2>`,

docs/index.rst

@@ -50,6 +50,9 @@ Features
 
 * Client, proxy and server classes implemented for some of the protocols.
 
+* Command-line tools for offline work with SAP archive, PSE, Credv2, and SSFS
+  files.
+
 * Example scripts to illustrate the use of the different modules and protocols.
 
 
@@ -62,6 +65,7 @@ User guide
    user/index
    protocols/index
    fileformats/index
+   tools/index
    examples/index
 
 Development guide

docs/tools/index.rst

@@ -0,0 +1,90 @@
+.. Command-line tools frontend
+
+Command-line tools
+==================
+
+pysap installs a small set of command-line tools for offline work with SAP file
+formats. These are persistent utilities, not example scripts. They are installed
+with the package and operate on local files supplied by the user.
+
+The tools are experimental and focused on pysap-supported formats. Use the
+module APIs directly when an application needs stricter error handling or a
+stable integration contract.
+
+
+``pysapcar``
+------------
+
+``pysapcar`` works with SAP ``SAR`` archive files through the
+:mod:`pysap.SAPCAR` module. It can create, append, list, and extract archives.
+
+List archive contents::
+
+    $ pysapcar -t -f archive.sar
+
+Extract an archive into a directory::
+
+    $ pysapcar -x -f archive.sar -o output-dir
+
+Create an archive from local files::
+
+    $ pysapcar -c -f archive.sar file1.txt file2.txt
+
+Append a file to an existing archive::
+
+    $ pysapcar -a -f archive.sar file3.txt
+
+Relevant options include ``-v`` for verbose output,
+``--enforce-checksum`` to stop extraction of files with invalid checksums, and
+``--break-on-error`` to stop processing after an extraction error.
+
+
+``pysapgenpse``
+---------------
+
+``pysapgenpse`` provides offline helpers for SAP Personal Security Environment
+(``PSE``) and SSO Credential (``Credv2``) files through :mod:`pysap.SAPPSE` and
+:mod:`pysap.SAPCredv2`.
+
+List credentials stored in a Credv2 file::
+
+    $ pysapgenpse -c seclogin -l -f cred_v2
+
+Decrypt a credential PIN with a known user name::
+
+    $ pysapgenpse -c seclogin -d -f cred_v2 -u username
+
+Decrypt PSE encrypted content with a known PIN::
+
+    $ pysapgenpse -c get_pse_certs -f local.pse -x pin
+
+Write decrypted output to a file::
+
+    $ pysapgenpse -c get_pse_certs -f local.pse -x pin -o output.der
+
+If ``-f`` is omitted for ``seclogin`` and ``SECUDIR`` is set, the tool looks for
+``cred_v2`` in that directory. The ``-u`` option controls the user name used for
+credential decryption; otherwise ``USER`` or ``USERNAME`` is used when present.
+
+
+``pysaphdbuserstore``
+---------------------
+
+``pysaphdbuserstore`` inspects SAP HANA client secure user store files backed by
+SSFS key/data files through :mod:`pysap.SAPSSFS`.
+
+List records in an SSFS data file::
+
+    $ pysaphdbuserstore -c list -d SSFS_HDB.DAT
+
+Show a record without decrypting encrypted content::
+
+    $ pysaphdbuserstore -c get -d SSFS_HDB.DAT HDB/KEYNAME/DB_USER
+
+Decrypt an encrypted record when the matching key file is available::
+
+    $ pysaphdbuserstore -c get -d SSFS_HDB.DAT -k SSFS_HDB.KEY --decrypt HDB/KEYNAME/DB_PASSWORD
+
+If ``-d`` or ``-k`` is omitted, the tool uses the default HANA client secure
+store paths under ``$HOME/.hdb/<hostname>/``. Use ``--deleted`` to include
+records marked as deleted.