From 54c701053689d257669218c73ba2213119d8e886 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 17 Apr 2020 16:04:39 -0400
Subject: [PATCH 01/79] Add requests.

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 17bb28c..38c5644 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 sal_python_pkg/
 pyobjc==6.2
+requests==2.23.0

From 3b4f557523d8238e528c2b54e24dc1bec1964997 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Thu, 30 Apr 2020 15:54:09 -0400
Subject: [PATCH 02/79] WS fix.

---
 payload/usr/local/sal/bin/sal-submit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 446c1dd..5c34f54 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -237,7 +237,7 @@ def send_inventory(server_url, serial):
     inventory_plist = pathlib.Path(managed_install_dir) / 'ApplicationInventory.plist'
     logging.debug('ApplicationInventory.plist Path: %s', inventory_plist)
 
-    if inventory:= inventory_plist.read_bytes():
+    if inventory := inventory_plist.read_bytes():
         inventory_hash = sal.get_hash(inventory_plist)
         serverhash = None
         serverhash, stderr = sal.curl(hash_url)

From 4ed8bbcf2f6d6fde4c59eca84b96b0adcb2060f7 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:11:30 -0400
Subject: [PATCH 03/79] Add MacSesh package to sal-scripts Python
 requirements.txt.

This is used for making python requests use the keychain.
---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 38c5644..c324346 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
 sal_python_pkg/
 pyobjc==6.2
 requests==2.23.0
+MacSesh==0.2.1

From ad08a513896ba247bf471b21530ff559a04df871 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:12:07 -0400
Subject: [PATCH 04/79] Add SalClient class to replace curl func and related.

---
 sal_python_pkg/sal/utils.py | 88 ++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 50 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index b98e17c..8859924 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -7,6 +7,7 @@
 import datetime
 import hashlib
 import json
+import logging
 import os
 import pathlib
 import plistlib
@@ -15,6 +16,7 @@
 import time
 import urllib.parse
 
+import macsesh
 from Foundation import (kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
                         CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, NSDate, NSArray,
                         NSDictionary, NSData, NSNull)
@@ -132,68 +134,54 @@ def script_is_running(scriptname):
     return False
 
 
-def curl(url, data=None, json_path=None):
-    cmd = ['/usr/bin/curl', '--silent', '--show-error', '--connect-timeout', '2']
-
-    # Use a PEM format certificate file to verify the peer. This is
-    # useful primarily to support self-signed certificates, which are
-    # rejected on 10.13's bundled curl. In cases where you have a cert
-    # signed by an internal or external trusted CA, curl will happily
-    # use the keychain.
-    ca_cert = pref('CACert')
-    if ca_cert:
-        cmd += ['--cacert', ca_cert]
+def get_hash(file_path):
+    """Return sha256 hash of file_path."""
+    text = b''
+    if (path := pathlib.Path(file_path)).is_file():
+        text = path.read_bytes()
+    return hashlib.sha256(text).hexdigest()
 
-    basic_auth = pref('BasicAuth')
-    if basic_auth:
-        key = pref('key')
-        user_pass = f'sal:{key}'
-        cmd += ['--user', user_pass]
 
-    ssl_client_cert = pref('SSLClientCertificate')
-    ssl_client_key = pref('SSLClientKey')
-    if ssl_client_cert:
-        cmd += ['--cert', ssl_client_cert]
-        if ssl_client_key:
-            cmd += ['--key', ssl_client_key]
+class SalClient():
 
-    max_time = '8' if data else '4'
-    cmd += ['--max-time', max_time]
+    basic_timeout = (3.05, 4)
+    post_timeout = (3.05, 8)
 
-    cmd += ['--header', f'SalScript-Version: {sal.version.__version__}']
+    def __init__(self):
+        sesh = macsesh.KeychainSession()
 
-    if data:
-        cmd += ['--data', data]
-    elif json_path:
-        cmd += ['--header', 'Content-Type: application/json']
-        # Use the @ syntax for curl to open the file and do any required
-        # encoding for us.
-        cmd += ['--data', f'@{json_path}']
+        ca_cert = pref('CACert')
+        if ca_cert:
+            sesh.verify = ca_cert
 
-    cmd.append(url)
+        basic_auth = pref('BasicAuth')
+        if basic_auth:
+            key = pref('key', '')
+            sesh.auth = ('sal', key)
 
-    task = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    return task.communicate()
+        # TODO: Handle keychain-based certs.
+        ssl_client_cert = pref('SSLClientCertificate')
+        ssl_client_key = pref('SSLClientKey')
+        if ssl_client_cert:
+            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
 
+        self.sesh = sesh
 
-def get_hash(file_path):
-    """Return sha256 hash of file_path."""
-    text = b''
-    if (path := pathlib.Path(file_path)).is_file():
-        text = path.read_bytes()
-    return hashlib.sha256(text).hexdigest()
+    def get(self, url):
+        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
 
 
-def send_report(url, form_data=None, json_data=None, json_path=None):
-    if form_data:
-        # urlencode allows bytes and str in its dict arg.
-        stdout, stderr = curl(url, data=urllib.parse.urlencode(form_data))
-    elif json_data:
-        raise NotImplementedError
-    elif json_path:
-        stdout, stderr = curl(url, json_path=RESULTS_PATH)
+    def post(self, url, data=None, json=None):
+        kwargs = {'timeout': self.post_timeout}
+        if json:
+            kwargs['json'] = json
+        else:
+            kwargs['data'] = data
+        return self.log_response(self.sesh.post(url, **kwargs))
 
-    return stdout, stderr
+    def log_response(self, response):
+        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
+        return response
 
 
 def add_plugin_results(plugin, data, historical=False):

From 70c29106ff4fd1216a15a5f113b4a7eb53bcb145 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:12:49 -0400
Subject: [PATCH 05/79] Update preflight script to use requests.

---
 .../usr/local/munki/preflight.d/sal-preflight | 37 +++++++++++--------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 3539902..0b227a6 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -6,7 +6,6 @@ Retrieves plugin scripts to run on client.
 
 
 import argparse
-import json
 import os
 import pathlib
 import shutil
@@ -27,7 +26,7 @@ def main():
     if sal.pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
             os.makedirs(EXTERNAL_SCRIPTS_DIR)
-        server_scripts = get_checksum()
+        server_scripts = get_checksums()
         if server_scripts:
             create_dirs(server_scripts)
             download_scripts(server_scripts)
@@ -48,7 +47,7 @@ def get_prefs():
     return required_prefs
 
 
-def get_checksum():
+def get_checksums():
     """Downloads the checksum of existing scripts.
 
     Returns:
@@ -56,17 +55,24 @@ def get_checksum():
         or None if no external scripts are used.
     """
     preflight_url = f"{sal.pref('ServerURL')}/preflight-v2/"
-    stdout, stderr = sal.send_report(preflight_url, form_data={'os_family': 'Darwin'})
+    sal_client = sal.SalClient()
+    error_msg = None
+    try:
+        response = sal_client.post(preflight_url, data={'os_family': 'Darwin'})
+    except Exception as error:
+        error_msg = str(error)
+    if response.status_code != 200:
+        error_msg = f'Request failed with HTTP {response.status_code}'
+
+    if "<h1>Page not found</h1>" not in response.text:
+        munkicommon.display_debug2(response.text)
 
-    if stderr:
-        munkicommon.display_debug2(stderr)
-    stdout_list = stdout.split("\n")
-    if "<h1>Page not found</h1>" not in stdout_list:
-        munkicommon.display_debug2(stdout)
+    if error_msg:
+        munkicommon.display_debug2(error_msg)
 
     try:
-        return json.loads(stdout)
-    except:
+        return response.json()
+    except ValueError:
         munkicommon.display_debug2("Didn't receive valid JSON.")
         return None
 
@@ -94,16 +100,17 @@ def download_and_write_script(server_script):
     script_url = (
         f"{sal.pref('ServerURL')}/preflight-v2/get-script/"
         f"{server_script['plugin']}/{server_script['filename']}/")
-    stdout, stderr = sal.curl(script_url)
-    if stderr:
+    sal_client = sal.SalClient()
+    response = sal_client.get(script_url)
+    if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
-        munkicommon.display_debug2(stderr)
+        munkicommon.display_debug2(response.text)
 
     script = open(
         os.path.join(EXTERNAL_SCRIPTS_DIR, server_script['plugin'], server_script['filename']),
         'w')
     try:
-        data = json.loads(stdout)
+        data = response.json()
     except:
         munkicommon.display_debug2('Did not receive valid JSON when requesting script content.')
         return False

From f125703479f97d82cb445daaa253e88261500cbf Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:16:42 -0400
Subject: [PATCH 06/79] Update machine_checkin model lookup to use requests.

Also, this prevents a bug I discovered that would have a null model
being written to disk, and then no further attempts to check later.
---
 .../sal/checkin_modules/machine_checkin.py      | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index bcc2cb9..a523b71 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -86,7 +86,8 @@ def get_friendly_model(serial):
     """Return friendly model name"""
     if not MODEL_PATH.exists():
         model = cleanup_model(query_apple_support(serial))
-        MODEL_PATH.write_text(model)
+        if model:
+            MODEL_PATH.write_text(model)
     else:
         try:
             model = MODEL_PATH.read_text().strip()
@@ -111,17 +112,13 @@ def get_model_code(serial):
 def query_apple_support(serial):
     model_code = get_model_code(serial)
     tree = ElementTree.ElementTree()
+    session = macsesh.KeychainSession()
+    response = session.get(f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US")
     try:
-        response = subprocess.check_output(
-            ['curl', f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US"],
-            text=True)
-    except subprocess.CalledProcessError:
-        pass
-    try:
-        tree = ElementTree.fromstring(response)
+        tree = ElementTree.fromstring(response.text)
     except ElementTree.ParseError:
-        pass
-    return tree.findtext("configCode")
+        tree = None
+    return tree.findtext("configCode") if tree else None
 
 
 def cleanup_model(model):

From ae2e5028447427e565c5d0fbb81f0922dc8d7e96 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:18:01 -0400
Subject: [PATCH 07/79] Update sal-submit to use requests.

---
 payload/usr/local/sal/bin/sal-submit | 79 ++++++++++++----------------
 1 file changed, 35 insertions(+), 44 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 5c34f54..0a6c5c4 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -53,9 +53,9 @@ def main():
         submission = sal.get_checkin_results()
         logging.debug('Checkin submission:')
         logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
-    _, errors = send_checkin(server_url)
+    response = send_checkin(server_url)
 
-    if not errors:
+    if response.status_code == 200:
         sal.clean_results()
 
     # Speed up manual runs by skipping these potentially slow-running,
@@ -214,16 +214,8 @@ def send_checkin(server_url):
     checkinurl = os.path.join(server_url, 'checkin', '')
     logging.debug(f"Sending report to {checkinurl}")
     logging.debug("Checkin Response:")
-    out, error = sal.send_report(checkinurl, json_path=sal.RESULTS_PATH)
-    log(out, error)
-    return out, error
-
-
-def log(out, error):
-    if out:
-        logging.debug(out.strip())
-    if error:
-        logging.debug(error.strip())
+    sal_client = sal.SalClient()
+    return sal_client.post(checkinurl, json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
 def send_inventory(server_url, serial):
@@ -239,18 +231,17 @@ def send_inventory(server_url, serial):
 
     if inventory := inventory_plist.read_bytes():
         inventory_hash = sal.get_hash(inventory_plist)
+        logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
-        serverhash, stderr = sal.curl(hash_url)
-        if stderr:
-            return
-        if serverhash != inventory_hash:
+        sal_client = sal.SalClient()
+        response = sal_client.get(hash_url)
+        if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
             logging.debug("Inventory report response:")
-            out, error = sal.send_report(inventory_submit_url, form_data=inventory_submission)
-            log(out, error)
+            sal_client.post(inventory_submit_url, data=inventory_submission)
 
 
 def send_catalogs(server_url, machine_group_key):
@@ -275,29 +266,31 @@ def send_catalogs(server_url, machine_group_key):
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    response, stderr = sal.send_report(hash_url, form_data=hash_submission)
-
-    if stderr is not None:
-        try:
-            remote_data = plistlib.loads(response.encode())
-        except plistlib.InvalidFileException:
-            remote_data = []
-
-        for catalog in check_list:
-            if catalog not in remote_data:
-                contents = (pathlib.Path(catalog_dir) / catalog['name']).read_bytes()
-                catalog_submission = {
-                    'key': machine_group_key,
-                    'base64bz2catalog': sal.submission_encode(contents),
-                    'name': catalog['name'],
-                    'sha256hash': catalog['sha256hash']}
+    sal_client = sal.SalClient()
+    try:
+        response = sal_client.post(hash_url, data=hash_submission)
+    except:
+        return
 
-                logging.debug("Submitting Catalog: %s", catalog['name'])
-                try:
-                    out, error = sal.send_report(catalog_submit_url, form_data=catalog_submission)
-                    log(out, error)
-                except OSError:
-                    logging.warning("Error while submitting Catalog: %s", catalog['name'])
+    try:
+        remote_data = plistlib.loads(response.content)
+    except plistlib.InvalidFileException:
+        remote_data = []
+
+    for catalog in check_list:
+        if catalog not in remote_data:
+            contents = (pathlib.Path(catalog_dir) / catalog['name']).read_bytes()
+            catalog_submission = {
+                'key': machine_group_key,
+                'base64bz2catalog': sal.submission_encode(contents),
+                'name': catalog['name'],
+                'sha256hash': catalog['sha256hash']}
+
+            logging.debug("Submitting Catalog: %s", catalog['name'])
+            try:
+                sal_client.post(catalog_submit_url, data=catalog_submission)
+            except OSError:
+                logging.warning("Error while submitting Catalog: %s", catalog['name'])
 
 
 def send_profiles(server_url, serial):
@@ -319,10 +312,8 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-
-    logging.debug("Profiles Response:")
-    out, error = sal.send_report(profile_submit_url, form_data=profile_submission)
-    log(out, error)
+    sal_client = sal.SalClient()
+    sal_client.post(profile_submit_url, data=profile_submission)
 
 
 if __name__ == "__main__":

From 8a8d04e0474e9f06da1350c93fefb898a0401fd6 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:40:43 -0400
Subject: [PATCH 08/79] Skip trying to execute py3 cache, and set up place to
 skip others.

---
 sal_python_pkg/sal/utils.py | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 8859924..0d27d7d 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -251,25 +251,28 @@ def serializer(obj):
     return obj
 
 
-def run_scripts(dir_path, cli_args=None):
+def run_scripts(dir_path, cli_args=None, error=False):
     results = []
-    for script in os.listdir(dir_path):
-        script_stat = os.stat(os.path.join(dir_path, script))
-        if not script_stat.st_mode & stat.S_IWOTH:
-            cmd = [os.path.join(dir_path, script)]
-            if cli_args:
-                cmd.append(cli_args)
-            try:
-                subprocess.check_call(cmd, stdin=None)
-                results.append("'{}' ran successfully")
-            except (OSError, subprocess.CalledProcessError):
-                errormsg = "'{}' had error during execution!".format(script)
-                if not error:
-                    results.append(errormsg)
-                else:
-                    raise RuntimeError(errormsg)
-        else:
+    skip_names = {'__pycache__'}
+    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
+    for script in scripts:
+        if script.stat().st_mode & stat.S_IWOTH:
             results.append(f"'{script}' is not executable or has bad permissions")
+            continue
+
+        cmd = [script]
+        if cli_args:
+            cmd.append(cli_args)
+        try:
+            subprocess.check_call(cmd)
+            results.append(f"'{script}' ran successfully")
+        except (OSError, subprocess.CalledProcessError):
+            errormsg = f"'{script}' had errors during execution!"
+            if not error:
+                results.append(errormsg)
+            else:
+                raise RuntimeError(errormsg)
+
     return results
 
 

From d220a919ff562b40fcb70ab577ce8c8d15e7fc79 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:24:37 -0400
Subject: [PATCH 09/79] Handle json errors in checkin results by using an empty
 dict.

Just start again!
---
 sal_python_pkg/sal/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 0d27d7d..3580b60 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -209,7 +209,10 @@ def add_plugin_results(plugin, data, historical=False):
 def get_checkin_results():
     if os.path.exists(RESULTS_PATH):
         with open(RESULTS_PATH) as results_handle:
-            results = json.load(results_handle)
+            try:
+                results = json.load(results_handle)
+            except json.decoder.JSONDecodeError:
+                results = {}
     else:
         results = {}
 

From 1e2089ebcb795900538671984375e35785ddbb32 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:54:20 -0400
Subject: [PATCH 10/79] Move SalClient into its own module.

---
 sal_python_pkg/sal/__init__.py   |  1 +
 sal_python_pkg/sal/client.py     | 58 ++++++++++++++++++++++++++++++++
 sal_python_pkg/sal/exceptions.py |  2 ++
 sal_python_pkg/sal/utils.py      | 42 -----------------------
 4 files changed, 61 insertions(+), 42 deletions(-)
 create mode 100644 sal_python_pkg/sal/client.py
 create mode 100644 sal_python_pkg/sal/exceptions.py

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index 789f79b..e987577 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,2 +1,3 @@
+from sal.client import SalClient
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
new file mode 100644
index 0000000..8cfad40
--- /dev/null
+++ b/sal_python_pkg/sal/client.py
@@ -0,0 +1,58 @@
+import logging
+
+import macsesh
+
+from sal.utils import pref
+
+
+class SalClient():
+
+    basic_timeout = (3.05, 4)
+    post_timeout = (3.05, 8)
+    base_url = ''
+
+    def __init__(self):
+        sesh = macsesh.KeychainSession()
+        # sesh = macsesh.SecureTransportSession()
+
+        base_url = pref('ServerURL')
+        self.base_url = base_url if not base_url.endswith('/') else base_url[:-1]
+
+        ca_cert = pref('CACert')
+        if ca_cert:
+            sesh.verify = ca_cert
+
+        basic_auth = pref('BasicAuth')
+        if basic_auth:
+            key = pref('key', '')
+            sesh.auth = ('sal', key)
+
+        # TODO: Handle keychain-based certs.
+        ssl_client_cert = pref('SSLClientCertificate')
+        ssl_client_key = pref('SSLClientKey')
+        if ssl_client_cert:
+            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
+
+        self.sesh = sesh
+
+    def get(self, url):
+        url = self.build_url(url)
+        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
+
+    def post(self, url, data=None, json=None):
+        url = self.build_url(url)
+        kwargs = {'timeout': self.post_timeout}
+        if json:
+            kwargs['json'] = json
+        else:
+            kwargs['data'] = data
+        return self.log_response(self.sesh.post(url, **kwargs))
+
+    def log_response(self, response):
+        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
+        return response
+
+    def build_url(self, url):
+        url = url[1:] if url.startswith('/') else url
+        url = url[:-1] if url.endswith('/') else url
+        return '/'.join((self.base_url, url)) + '/'
diff --git a/sal_python_pkg/sal/exceptions.py b/sal_python_pkg/sal/exceptions.py
new file mode 100644
index 0000000..32cd949
--- /dev/null
+++ b/sal_python_pkg/sal/exceptions.py
@@ -0,0 +1,2 @@
+class SalClientError(Exception):
+    pass
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 3580b60..c5534ce 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -142,48 +142,6 @@ def get_hash(file_path):
     return hashlib.sha256(text).hexdigest()
 
 
-class SalClient():
-
-    basic_timeout = (3.05, 4)
-    post_timeout = (3.05, 8)
-
-    def __init__(self):
-        sesh = macsesh.KeychainSession()
-
-        ca_cert = pref('CACert')
-        if ca_cert:
-            sesh.verify = ca_cert
-
-        basic_auth = pref('BasicAuth')
-        if basic_auth:
-            key = pref('key', '')
-            sesh.auth = ('sal', key)
-
-        # TODO: Handle keychain-based certs.
-        ssl_client_cert = pref('SSLClientCertificate')
-        ssl_client_key = pref('SSLClientKey')
-        if ssl_client_cert:
-            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
-
-        self.sesh = sesh
-
-    def get(self, url):
-        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
-
-
-    def post(self, url, data=None, json=None):
-        kwargs = {'timeout': self.post_timeout}
-        if json:
-            kwargs['json'] = json
-        else:
-            kwargs['data'] = data
-        return self.log_response(self.sesh.post(url, **kwargs))
-
-    def log_response(self, response):
-        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
-        return response
-
-
 def add_plugin_results(plugin, data, historical=False):
     """Add data to the shared plugin results plist.
 

From decd2fe4f7e2bc321924f1646f3039e8d699e053 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:54:42 -0400
Subject: [PATCH 11/79] Move URL building to SalClient for sal-submit.

---
 payload/usr/local/sal/bin/sal-submit | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 0a6c5c4..0b1cfe3 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -211,16 +211,13 @@ def sanitize_submission():
 
 
 def send_checkin(server_url):
-    checkinurl = os.path.join(server_url, 'checkin', '')
-    logging.debug(f"Sending report to {checkinurl}")
-    logging.debug("Checkin Response:")
+    logging.debug("Sending report")
     sal_client = sal.SalClient()
-    return sal_client.post(checkinurl, json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
 def send_inventory(server_url, serial):
     logging.info('Processing inventory...')
-    hash_url = os.path.join(server_url, 'inventory/hash', serial, '')
     inventory_submit_url = os.path.join(server_url, 'inventory/submit', '')
 
     managed_install_dir = (
@@ -234,7 +231,7 @@ def send_inventory(server_url, serial):
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
         sal_client = sal.SalClient()
-        response = sal_client.get(hash_url)
+        response = sal_client.get(f'inventory/hash/{serial}/')
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
@@ -246,7 +243,6 @@ def send_inventory(server_url, serial):
 
 def send_catalogs(server_url, machine_group_key):
     logging.info('Processing catalogs...')
-    hash_url = os.path.join(server_url, 'catalog/hash', '')
     catalog_submit_url = os.path.join(server_url, 'catalog/submit', '')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -268,7 +264,7 @@ def send_catalogs(server_url, machine_group_key):
         'catalogs': sal.submission_encode(catalog_check_plist)}
     sal_client = sal.SalClient()
     try:
-        response = sal_client.post(hash_url, data=hash_submission)
+        response = sal_client.post('catalog/hash/', data=hash_submission)
     except:
         return
 
@@ -295,8 +291,6 @@ def send_catalogs(server_url, machine_group_key):
 
 def send_profiles(server_url, serial):
     logging.info('Processing profiles...')
-    profile_submit_url = os.path.join(server_url, 'profiles/submit', '')
-
     temp_dir = tempfile.mkdtemp()
     profile_out = pathlib.Path(temp_dir) / 'profiles.plist'
 
@@ -313,7 +307,7 @@ def send_profiles(server_url, serial):
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
     sal_client = sal.SalClient()
-    sal_client.post(profile_submit_url, data=profile_submission)
+    sal_client.post('profiles/submit/', data=profile_submission)
 
 
 if __name__ == "__main__":

From dce785376de13c0c04002d5d889d310e0ed37519 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:59:06 -0400
Subject: [PATCH 12/79] Use SalClient URL builder for preflight.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 0b227a6..6ad71b5 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -54,11 +54,10 @@ def get_checksums():
         A dict with the script name, plugin name and hash of the script
         or None if no external scripts are used.
     """
-    preflight_url = f"{sal.pref('ServerURL')}/preflight-v2/"
     sal_client = sal.SalClient()
     error_msg = None
     try:
-        response = sal_client.post(preflight_url, data={'os_family': 'Darwin'})
+        response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
     except Exception as error:
         error_msg = str(error)
     if response.status_code != 200:
@@ -97,11 +96,9 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    script_url = (
-        f"{sal.pref('ServerURL')}/preflight-v2/get-script/"
-        f"{server_script['plugin']}/{server_script['filename']}/")
     sal_client = sal.SalClient()
-    response = sal_client.get(script_url)
+    response = sal_client.get(
+        f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
     if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)

From 01d1d34e255224faacb87cc96ced0e4c59e463b2 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:31:50 -0400
Subject: [PATCH 13/79] Fix missed URL update to new SalClient.

---
 payload/usr/local/sal/bin/sal-submit | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 0b1cfe3..a7fc04e 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -218,8 +218,6 @@ def send_checkin(server_url):
 
 def send_inventory(server_url, serial):
     logging.info('Processing inventory...')
-    inventory_submit_url = os.path.join(server_url, 'inventory/submit', '')
-
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
         '/Library/Managed Installs')
@@ -237,8 +235,7 @@ def send_inventory(server_url, serial):
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
-            logging.debug("Inventory report response:")
-            sal_client.post(inventory_submit_url, data=inventory_submission)
+            sal_client.post('inventory/submit/', data=inventory_submission)
 
 
 def send_catalogs(server_url, machine_group_key):

From 7ba494cd454d10e60a4a3f2a57ec6ee0a38fe9fb Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:32:22 -0400
Subject: [PATCH 14/79] Add module-private SalClient "singleton" for session
 reuse purposes.

---
 payload/usr/local/munki/preflight.d/sal-preflight |  4 ++--
 payload/usr/local/sal/bin/sal-submit              |  8 ++++----
 sal_python_pkg/sal/__init__.py                    |  2 +-
 sal_python_pkg/sal/client.py                      | 10 ++++++++++
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 6ad71b5..46b9267 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -54,7 +54,7 @@ def get_checksums():
         A dict with the script name, plugin name and hash of the script
         or None if no external scripts are used.
     """
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     error_msg = None
     try:
         response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
@@ -96,7 +96,7 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     response = sal_client.get(
         f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
     if response.status_code != 200:
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index a7fc04e..1478a8d 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -212,7 +212,7 @@ def sanitize_submission():
 
 def send_checkin(server_url):
     logging.debug("Sending report")
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
@@ -228,7 +228,7 @@ def send_inventory(server_url, serial):
         inventory_hash = sal.get_hash(inventory_plist)
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
-        sal_client = sal.SalClient()
+        sal_client = sal.get_sal_client()
         response = sal_client.get(f'inventory/hash/{serial}/')
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
@@ -259,7 +259,7 @@ def send_catalogs(server_url, machine_group_key):
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
     except:
@@ -303,7 +303,7 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     sal_client.post('profiles/submit/', data=profile_submission)
 
 
diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index e987577..b49c5eb 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,3 @@
-from sal.client import SalClient
+from sal.client import SalClient, get_sal_client
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 8cfad40..db7e9f4 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -5,6 +5,9 @@
 from sal.utils import pref
 
 
+_client_instance = None
+
+
 class SalClient():
 
     basic_timeout = (3.05, 4)
@@ -56,3 +59,10 @@ def build_url(self, url):
         url = url[1:] if url.startswith('/') else url
         url = url[:-1] if url.endswith('/') else url
         return '/'.join((self.base_url, url)) + '/'
+
+
+def get_sal_client():
+    global _client_instance
+    if _client_instance is None:
+        _client_instance = SalClient()
+    return _client_instance

From 45ceb7d945208c7c25f21fe6e869155e60c66edf Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:36:31 -0400
Subject: [PATCH 15/79] Remove unused import.

---
 payload/usr/local/sal/checkin_modules/munki_checkin.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/payload/usr/local/sal/checkin_modules/munki_checkin.py b/payload/usr/local/sal/checkin_modules/munki_checkin.py
index 1a8244d..e7c4b1b 100755
--- a/payload/usr/local/sal/checkin_modules/munki_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/munki_checkin.py
@@ -2,7 +2,6 @@
 
 
 import datetime
-import os
 import pathlib
 import plistlib
 import sys

From 2a89112b6e83ae863731f3f0ca0aa2552bedc7a9 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 17:26:33 -0400
Subject: [PATCH 16/79] Do a better job of handling requests exceptions.

---
 .../usr/local/munki/preflight.d/sal-preflight | 31 ++++++++------
 payload/usr/local/sal/bin/sal-submit          | 42 ++++++++++++++-----
 2 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 46b9267..92a2e56 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -12,6 +12,7 @@ import shutil
 import sys
 import urllib
 
+import requests.exceptions
 import sal
 sys.path.append('/usr/local/munki')
 from munkilib import munkicommon
@@ -58,16 +59,15 @@ def get_checksums():
     error_msg = None
     try:
         response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
-    except Exception as error:
-        error_msg = str(error)
+    except requests.exceptions.RequestException as error:
+        munkicommon.display_debug2(str(error_msg))
+        return
     if response.status_code != 200:
-        error_msg = f'Request failed with HTTP {response.status_code}'
-
-    if "<h1>Page not found</h1>" not in response.text:
+        munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
+        return
+    if response and "<h1>Page not found</h1>" not in response.text:
         munkicommon.display_debug2(response.text)
-
-    if error_msg:
-        munkicommon.display_debug2(error_msg)
+        return
 
     try:
         return response.json()
@@ -96,9 +96,14 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    sal_client = sal.get_sal_client()
-    response = sal_client.get(
-        f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
+    try:
+        response = sal.get_sal_client().get(
+            f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
+    except requests.exceptions.RequestException as error:
+        munkicommon.display_debug2('Error received downloading script:')
+        munkicommon.display_debug2(str(error))
+        return
+
     if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)
@@ -108,9 +113,9 @@ def download_and_write_script(server_script):
         'w')
     try:
         data = response.json()
-    except:
+    except ValueError:
         munkicommon.display_debug2('Did not receive valid JSON when requesting script content.')
-        return False
+        return
 
     script.write(data[0]['content'])
     script.close()
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 1478a8d..bcdb09b 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -17,6 +17,7 @@ import subprocess
 import tempfile
 
 from Foundation import CFPreferencesCopyAppValue
+import requests.exceptions
 
 import sal
 
@@ -55,7 +56,7 @@ def main():
         logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
     response = send_checkin(server_url)
 
-    if response.status_code == 200:
+    if response and response.status_code == 200:
         sal.clean_results()
 
     # Speed up manual runs by skipping these potentially slow-running,
@@ -212,8 +213,14 @@ def sanitize_submission():
 
 def send_checkin(server_url):
     logging.debug("Sending report")
-    sal_client = sal.get_sal_client()
-    return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    try:
+        response = sal.get_sal_client().post(
+            'checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to send report')
+        logging.debug(error)
+        response = None
+    return response
 
 
 def send_inventory(server_url, serial):
@@ -229,13 +236,22 @@ def send_inventory(server_url, serial):
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
         sal_client = sal.get_sal_client()
-        response = sal_client.get(f'inventory/hash/{serial}/')
+        try:
+            response = sal_client.get(f'inventory/hash/{serial}/')
+        except requests.exceptions.RequestException as error:
+            logging.error('Failed to get inventory hash')
+            logging.debug(error)
+            return
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
-            sal_client.post('inventory/submit/', data=inventory_submission)
+            try:
+                sal_client.post('inventory/submit/', data=inventory_submission)
+            except requests.exceptions.RequestException as error:
+                logging.error('Failed to submit inventory')
+                logging.debug(error)
 
 
 def send_catalogs(server_url, machine_group_key):
@@ -262,7 +278,9 @@ def send_catalogs(server_url, machine_group_key):
     sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
-    except:
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to get catalog hashes')
+        logging.debug(error)
         return
 
     try:
@@ -282,8 +300,9 @@ def send_catalogs(server_url, machine_group_key):
             logging.debug("Submitting Catalog: %s", catalog['name'])
             try:
                 sal_client.post(catalog_submit_url, data=catalog_submission)
-            except OSError:
-                logging.warning("Error while submitting Catalog: %s", catalog['name'])
+            except requests.exceptions.RequestException as error:
+                logging.error("Error while submitting Catalog: %s", catalog['name'])
+                logging.debug(error)
 
 
 def send_profiles(server_url, serial):
@@ -303,8 +322,11 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-    sal_client = sal.get_sal_client()
-    sal_client.post('profiles/submit/', data=profile_submission)
+    try:
+        sal.get_sal_client().post('profiles/submit/', data=profile_submission)
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to submit profiles')
+        logging.debug(error)
 
 
 if __name__ == "__main__":

From 02116b1c9b1800613da3f714cf355ecd0f088ef6 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:36:11 -0400
Subject: [PATCH 17/79] Remove now unused arg.

---
 payload/usr/local/sal/bin/sal-submit | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index bcdb09b..f4029ab 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -49,12 +49,11 @@ def main():
     remove_skipped_facts()
     sanitize_submission()
 
-    server_url, _, machine_group_key = sal.get_server_prefs()
+    report = sal.get_checkin_results()
     if logging.getLogger().level <= 10:
-        submission = sal.get_checkin_results()
         logging.debug('Checkin submission:')
-        logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
-    response = send_checkin(server_url)
+        logging.debug(json.dumps(report, indent=4, default=sal.serializer))
+    response = send_checkin(report)
 
     if response and response.status_code == 200:
         sal.clean_results()
@@ -62,9 +61,9 @@ def main():
     # Speed up manual runs by skipping these potentially slow-running,
     # and infrequently changing tasks.
     if run_type != 'manual':
-        send_inventory(server_url, submission['Machine']['extra_data']['serial'])
-        send_catalogs(server_url, machine_group_key)
-        send_profiles(server_url, submission['Machine']['extra_data']['serial'])
+        send_inventory(submission['Machine']['extra_data']['serial'])
+        send_catalogs(machine_group_key)
+        send_profiles(submission['Machine']['extra_data']['serial'])
 
     pathlib.Path('/Users/Shared/.com.salopensource.sal.run').unlink(missing_ok=True)
 
@@ -211,11 +210,10 @@ def sanitize_submission():
     sal.save_results(json.loads(submission_str))
 
 
-def send_checkin(server_url):
+def send_checkin(report):
     logging.debug("Sending report")
     try:
-        response = sal.get_sal_client().post(
-            'checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+        response = sal.get_sal_client().post('checkin/', json=report)
     except requests.exceptions.RequestException as error:
         logging.error('Failed to send report')
         logging.debug(error)
@@ -223,7 +221,7 @@ def send_checkin(server_url):
     return response
 
 
-def send_inventory(server_url, serial):
+def send_inventory(serial):
     logging.info('Processing inventory...')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -254,9 +252,8 @@ def send_inventory(server_url, serial):
                 logging.debug(error)
 
 
-def send_catalogs(server_url, machine_group_key):
+def send_catalogs(machine_group_key):
     logging.info('Processing catalogs...')
-    catalog_submit_url = os.path.join(server_url, 'catalog/submit', '')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
         '/Library/Managed Installs')
@@ -299,13 +296,13 @@ def send_catalogs(server_url, machine_group_key):
 
             logging.debug("Submitting Catalog: %s", catalog['name'])
             try:
-                sal_client.post(catalog_submit_url, data=catalog_submission)
+                sal_client.post('catalog/submit/', data=catalog_submission)
             except requests.exceptions.RequestException as error:
                 logging.error("Error while submitting Catalog: %s", catalog['name'])
                 logging.debug(error)
 
 
-def send_profiles(server_url, serial):
+def send_profiles(serial):
     logging.info('Processing profiles...')
     temp_dir = tempfile.mkdtemp()
     profile_out = pathlib.Path(temp_dir) / 'profiles.plist'

From 0deb41eb66ff9b39ba82af8208fdef838a9a0a1d Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:37:59 -0400
Subject: [PATCH 18/79] Rethink how `[In|Unin]stallResults` get merged into
 `ManagedItem`s.

---
 .../sal/checkin_modules/munki_checkin.py      | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/payload/usr/local/sal/checkin_modules/munki_checkin.py b/payload/usr/local/sal/checkin_modules/munki_checkin.py
index e7c4b1b..7e7ba69 100755
--- a/payload/usr/local/sal/checkin_modules/munki_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/munki_checkin.py
@@ -80,24 +80,30 @@ def main():
         munki_submission['managed_items'][item] = submission_item
 
     # Process InstallResults and RemovalResults into update history
-    for report_key, result_type in (('InstallResults', 'PRESENT'), ('RemovalResults', 'ABSENT')):
+    for report_key in ('InstallResults', 'RemovalResults'):
         for item in munki_report.get(report_key, []):
             # Skip Apple software update items.
             if item.get('applesus'):
                 continue
-            history = {}
-            # history = {'update_type': 'apple' if item.get('applesus') else 'third_party'}
-            history['status'] = 'ERROR' if item.get('status') != 0 else result_type
+            # Construct key; we pop the name off because we don't need
+            # to submit it again when we stuff `item` into `data`.
+            name = f'{item.pop("name")} {item["version"]}'
+            submission_item = munki_submission['managed_items'].get(name, {'name': name})
+            if item.get('status') != 0:
+                # Something went wrong, so change the status.
+                submission_item['status'] = 'ERROR'
+            if 'data' in submission_item:
+                submission_item['data'].update(item)
+            else:
+                submission_item['data'] = item
+            if 'type' not in submission_item['data']:
+                submission_item['data']['type'] = (
+                    'ManagedInstalls' if report_key == 'InstallResults' else 'ManagedUninstalls')
             # This UTC datetime gets converted to a naive datetime by
             # plistlib. Fortunately, we can just tell it that it's UTC.
-            history['date_managed'] = item['time'].replace(
+            submission_item['date_managed'] = item['time'].replace(
                 tzinfo=datetime.timezone.utc).isoformat()
-            history['data'] = {'version': item.get('version', '0')}
-            # Add over top of any pending items we may have already built.
-            if item['name'] in munki_submission['managed_items']:
-                munki_submission['managed_items'][item['name']].update(history)
-            else:
-                munki_submission['managed_items'][item['name']] = history
+            munki_submission['managed_items'][name] = submission_item
 
     sal.set_checkin_results('Munki', munki_submission)
 

From 06239ca6141347d027a9020da4e8a9d81727e4b5 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:54:03 -0400
Subject: [PATCH 19/79] Add property for setting auth.

---
 sal_python_pkg/sal/client.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index db7e9f4..3fbdc96 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -15,7 +15,7 @@ class SalClient():
     base_url = ''
 
     def __init__(self):
-        sesh = macsesh.KeychainSession()
+        self.sesh = macsesh.KeychainSession()
         # sesh = macsesh.SecureTransportSession()
 
         base_url = pref('ServerURL')
@@ -23,20 +23,26 @@ def __init__(self):
 
         ca_cert = pref('CACert')
         if ca_cert:
-            sesh.verify = ca_cert
+            self.sesh.verify = ca_cert
 
         basic_auth = pref('BasicAuth')
         if basic_auth:
             key = pref('key', '')
-            sesh.auth = ('sal', key)
+            self.sesh.auth = ('sal', key)
 
         # TODO: Handle keychain-based certs.
-        ssl_client_cert = pref('SSLClientCertificate')
-        ssl_client_key = pref('SSLClientKey')
-        if ssl_client_cert:
-            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
-
-        self.sesh = sesh
+        cert = pref('SSLClientCertificate')
+        key = pref('SSLClientKey')
+        if cert:
+            self.sesh.cert = (cert, key) if key else cert
+
+    @property
+    def auth(self):
+        return self.sesh.auth
+
+    @auth.setter
+    def auth(self, creds):
+        self.sesh.auth = creds
 
     def get(self, url):
         url = self.build_url(url)

From 7192a8220de9274bc6496252e50786a62458b28e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:54:37 -0400
Subject: [PATCH 20/79] Add commandline args to allow you to override ServerURL
 and key.

This is used primarily for debugging to a local Sal instance, or
submitting to a staging/test server.
---
 payload/usr/local/sal/bin/sal-submit | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index f4029ab..cccd31e 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -26,8 +26,9 @@ CHECKIN_MODULES_DIR = '/usr/local/sal/checkin_modules'
 
 
 def main():
+    args = get_args()
     logging.basicConfig(
-        level=get_log_level(), format='%(asctime)s %(levelname)s %(message)s')
+        level=get_log_level(args), format='%(asctime)s %(levelname)s %(message)s')
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
 
     exit_if_not_root()
@@ -50,6 +51,15 @@ def main():
     sanitize_submission()
 
     report = sal.get_checkin_results()
+    if args.url:
+        sal.get_sal_client().base_url = args.url
+        logging.debug('Server URL overridden with %s', args.url)
+
+    if args.key:
+        sesh = sal.get_sal_client().auth = ('sal', args.key)
+        # Override the key in the report, since it's used for querying.
+        report['Sal']['extra_data']['key'] = args.key
+        logging.debug('Machine group key overridden with %s', args.key)
     if logging.getLogger().level <= 10:
         logging.debug('Checkin submission:')
         logging.debug(json.dumps(report, indent=4, default=sal.serializer))
@@ -62,7 +72,7 @@ def main():
     # and infrequently changing tasks.
     if run_type != 'manual':
         send_inventory(submission['Machine']['extra_data']['serial'])
-        send_catalogs(machine_group_key)
+        send_catalogs()
         send_profiles(submission['Machine']['extra_data']['serial'])
 
     pathlib.Path('/Users/Shared/.com.salopensource.sal.run').unlink(missing_ok=True)
@@ -70,10 +80,9 @@ def main():
     logging.info('Checkin complete.')
 
 
-def get_log_level():
+def get_log_level(args):
     """Set the verbosity based on options."""
     loglevel = logging.CRITICAL
-    args = get_args()
     if args.debug:
         loglevel = logging.DEBUG
     elif args.verbose:
@@ -88,6 +97,8 @@ def get_args():
         "-d", "--debug", default=False, action="store_true", help="Enable full debug output.")
     parser.add_argument(
         "-v", "--verbose", default=False, action="store_true", help="Enable verbose output.")
+    parser.add_argument("-u", "--url", default=None, help="Override the server URL for testing.")
+    parser.add_argument("-k", "--key", default=None, help="Override the machine group key.")
     return parser.parse_args()
 
 
@@ -252,7 +263,7 @@ def send_inventory(serial):
                 logging.debug(error)
 
 
-def send_catalogs(machine_group_key):
+def send_catalogs():
     logging.info('Processing catalogs...')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -269,10 +280,11 @@ def send_catalogs(machine_group_key):
 
         catalog_check_plist = plistlib.dumps(check_list)
 
+    sal_client = sal.get_sal_client()
+    machine_group_key = sal_client.auth[1]
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
     except requests.exceptions.RequestException as error:

From 4569d028a71b74f3b2663c8169f58fe69e989426 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 12:41:32 -0400
Subject: [PATCH 21/79] Add prefs debug output.

---
 payload/usr/local/sal/bin/sal-submit | 13 +++++++++++--
 sal_python_pkg/sal/utils.py          | 18 +++++++++++++++---
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index cccd31e..7598fab 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -27,9 +27,18 @@ CHECKIN_MODULES_DIR = '/usr/local/sal/checkin_modules'
 
 def main():
     args = get_args()
-    logging.basicConfig(
-        level=get_log_level(args), format='%(asctime)s %(levelname)s %(message)s')
+    log_level = get_log_level(args)
+    logging.basicConfig(level=log_level, format='%(asctime)s %(levelname)s %(message)s')
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
+    if log_level == logging.DEBUG:
+        logging.debug("Sal client prefs:")
+        prefs = sal.prefs()
+        if args.url:
+            prefs['ServerURL'] = {'value': args.url, 'forced': 'commandline'}
+        if args.key:
+            prefs['key'] = {'value': args.key, 'forced': 'commandline'}
+        for k, v in prefs.items():
+            logging.debug(f'\t{k}: {v["value"]} ({"profile" if v["forced"] else "prefs"})')
 
     exit_if_not_root()
     if sal.wait_for_script('sal-submit'):
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index c5534ce..ddc97f7 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -17,9 +17,10 @@
 import urllib.parse
 
 import macsesh
-from Foundation import (kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
-                        CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, NSDate, NSArray,
-                        NSDictionary, NSData, NSNull)
+from Foundation import (
+    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
+    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
+    NSArray, NSDictionary, NSData, NSNull)
 
 import sal.version
 
@@ -85,6 +86,17 @@ def pref(pref_name, default=None):
     return unobjctify(pref_value)
 
 
+def prefs():
+    prefs = (
+        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
+        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
+    return {k: {'value': pref(k), 'forced': forced(k)} for k in prefs}
+
+
+def forced(pref):
+    return CFPreferencesAppValueIsForced(pref, BUNDLE_ID)
+
+
 def wait_for_script(scriptname, repeat=3, pause=1):
     """Tries a few times to wait for a script to finish."""
     count = 0

From 88953bf4497f6055976adf7c162c13273b4cb5a7 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 17:16:47 -0400
Subject: [PATCH 22/79] Do some processing client side so we don't send so much
 profile data.

---
 payload/usr/local/sal/bin/sal-submit | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 7598fab..86cad29 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -336,10 +336,15 @@ def send_profiles(serial):
         logging.warning("Couldn't output profiles.")
         return
 
-    profiles = sal.submission_encode(profile_out.read_bytes())
+    profiles = plistlib.loads(profile_out.read_bytes())
     profile_out.unlink()
-
-    profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
+    # Drop all of the payload info we're not going to actual store.
+    for profile in profiles['_computerlevel']:
+        cleansed_payloads = [_payload_cleanse(p) for p in profile.get('ProfileItems', [])]
+        profile['ProfileItems'] = cleansed_payloads
+    logging.debug(profiles)
+    profile_submission = {
+        'serial': serial, 'base64bz2profiles': sal.submission_encode(plistlib.dumps(profiles))}
     try:
         sal.get_sal_client().post('profiles/submit/', data=profile_submission)
     except requests.exceptions.RequestException as error:
@@ -347,5 +352,10 @@ def send_profiles(serial):
         logging.debug(error)
 
 
+def _payload_cleanse(payload):
+    stored = ('PayloadIdentifier', 'PayloadUUID', 'PayloadType')
+    return {k: payload[k] for k in stored}
+
+
 if __name__ == "__main__":
     main()

From 673a0ec621eebdfbc9ad7d1651be848bf48f9d93 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 16:32:04 -0400
Subject: [PATCH 23/79] Fix bug with downloading plugin scripts.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 92a2e56..ecb0ee1 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -65,7 +65,7 @@ def get_checksums():
     if response.status_code != 200:
         munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
         return
-    if response and "<h1>Page not found</h1>" not in response.text:
+    if response and "<h1>Page not found</h1>" in response.text:
         munkicommon.display_debug2(response.text)
         return
 

From 1a657c492d2553e2b915456239e2696a91048a66 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 18:09:10 -0400
Subject: [PATCH 24/79] Fix executable script check in utils.

---
 sal_python_pkg/sal/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index ddc97f7..6ac477e 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -229,7 +229,7 @@ def run_scripts(dir_path, cli_args=None, error=False):
     skip_names = {'__pycache__'}
     scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
     for script in scripts:
-        if script.stat().st_mode & stat.S_IWOTH:
+        if not os.access(script, os.X_OK):
             results.append(f"'{script}' is not executable or has bad permissions")
             continue
 

From 46e078560ea930176798f0af9ec3b86f690a5cff Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 18:13:05 -0400
Subject: [PATCH 25/79] Remove unused imports.

---
 sal_python_pkg/sal/utils.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 6ac477e..269dbc7 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -7,23 +7,17 @@
 import datetime
 import hashlib
 import json
-import logging
 import os
 import pathlib
 import plistlib
-import stat
 import subprocess
 import time
-import urllib.parse
 
-import macsesh
 from Foundation import (
     kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
     CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
     NSArray, NSDictionary, NSData, NSNull)
 
-import sal.version
-
 
 BUNDLE_ID = 'com.github.salopensource.sal'
 RESULTS_PATH = '/usr/local/sal/checkin_results.json'

From 025ff8456a0d277b5920eac373762c88b3a7f1aa Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:07:16 -0400
Subject: [PATCH 26/79] Move preference setting code out of client and bump
 macsesh version.

---
 requirements.txt               |  2 +
 sal_python_pkg/sal/__init__.py |  2 +-
 sal_python_pkg/sal/client.py   | 91 +++++++++++++++++++++++-----------
 sal_python_pkg/setup.py        |  4 ++
 4 files changed, 69 insertions(+), 30 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index c324346..624a53e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,6 @@
 sal_python_pkg/
+# These come along with the sal python pkg, but it doesn't hurt to
+# document that we need them here.
 pyobjc==6.2
 requests==2.23.0
 MacSesh==0.2.1
diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index b49c5eb..d06b704 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,3 @@
-from sal.client import SalClient, get_sal_client
+from sal.client import MacKeychainClient, SalClient, get_sal_client
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 3fbdc96..78e868f 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -1,6 +1,11 @@
 import logging
+import os
 
-import macsesh
+try:
+    from macsesh import Session as MacSeshSession
+except ImportError:
+    MacSeshSession = None
+import requests
 
 from sal.utils import pref
 
@@ -8,45 +13,67 @@
 _client_instance = None
 
 
-class SalClient():
+class SalClient:
 
+    session_class = requests.Session
+    _base_url = ''
+    _auth = None
+    _cert = None
+    _verify = None
     basic_timeout = (3.05, 4)
     post_timeout = (3.05, 8)
-    base_url = ''
 
     def __init__(self):
-        self.sesh = macsesh.KeychainSession()
-        # sesh = macsesh.SecureTransportSession()
-
-        base_url = pref('ServerURL')
-        self.base_url = base_url if not base_url.endswith('/') else base_url[:-1]
-
-        ca_cert = pref('CACert')
-        if ca_cert:
-            self.sesh.verify = ca_cert
-
-        basic_auth = pref('BasicAuth')
-        if basic_auth:
-            key = pref('key', '')
-            self.sesh.auth = ('sal', key)
+        self.create_session()
+
+    def create_session(self):
+        self.session = self.session_class()
+        if self.auth:
+            self.session.auth = self._auth
+        if self.cert:
+            self.session.cert = self._cert
+        if self.verify:
+            self.session.verify = self._verify
+
+            # self.session.cert = (self._cert, self._key) if self._key else self._cert
+    @property
+    def base_url(self):
+        return self._base_url
 
-        # TODO: Handle keychain-based certs.
-        cert = pref('SSLClientCertificate')
-        key = pref('SSLClientKey')
-        if cert:
-            self.sesh.cert = (cert, key) if key else cert
+    @base_url.setter
+    def base_url(self, base_url):
+        self._base_url = base_url if not base_url.endswith('/') else base_url[:-1]
 
     @property
     def auth(self):
-        return self.sesh.auth
+        return self._auth
 
     @auth.setter
     def auth(self, creds):
-        self.sesh.auth = creds
+        self._auth = creds
+        self.create_session()
+
+    @property
+    def cert(self):
+        return self._cert
+
+    @cert.setter
+    def cert(self, cert, key=None):
+        self._cert = (cert, key) if key else cert
+        self.create_session()
+
+    @property
+    def verify(self):
+        return self._verify
+
+    @verify.setter
+    def verify(self, path):
+        self._verify = path
+        self.create_session()
 
     def get(self, url):
         url = self.build_url(url)
-        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
+        return self.log_response(self.session.get(url, timeout=self.basic_timeout))
 
     def post(self, url, data=None, json=None):
         url = self.build_url(url)
@@ -55,7 +82,7 @@ def post(self, url, data=None, json=None):
             kwargs['json'] = json
         else:
             kwargs['data'] = data
-        return self.log_response(self.sesh.post(url, **kwargs))
+        return self.log_response(self.session.post(url, **kwargs))
 
     def log_response(self, response):
         logging.debug(f'Response HTTP {response.status_code}: {response.text}')
@@ -67,8 +94,14 @@ def build_url(self, url):
         return '/'.join((self.base_url, url)) + '/'
 
 
-def get_sal_client():
+class MacKeychainClient(SalClient):
+
+    session_class = MacSeshSession
+
+
+def get_sal_client(with_client_class=None):
     global _client_instance
-    if _client_instance is None:
-        _client_instance = SalClient()
+    if _client_instance is None or (
+            with_client_class is not None and not isinstance(_client_instance, with_client_class)):
+        _client_instance =  with_client_class() if with_client_class is not None else SalClient()
     return _client_instance
diff --git a/sal_python_pkg/setup.py b/sal_python_pkg/setup.py
index 797fa57..3af1887 100644
--- a/sal_python_pkg/setup.py
+++ b/sal_python_pkg/setup.py
@@ -10,4 +10,8 @@
     name='sal',
     version=namespace['__version__'],
     description='Sal client utilities',
+    install_requires=[
+        'pyobjc == 6.2 ; platform_system=="Darwin"',
+        'macsesh == 0.3.0 ; platform_system=="Darwin"',
+        'requests >= 2.23.0'],
     packages=['sal'])
\ No newline at end of file

From 068d3c209784657bfa9e9c3f738e3588ea8e957f Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:09:32 -0400
Subject: [PATCH 27/79] Use new session name.

---
 payload/usr/local/sal/checkin_modules/machine_checkin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index a523b71..3092fba 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -112,7 +112,7 @@ def get_model_code(serial):
 def query_apple_support(serial):
     model_code = get_model_code(serial)
     tree = ElementTree.ElementTree()
-    session = macsesh.KeychainSession()
+    session = macsesh.Session()
     response = session.get(f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US")
     try:
         tree = ElementTree.fromstring(response.text)

From 18ce3cb1d109f1bef8124b2a2ba50fba6ffa2d9c Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:30:52 -0400
Subject: [PATCH 28/79] Bump python version.

---
 build_python_framework.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_python_framework.sh b/build_python_framework.sh
index 786aa28..835bb04 100755
--- a/build_python_framework.sh
+++ b/build_python_framework.sh
@@ -1,7 +1,7 @@
 #!/bin/zsh
 # Build script for Python 3 framework for Sal scripts
 TOOLSDIR=$(dirname "$0")
-PYTHON_VERSION=3.8.2
+PYTHON_VERSION=3.8.3
 
 # build the framework
 /tmp/relocatable-python-git/make_relocatable_python_framework.py \

From 6326022bfdea174024e986b8893dd315f2570cf6 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:08 -0400
Subject: [PATCH 29/79] Add pref-based client setup code into new mac_utils
 module.

---
 sal_python_pkg/sal/__init__.py  |  1 +
 sal_python_pkg/sal/mac_utils.py | 44 +++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 sal_python_pkg/sal/mac_utils.py

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index d06b704..7591db4 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,4 @@
 from sal.client import MacKeychainClient, SalClient, get_sal_client
+from sal.mac_utils import *
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
new file mode 100644
index 0000000..5734c3f
--- /dev/null
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -0,0 +1,44 @@
+import logging
+import os
+
+from sal.client import get_sal_client, MacKeychainClient
+from sal.utils import pref
+
+
+def setup_sal_client():
+    ca_cert = pref('CACert', '')
+    cert = pref('SSLClientCertificate', '')
+    key = pref('SSLClientKey', '')
+    exists = map(os.path.exists, (ca_cert, cert, key))
+    if any(exists):
+        if not all(exists):
+            logging.warning(
+                'Argument warning! If using the `CACert`, `SSLClientCertificate`, or '
+                '`SSLClientKey` prefs, they must all be either paths to cert files or the '
+                'common name of the certs to find in the keychain.')
+
+        # If any of the above have been passed as a path, we have to
+        # use a vanilla Session.
+        logging.debug('Using SalClient')
+        client = get_sal_client()
+    else:
+        # Assume that any passed certs are by CN since they don't
+        # exist as files anywhere.
+        # If we're going to use the keychain, we need to use a
+        # macsesh
+        logging.debug('Using MacKeychainClient')
+        client = get_sal_client(MacKeychainClient)
+
+    if ca_cert:
+        client.verify = ca_cert
+    if cert:
+        client.cert = (cert, key) if key else cert
+
+    basic_auth = pref('BasicAuth')
+    if basic_auth:
+        key = pref('key', '')
+        client.auth = ('sal', key)
+
+    client.base_url = pref('ServerURL')
+
+

From 03466a710ffb5b91ef8dbd71af218a060af82dd9 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:33 -0400
Subject: [PATCH 30/79] Use newer macsesh.

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 624a53e..5d03e47 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,4 @@ sal_python_pkg/
 # document that we need them here.
 pyobjc==6.2
 requests==2.23.0
-MacSesh==0.2.1
+MacSesh==0.3.0

From c8006ee9e1ed94f6c6c5cf8f127ab6ee1bdc9282 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:38 -0400
Subject: [PATCH 31/79] Use client setup util.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 1 +
 payload/usr/local/sal/bin/sal-submit              | 1 +
 2 files changed, 2 insertions(+)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index ecb0ee1..03a40ae 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -23,6 +23,7 @@ EXTERNAL_SCRIPTS_DIR = '/usr/local/sal/external_scripts'
 
 def main():
     set_verbosity()
+    sal.setup_sal_client()
 
     if sal.pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 86cad29..871ab6d 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -60,6 +60,7 @@ def main():
     sanitize_submission()
 
     report = sal.get_checkin_results()
+    sal.setup_sal_client()
     if args.url:
         sal.get_sal_client().base_url = args.url
         logging.debug('Server URL overridden with %s', args.url)

From 401a347242c73024af90207f001029fa87a497d2 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:45:00 -0400
Subject: [PATCH 32/79] Create generic pref getting mac util.

---
 sal_python_pkg/sal/__init__.py  | 6 +++++-
 sal_python_pkg/sal/mac_utils.py | 5 +++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index 7591db4..2e584bd 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,4 +1,8 @@
 from sal.client import MacKeychainClient, SalClient, get_sal_client
-from sal.mac_utils import *
+try:
+    from sal.mac_utils import *
+except ImportError:
+    # Allow non-macOS to import safely.
+    pass
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index 5734c3f..47ea805 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,6 +1,8 @@
 import logging
 import os
 
+from Foundation import CFPreferencesCopyAppValue
+
 from sal.client import get_sal_client, MacKeychainClient
 from sal.utils import pref
 
@@ -42,3 +44,6 @@ def setup_sal_client():
     client.base_url = pref('ServerURL')
 
 
+def mac_pref(domain, key, default=None):
+    val = CFPreferencesCopyAppValue(key, domain)
+    return val if val is not None else default

From 9d7cc6aebb8183a9e1d5afdd21aed2d148219099 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:46:40 -0400
Subject: [PATCH 33/79] Use mac_pref util in sal-submit.

---
 payload/usr/local/sal/bin/sal-submit | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 871ab6d..c772642 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -16,7 +16,6 @@ import stat
 import subprocess
 import tempfile
 
-from Foundation import CFPreferencesCopyAppValue
 import requests.exceptions
 
 import sal
@@ -244,9 +243,8 @@ def send_checkin(report):
 
 def send_inventory(serial):
     logging.info('Processing inventory...')
-    managed_install_dir = (
-        CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
-        '/Library/Managed Installs')
+    managed_install_dir = sal.mac_pref(
+        'ManagedInstalls', 'ManagedInstallDir', '/Library/Managed Installs')
     inventory_plist = pathlib.Path(managed_install_dir) / 'ApplicationInventory.plist'
     logging.debug('ApplicationInventory.plist Path: %s', inventory_plist)
 
@@ -275,9 +273,8 @@ def send_inventory(serial):
 
 def send_catalogs():
     logging.info('Processing catalogs...')
-    managed_install_dir = (
-        CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
-        '/Library/Managed Installs')
+    managed_install_dir = sal.mac_pref(
+        'ManagedInstalls', 'ManagedInstallDir', '/Library/Managed Installs')
     catalog_dir = pathlib.Path(managed_install_dir) / 'catalogs'
 
     check_list = []

From aec86ba1197a75f97a2d4feea3c40866947cb6b1 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 20:23:01 -0400
Subject: [PATCH 34/79] Move Mac-specific sal pkg code to a dedicated mac_utils
 module.

---
 .../local/munki/postflight.d/sal-postflight   |   8 +-
 .../usr/local/munki/preflight.d/sal-preflight |   6 +-
 payload/usr/local/sal/bin/sal-submit          |   6 +-
 .../sal/checkin_modules/machine_checkin.py    |   2 +-
 .../local/sal/checkin_modules/sal_checkin.py  |   3 +-
 sal_python_pkg/sal/client.py                  |   2 -
 sal_python_pkg/sal/mac_utils.py               | 139 ++++++++++++++++-
 sal_python_pkg/sal/utils.py                   | 143 ------------------
 8 files changed, 143 insertions(+), 166 deletions(-)

diff --git a/payload/usr/local/munki/postflight.d/sal-postflight b/payload/usr/local/munki/postflight.d/sal-postflight
index e41baa8..a5043a3 100644
--- a/payload/usr/local/munki/postflight.d/sal-postflight
+++ b/payload/usr/local/munki/postflight.d/sal-postflight
@@ -57,21 +57,21 @@ def check_server_connection():
 
 def check_server_online():
     # is the offline report pref true?
-    if not sal.pref('SendOfflineReport'):
+    if not sal.sal_pref('SendOfflineReport'):
         return
     # read report
     report = munki_checkin.get_managed_install_report()
     # check for errors and warnings
     if not check_for_errors(report):
-        sal.set_pref('LastRunWasOffline', False)
+        sal.set_sal_pref('LastRunWasOffline', False)
         return
     # if they're there check is server is really offline
     if check_server_connection():
-        sal.set_pref('LastRunWasOffline', True)
+        sal.set_sal_pref('LastRunWasOffline', True)
         return
 
     # If we get here, it's online
-    sal.set_pref('LastRunWasOffline', False)
+    sal.set_sal_pref('LastRunWasOffline', False)
 
 def write_touch_file():
     if os.path.exists(TOUCH_FILE_PATH):
diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 03a40ae..6bde9f7 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -25,7 +25,7 @@ def main():
     set_verbosity()
     sal.setup_sal_client()
 
-    if sal.pref('SyncScripts') == True:
+    if sal.sal_pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
             os.makedirs(EXTERNAL_SCRIPTS_DIR)
         server_scripts = get_checksums()
@@ -40,8 +40,8 @@ def main():
 def get_prefs():
     # Check for mandatory prefs and bail if any are missing.
     required_prefs = {}
-    required_prefs["key"] = sal.pref('key')
-    required_prefs["ServerURL"] = sal.pref('ServerURL').rstrip('/')
+    required_prefs["key"] = sal.sal_pref('key')
+    required_prefs["ServerURL"] = sal.sal_pref('ServerURL').rstrip('/')
 
     for key, val in required_prefs.items():
         if not val:
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index c772642..8c85b11 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -31,7 +31,7 @@ def main():
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
     if log_level == logging.DEBUG:
         logging.debug("Sal client prefs:")
-        prefs = sal.prefs()
+        prefs = sal.prefs_report()
         if args.url:
             prefs['ServerURL'] = {'value': args.url, 'forced': 'commandline'}
         if args.key:
@@ -168,7 +168,7 @@ def get_plugin_results(plugin_results_plist):
 
 
 def remove_blacklisted_messages():
-    patterns = sal.pref('MessageBlacklistPatterns', [])
+    patterns = sal.sal_pref('MessageBlacklistPatterns', [])
     if patterns:
         compiled = [re.compile(p) for p in patterns]
         update = False
@@ -196,7 +196,7 @@ def remove_blacklisted_messages():
 
 
 def remove_skipped_facts():
-    if skip_facts := sal.pref('SkipFacts'):
+    if skip_facts := sal.sal_pref('SkipFacts'):
         update = False
         submission = sal.get_checkin_results()
 
diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index 3092fba..e373c8e 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -68,7 +68,7 @@ def process_system_profile():
 
 
 def get_hostname():
-    _, name_type, _ = sal.get_server_prefs()
+    name_type = sal.sal_pref('NameType', default='ComputerName')
     net_config = SCDynamicStoreCreate(None, "net", None, None)
     return get_machine_name(net_config, name_type)
 
diff --git a/payload/usr/local/sal/checkin_modules/sal_checkin.py b/payload/usr/local/sal/checkin_modules/sal_checkin.py
index f273d75..72e6638 100755
--- a/payload/usr/local/sal/checkin_modules/sal_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/sal_checkin.py
@@ -8,11 +8,10 @@
 
 
 def main():
-    _, _, bu_key = sal.get_server_prefs()
     sal_submission = {
         'extra_data': {
             'sal_version': sal.__version__,
-            'key': bu_key,},
+            'key': sal.sal_pref('key'),},
         'facts': {'checkin_module_version': __version__}}
     sal.set_checkin_results('Sal', sal_submission)
 
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 78e868f..cae3fd1 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -7,8 +7,6 @@
     MacSeshSession = None
 import requests
 
-from sal.utils import pref
-
 
 _client_instance = None
 
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index 47ea805..f774040 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,16 +1,23 @@
+import datetime
 import logging
 import os
 
-from Foundation import CFPreferencesCopyAppValue
+from Foundation import (
+    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
+    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
+    NSArray, NSDictionary, NSData, NSNull)
 
 from sal.client import get_sal_client, MacKeychainClient
-from sal.utils import pref
+
+
+BUNDLE_ID = 'com.github.salopensource.sal'
+ISO_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
 
 
 def setup_sal_client():
-    ca_cert = pref('CACert', '')
-    cert = pref('SSLClientCertificate', '')
-    key = pref('SSLClientKey', '')
+    ca_cert = sal_pref('CACert', '')
+    cert = sal_pref('SSLClientCertificate', '')
+    key = sal_pref('SSLClientKey', '')
     exists = map(os.path.exists, (ca_cert, cert, key))
     if any(exists):
         if not all(exists):
@@ -36,14 +43,130 @@ def setup_sal_client():
     if cert:
         client.cert = (cert, key) if key else cert
 
-    basic_auth = pref('BasicAuth')
+    basic_auth = sal_pref('BasicAuth')
     if basic_auth:
-        key = pref('key', '')
+        key = sal_pref('key', '')
         client.auth = ('sal', key)
 
-    client.base_url = pref('ServerURL')
+    client.base_url = sal_pref('ServerURL')
 
 
 def mac_pref(domain, key, default=None):
     val = CFPreferencesCopyAppValue(key, domain)
     return val if val is not None else default
+
+
+def set_sal_pref(pref_name, pref_value):
+    """Sets a Sal preference.
+
+    The preference file on disk is located at
+    /Library/Preferences/com.github.salopensource.sal.plist.  This should
+    normally be used only for 'bookkeeping' values; values that control
+    the behavior of munki may be overridden elsewhere (by MCX, for
+    example)
+    """
+    try:
+        CFPreferencesSetValue(
+            pref_name, pref_value, BUNDLE_ID, kCFPreferencesAnyUser, kCFPreferencesCurrentHost)
+        CFPreferencesAppSynchronize(BUNDLE_ID)
+    except Exception:
+        pass
+
+
+def sal_pref(pref_name, default=None):
+    """Return a preference value.
+
+    Since this uses CFPreferencesCopyAppValue, Preferences can be defined
+    several places. Precedence is:
+        - MCX
+        - /var/root/Library/Preferences/com.github.salopensource.sal.plist
+        - /Library/Preferences/com.github.salopensource.sal.plist
+        - default_prefs defined here.
+
+    Returned values are all converted to native python types through the
+    `unobjctify` function; e.g. dates are returned as aware-datetimes,
+    NSDictionary to dict, etc.
+    """
+    default_prefs = {
+        'ServerURL': 'http://sal',
+        'osquery_launchd': 'com.facebook.osqueryd.plist',
+        'SkipFacts': [],
+        'SyncScripts': True,
+        'BasicAuth': True,
+        'GetGrains': False,
+        'GetOhai': False,
+        'LastRunWasOffline': False,
+        'SendOfflineReport': False,
+    }
+
+    pref_value = mac_pref(BUNDLE_ID, pref_name, default)
+    if pref_value is None and pref_name in default_prefs:
+        # If we got here, the pref value was either set to None or never
+        # set, AND the default was also None. Fall back to auto prefs.
+        pref_value = default_prefs.get(pref_name)
+        # we're using a default value. We'll write it out to
+        # /Library/Preferences/<BUNDLE_ID>.plist for admin
+        # discoverability
+        set_sal_pref(pref_name, pref_value)
+
+    return unobjctify(pref_value)
+
+
+def forced(pref, bundle_identifier=BUNDLE_ID):
+    return CFPreferencesAppValueIsForced(pref, bundle_identifier)
+
+
+def prefs_report():
+    prefs = (
+        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
+        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
+    return {k: {'value': sal_pref(k), 'forced': forced(k)} for k in prefs}
+
+
+def unobjctify(element, safe=False):
+    """Recursively convert nested elements to native python datatypes.
+
+    Types accepted include str, bytes, int, float, bool, None, list,
+    dict, set, tuple, NSArray, NSDictionary, NSData, NSDate, NSNull.
+
+    element: Some (potentially) nested data you want to convert.
+
+    safe: Bool (defaults to False) whether you want printable
+        representations instead of the python equivalent. e.g.  NSDate
+        safe=True becomes a str, safe=False becomes a datetime.datetime.
+        NSData safe=True bcomes a hex str, safe=False becomes bytes. Any
+        type not explicitly handled by this module will raise an
+        exception unless safe=True, where it will instead replace the
+        data with a str of '<UNSUPPORTED TYPE>'
+
+        This is primarily for safety in serialization to plists or
+        output.
+
+    returns: Python equivalent of the original input.
+        e.g. NSArray -> List, NSDictionary -> Dict, etc.
+
+    raises: ValueError for any data that isn't supported (yet!) by this
+        function.
+    """
+    supported_types = (str, bytes, int, float, bool, datetime.datetime)
+    if isinstance(element, supported_types):
+        return element
+    elif isinstance(element, (dict, NSDictionary)):
+        return {k: unobjctify(v, safe=safe) for k, v in element.items()}
+    elif isinstance(element, (list, NSArray)):
+        return [unobjctify(i, safe=safe) for i in element]
+    elif isinstance(element, set):
+        return set([unobjctify(i, safe=safe) for i in element])
+    elif isinstance(element, tuple):
+        return tuple([unobjctify(i, safe=safe) for i in element])
+    elif isinstance(element, NSData):
+        return binascii.hexlify(element) if safe else bytes(element)
+    elif isinstance(element, NSDate):
+        return str(element) if safe else datetime.datetime.strptime(
+            element.description(), ISO_TIME_FORMAT)
+    elif isinstance(element, NSNull) or element is None:
+        return '' if safe else None
+    elif safe:
+        return '<UNSUPPORTED TYPE>'
+    raise ValueError(f"Element type '{type(element)}' is not supported!")
+
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 269dbc7..6296e62 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -13,82 +13,8 @@
 import subprocess
 import time
 
-from Foundation import (
-    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
-    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
-    NSArray, NSDictionary, NSData, NSNull)
 
-
-BUNDLE_ID = 'com.github.salopensource.sal'
 RESULTS_PATH = '/usr/local/sal/checkin_results.json'
-ISO_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
-
-
-def set_pref(pref_name, pref_value):
-    """Sets a Sal preference.
-
-    The preference file on disk is located at
-    /Library/Preferences/com.github.salopensource.sal.plist.  This should
-    normally be used only for 'bookkeeping' values; values that control
-    the behavior of munki may be overridden elsewhere (by MCX, for
-    example)
-    """
-    try:
-        CFPreferencesSetValue(
-            pref_name, pref_value, BUNDLE_ID, kCFPreferencesAnyUser, kCFPreferencesCurrentHost)
-        CFPreferencesAppSynchronize(BUNDLE_ID)
-    except Exception:
-        pass
-
-
-def pref(pref_name, default=None):
-    """Return a preference value.
-
-    Since this uses CFPreferencesCopyAppValue, Preferences can be defined
-    several places. Precedence is:
-        - MCX
-        - /var/root/Library/Preferences/com.github.salopensource.sal.plist
-        - /Library/Preferences/com.github.salopensource.sal.plist
-        - default_prefs defined here.
-
-    Returned values are all converted to native python types through the
-    `unobjctify` function; e.g. dates are returned as aware-datetimes,
-    NSDictionary to dict, etc.
-    """
-    default_prefs = {
-        'ServerURL': 'http://sal',
-        'osquery_launchd': 'com.facebook.osqueryd.plist',
-        'SkipFacts': [],
-        'SyncScripts': True,
-        'BasicAuth': True,
-        'GetGrains': False,
-        'GetOhai': False,
-        'LastRunWasOffline': False,
-        'SendOfflineReport': False,
-    }
-
-    pref_value = CFPreferencesCopyAppValue(pref_name, BUNDLE_ID)
-    if pref_value is None and default is not None:
-        pref_value = default
-    elif pref_value is None and pref_name in default_prefs:
-        pref_value = default_prefs.get(pref_name)
-        # we're using a default value. We'll write it out to
-        # /Library/Preferences/<BUNDLE_ID>.plist for admin
-        # discoverability
-        set_pref(pref_name, pref_value)
-
-    return unobjctify(pref_value)
-
-
-def prefs():
-    prefs = (
-        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
-        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
-    return {k: {'value': pref(k), 'forced': forced(k)} for k in prefs}
-
-
-def forced(pref):
-    return CFPreferencesAppValueIsForced(pref, BUNDLE_ID)
 
 
 def wait_for_script(scriptname, repeat=3, pause=1):
@@ -243,75 +169,6 @@ def run_scripts(dir_path, cli_args=None, error=False):
     return results
 
 
-def get_server_prefs():
-    """Get Sal preferences, bailing if required info is missing.
-
-    Returns:
-        Tuple of (Server URL, NameType, and key (business unit key)
-    """
-    # Check for mandatory prefs and bail if any are missing.
-    required_prefs = {
-        'key': pref('key'),
-        'server_url': pref('ServerURL').rstrip('/')}
-
-    for key, val in required_prefs.items():
-        if not val:
-            exit(f'Required Sal preference "{key}" is not set.')
-
-    # Get optional preferences.
-    name_type = pref('NameType', default='ComputerName')
-
-    return required_prefs["server_url"], name_type, required_prefs["key"]
-
-
-def unobjctify(element, safe=False):
-    """Recursively convert nested elements to native python datatypes.
-
-    Types accepted include str, bytes, int, float, bool, None, list,
-    dict, set, tuple, NSArray, NSDictionary, NSData, NSDate, NSNull.
-
-    element: Some (potentially) nested data you want to convert.
-
-    safe: Bool (defaults to False) whether you want printable
-        representations instead of the python equivalent. e.g.  NSDate
-        safe=True becomes a str, safe=False becomes a datetime.datetime.
-        NSData safe=True bcomes a hex str, safe=False becomes bytes. Any
-        type not explicitly handled by this module will raise an
-        exception unless safe=True, where it will instead replace the
-        data with a str of '<UNSUPPORTED TYPE>'
-
-        This is primarily for safety in serialization to plists or
-        output.
-
-    returns: Python equivalent of the original input.
-        e.g. NSArray -> List, NSDictionary -> Dict, etc.
-
-    raises: ValueError for any data that isn't supported (yet!) by this
-        function.
-    """
-    supported_types = (str, bytes, int, float, bool, datetime.datetime)
-    if isinstance(element, supported_types):
-        return element
-    elif isinstance(element, (dict, NSDictionary)):
-        return {k: unobjctify(v, safe=safe) for k, v in element.items()}
-    elif isinstance(element, (list, NSArray)):
-        return [unobjctify(i, safe=safe) for i in element]
-    elif isinstance(element, set):
-        return set([unobjctify(i, safe=safe) for i in element])
-    elif isinstance(element, tuple):
-        return tuple([unobjctify(i, safe=safe) for i in element])
-    elif isinstance(element, NSData):
-        return binascii.hexlify(element) if safe else bytes(element)
-    elif isinstance(element, NSDate):
-        return str(element) if safe else datetime.datetime.strptime(
-            element.description(), ISO_TIME_FORMAT)
-    elif isinstance(element, NSNull) or element is None:
-        return '' if safe else None
-    elif safe:
-        return '<UNSUPPORTED TYPE>'
-    raise ValueError(f"Element type '{type(element)}' is not supported!")
-
-
 def submission_encode(data: bytes) -> bytes:
     """Return a b64 encoded, bz2 compressed copy of text."""
     return base64.b64encode(bz2.compress(data))

From 63ad327b5a3c9d50980452e59f8a21570dbb32ec Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 21:02:39 -0400
Subject: [PATCH 35/79] Move more mac-specific code to mac_utils.

---
 sal_python_pkg/sal/mac_utils.py | 80 ++++++++++++++++++++++++++++++-
 sal_python_pkg/sal/utils.py     | 85 +++------------------------------
 2 files changed, 85 insertions(+), 80 deletions(-)

diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index f774040..4acfe76 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,6 +1,10 @@
+import binascii
 import datetime
 import logging
 import os
+import pathlib
+import subprocess
+import time
 
 from Foundation import (
     kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
@@ -156,7 +160,7 @@ def unobjctify(element, safe=False):
     elif isinstance(element, (list, NSArray)):
         return [unobjctify(i, safe=safe) for i in element]
     elif isinstance(element, set):
-        return set([unobjctify(i, safe=safe) for i in element])
+        return {unobjctify(i, safe=safe) for i in element}
     elif isinstance(element, tuple):
         return tuple([unobjctify(i, safe=safe) for i in element])
     elif isinstance(element, NSData):
@@ -170,3 +174,77 @@ def unobjctify(element, safe=False):
         return '<UNSUPPORTED TYPE>'
     raise ValueError(f"Element type '{type(element)}' is not supported!")
 
+
+def script_is_running(scriptname):
+    """Returns Process ID for a running python script.
+
+    Not at all stolen from Munki. Honest.
+    """
+    cmd = ['/bin/ps', '-eo', 'pid=,command=']
+    proc = subprocess.Popen(
+        cmd, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    out, _ = proc.communicate()
+    mypid = os.getpid()
+    for line in out.splitlines():
+        try:
+            pid, process = line.split(maxsplit=1)
+        except ValueError:
+            # funky process line, so we'll skip it
+            pass
+        else:
+            args = process.split()
+            try:
+                # first look for Python processes
+                if 'MacOS/Python' in args[0] or 'python' in args[0]:
+                    # look for first argument being scriptname
+                    if scriptname in args[1]:
+                        try:
+                            if int(pid) != mypid:
+                                return True
+                        except ValueError:
+                            # pid must have some funky characters
+                            pass
+            except IndexError:
+                pass
+
+    # if we get here we didn't find a Python script with scriptname
+    # (other than ourselves)
+    return False
+
+
+def run_scripts(dir_path, cli_args=None, error=False):
+    results = []
+    skip_names = {'__pycache__'}
+    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
+    for script in scripts:
+        if not os.access(script, os.X_OK):
+            results.append(f"'{script}' is not executable or has bad permissions")
+            continue
+
+        cmd = [script]
+        if cli_args:
+            cmd.append(cli_args)
+        try:
+            subprocess.check_call(cmd)
+            results.append(f"'{script}' ran successfully")
+        except (OSError, subprocess.CalledProcessError):
+            errormsg = f"'{script}' had errors during execution!"
+            if not error:
+                results.append(errormsg)
+            else:
+                raise RuntimeError(errormsg)
+
+    return results
+
+
+def wait_for_script(scriptname, repeat=3, pause=1):
+    """Tries a few times to wait for a script to finish."""
+    count = 0
+    while count < repeat:
+        if script_is_running(scriptname):
+            time.sleep(pause)
+            count += 1
+        else:
+            return False
+    return True
+
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 6296e62..51eaa8a 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -2,68 +2,17 @@
 
 
 import base64
-import binascii
 import bz2
 import datetime
 import hashlib
 import json
 import os
+import platform
 import pathlib
 import plistlib
-import subprocess
-import time
 
 
-RESULTS_PATH = '/usr/local/sal/checkin_results.json'
-
-
-def wait_for_script(scriptname, repeat=3, pause=1):
-    """Tries a few times to wait for a script to finish."""
-    count = 0
-    while count < repeat:
-        if script_is_running(scriptname):
-            time.sleep(pause)
-            count += 1
-        else:
-            return False
-    return True
-
-
-def script_is_running(scriptname):
-    """Returns Process ID for a running python script.
-
-    Not at all stolen from Munki. Honest.
-    """
-    cmd = ['/bin/ps', '-eo', 'pid=,command=']
-    proc = subprocess.Popen(
-        cmd, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    out, _ = proc.communicate()
-    mypid = os.getpid()
-    for line in out.splitlines():
-        try:
-            pid, process = line.split(maxsplit=1)
-        except ValueError:
-            # funky process line, so we'll skip it
-            pass
-        else:
-            args = process.split()
-            try:
-                # first look for Python processes
-                if 'MacOS/Python' in args[0] or 'python' in args[0]:
-                    # look for first argument being scriptname
-                    if scriptname in args[1]:
-                        try:
-                            if int(pid) != mypid:
-                                return True
-                        except ValueError:
-                            # pid must have some funky characters
-                            pass
-            except IndexError:
-                pass
-
-    # if we get here we didn't find a Python script with scriptname
-    # (other than ourselves)
-    return False
+RESULTS_PATH = {'Darwin': '/usr/local/sal/checkin_results.json'}.get(platform.system())
 
 
 def get_hash(file_path):
@@ -86,7 +35,10 @@ def add_plugin_results(plugin, data, historical=False):
         historical (bool): Whether to keep only one record (False) or
             all results (True). Optional, defaults to False.
     """
-    plist_path = pathlib.Path('/usr/local/sal/plugin_results.plist')
+    if platform.system() == 'Darwin':
+        plist_path = pathlib.Path('/usr/local/sal/plugin_results.plist')
+    else:
+        raise NotImplementedError('Please PR a plugin results path for your platform!')
     if plist_path.exists():
         plugin_results = plistlib.loads(plist_path.read_bytes())
     else:
@@ -144,31 +96,6 @@ def serializer(obj):
     return obj
 
 
-def run_scripts(dir_path, cli_args=None, error=False):
-    results = []
-    skip_names = {'__pycache__'}
-    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
-    for script in scripts:
-        if not os.access(script, os.X_OK):
-            results.append(f"'{script}' is not executable or has bad permissions")
-            continue
-
-        cmd = [script]
-        if cli_args:
-            cmd.append(cli_args)
-        try:
-            subprocess.check_call(cmd)
-            results.append(f"'{script}' ran successfully")
-        except (OSError, subprocess.CalledProcessError):
-            errormsg = f"'{script}' had errors during execution!"
-            if not error:
-                results.append(errormsg)
-            else:
-                raise RuntimeError(errormsg)
-
-    return results
-
-
 def submission_encode(data: bytes) -> bytes:
     """Return a b64 encoded, bz2 compressed copy of text."""
     return base64.b64encode(bz2.compress(data))

From 57d0069a75436a6de50b990881e03b69779eb526 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@users.noreply.github.com>
Date: Sat, 30 May 2020 15:21:28 -0400
Subject: [PATCH 36/79] Fix missing import.

---
 payload/usr/local/sal/checkin_modules/machine_checkin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index e373c8e..bd4d3e2 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -8,6 +8,7 @@
 import sys
 from xml.etree import ElementTree
 
+import macsesh
 from SystemConfiguration import (
     SCDynamicStoreCreate, SCDynamicStoreCopyValue, SCDynamicStoreCopyConsoleUser)
 

From 404565389d09bb5ede527587a4a3921aec301cf3 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 13 Jul 2020 17:04:05 -0400
Subject: [PATCH 37/79] Use status code objects rather than int literal.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 6bde9f7..fe53676 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -63,7 +63,7 @@ def get_checksums():
     except requests.exceptions.RequestException as error:
         munkicommon.display_debug2(str(error_msg))
         return
-    if response.status_code != 200:
+    if response.status_code != requests.status_codes.codes.okay:
         munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
         return
     if response and "<h1>Page not found</h1>" in response.text:
@@ -105,7 +105,7 @@ def download_and_write_script(server_script):
         munkicommon.display_debug2(str(error))
         return
 
-    if response.status_code != 200:
+    if response.status_code != requests.status_codes.codes.okay:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)
 

From c5a2d206e5492442fe33fb5cfceb86b8001092f2 Mon Sep 17 00:00:00 2001
From: johnmikep <45859899+johnmikep@users.noreply.github.com>
Date: Tue, 14 Jul 2020 11:58:57 -0500
Subject: [PATCH 38/79] Really old machines

---
 payload/usr/local/sal/checkin_modules/machine_checkin.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index bcc2cb9..a6fe562 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -105,6 +105,11 @@ def get_model_code(serial):
             # Remove S prefix from scanned codes.
             serial = serial[1:]
         return serial[8:].upper()
+    
+    elif 11 <= len(serial) <= 12:
+        # 2010 Mac Pros starting with H or Y are 11 characters
+        return serial[8:].upper()
+
     return
 
 

From 32b2989311dce7f90f7ec039a2a2c56b6577b901 Mon Sep 17 00:00:00 2001
From: Nick Zolotko <nzolotko@laika.com>
Date: Mon, 27 Jul 2020 15:31:11 -0700
Subject: [PATCH 39/79] Fix profile checkin error.

---
 payload/usr/local/sal/checkin_modules/profile_checkin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/sal/checkin_modules/profile_checkin.py b/payload/usr/local/sal/checkin_modules/profile_checkin.py
index a4eeed3..2165425 100755
--- a/payload/usr/local/sal/checkin_modules/profile_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/profile_checkin.py
@@ -32,7 +32,7 @@ def main():
         data['payload_types'] = ', '.join(p['PayloadType'] for p in payloads)
         data['profile_description'] = profile.get('ProfileDescription', 'None')
         data['identifier'] = profile['ProfileIdentifier']
-        data['organization'] = profile['ProfileOrganization'] or 'None'
+        data['organization'] = profile.get('ProfileOrganization' or 'None')
         data['uuid'] = profile['ProfileUUID']
         data['verification_state'] = profile.get('ProfileVerificationState', '')
         submission_item['data'] = data

From 59022e186c6c1ed52eec1e12cdeb588cebf489c1 Mon Sep 17 00:00:00 2001
From: Wesley Whetstone <wes@squareup.com>
Date: Mon, 3 Aug 2020 16:15:10 -0700
Subject: [PATCH 40/79] add urllib3 to requirements.txt that includes fixes for
 BigSur

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 17bb28c..113d042 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 sal_python_pkg/
 pyobjc==6.2
+urllib3==1.25.10

From b0cb706f0fec7fffbf0a6ea048961f871485d53e Mon Sep 17 00:00:00 2001
From: Wesley Whetstone <wes@squareup.com>
Date: Mon, 3 Aug 2020 16:17:24 -0700
Subject: [PATCH 41/79] update pyobjc to BigSur supported version

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 113d042..993fbea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
 sal_python_pkg/
-pyobjc==6.2
+pyobjc==6.2.2
 urllib3==1.25.10

From a4ec7318254eb9a608e7daccfdaa43454a1ef49e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 17 Apr 2020 16:04:39 -0400
Subject: [PATCH 42/79] Add requests.

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 993fbea..2a4d459 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
 sal_python_pkg/
 pyobjc==6.2.2
 urllib3==1.25.10
+requests==2.23.0

From 0a07dc99b3a74380136e0fa924df04b72b62c33e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Thu, 30 Apr 2020 15:54:09 -0400
Subject: [PATCH 43/79] WS fix.

---
 payload/usr/local/sal/bin/sal-submit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 446c1dd..5c34f54 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -237,7 +237,7 @@ def send_inventory(server_url, serial):
     inventory_plist = pathlib.Path(managed_install_dir) / 'ApplicationInventory.plist'
     logging.debug('ApplicationInventory.plist Path: %s', inventory_plist)
 
-    if inventory:= inventory_plist.read_bytes():
+    if inventory := inventory_plist.read_bytes():
         inventory_hash = sal.get_hash(inventory_plist)
         serverhash = None
         serverhash, stderr = sal.curl(hash_url)

From 9896100e4c06c567c7dc3ab93c00d9acc7c3c1db Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:11:30 -0400
Subject: [PATCH 44/79] Add MacSesh package to sal-scripts Python
 requirements.txt.

This is used for making python requests use the keychain.
---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 2a4d459..d9634ba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ sal_python_pkg/
 pyobjc==6.2.2
 urllib3==1.25.10
 requests==2.23.0
+MacSesh==0.2.1

From 6f33f267a78e28dfdc211a516480fd2a57e2c798 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:12:07 -0400
Subject: [PATCH 45/79] Add SalClient class to replace curl func and related.

---
 sal_python_pkg/sal/utils.py | 88 ++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 50 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index b98e17c..8859924 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -7,6 +7,7 @@
 import datetime
 import hashlib
 import json
+import logging
 import os
 import pathlib
 import plistlib
@@ -15,6 +16,7 @@
 import time
 import urllib.parse
 
+import macsesh
 from Foundation import (kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
                         CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, NSDate, NSArray,
                         NSDictionary, NSData, NSNull)
@@ -132,68 +134,54 @@ def script_is_running(scriptname):
     return False
 
 
-def curl(url, data=None, json_path=None):
-    cmd = ['/usr/bin/curl', '--silent', '--show-error', '--connect-timeout', '2']
-
-    # Use a PEM format certificate file to verify the peer. This is
-    # useful primarily to support self-signed certificates, which are
-    # rejected on 10.13's bundled curl. In cases where you have a cert
-    # signed by an internal or external trusted CA, curl will happily
-    # use the keychain.
-    ca_cert = pref('CACert')
-    if ca_cert:
-        cmd += ['--cacert', ca_cert]
+def get_hash(file_path):
+    """Return sha256 hash of file_path."""
+    text = b''
+    if (path := pathlib.Path(file_path)).is_file():
+        text = path.read_bytes()
+    return hashlib.sha256(text).hexdigest()
 
-    basic_auth = pref('BasicAuth')
-    if basic_auth:
-        key = pref('key')
-        user_pass = f'sal:{key}'
-        cmd += ['--user', user_pass]
 
-    ssl_client_cert = pref('SSLClientCertificate')
-    ssl_client_key = pref('SSLClientKey')
-    if ssl_client_cert:
-        cmd += ['--cert', ssl_client_cert]
-        if ssl_client_key:
-            cmd += ['--key', ssl_client_key]
+class SalClient():
 
-    max_time = '8' if data else '4'
-    cmd += ['--max-time', max_time]
+    basic_timeout = (3.05, 4)
+    post_timeout = (3.05, 8)
 
-    cmd += ['--header', f'SalScript-Version: {sal.version.__version__}']
+    def __init__(self):
+        sesh = macsesh.KeychainSession()
 
-    if data:
-        cmd += ['--data', data]
-    elif json_path:
-        cmd += ['--header', 'Content-Type: application/json']
-        # Use the @ syntax for curl to open the file and do any required
-        # encoding for us.
-        cmd += ['--data', f'@{json_path}']
+        ca_cert = pref('CACert')
+        if ca_cert:
+            sesh.verify = ca_cert
 
-    cmd.append(url)
+        basic_auth = pref('BasicAuth')
+        if basic_auth:
+            key = pref('key', '')
+            sesh.auth = ('sal', key)
 
-    task = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    return task.communicate()
+        # TODO: Handle keychain-based certs.
+        ssl_client_cert = pref('SSLClientCertificate')
+        ssl_client_key = pref('SSLClientKey')
+        if ssl_client_cert:
+            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
 
+        self.sesh = sesh
 
-def get_hash(file_path):
-    """Return sha256 hash of file_path."""
-    text = b''
-    if (path := pathlib.Path(file_path)).is_file():
-        text = path.read_bytes()
-    return hashlib.sha256(text).hexdigest()
+    def get(self, url):
+        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
 
 
-def send_report(url, form_data=None, json_data=None, json_path=None):
-    if form_data:
-        # urlencode allows bytes and str in its dict arg.
-        stdout, stderr = curl(url, data=urllib.parse.urlencode(form_data))
-    elif json_data:
-        raise NotImplementedError
-    elif json_path:
-        stdout, stderr = curl(url, json_path=RESULTS_PATH)
+    def post(self, url, data=None, json=None):
+        kwargs = {'timeout': self.post_timeout}
+        if json:
+            kwargs['json'] = json
+        else:
+            kwargs['data'] = data
+        return self.log_response(self.sesh.post(url, **kwargs))
 
-    return stdout, stderr
+    def log_response(self, response):
+        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
+        return response
 
 
 def add_plugin_results(plugin, data, historical=False):

From 9aad8975063cbc744d5e31a7e2b15f32890ce63b Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:12:49 -0400
Subject: [PATCH 46/79] Update preflight script to use requests.

---
 .../usr/local/munki/preflight.d/sal-preflight | 37 +++++++++++--------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 3539902..0b227a6 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -6,7 +6,6 @@ Retrieves plugin scripts to run on client.
 
 
 import argparse
-import json
 import os
 import pathlib
 import shutil
@@ -27,7 +26,7 @@ def main():
     if sal.pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
             os.makedirs(EXTERNAL_SCRIPTS_DIR)
-        server_scripts = get_checksum()
+        server_scripts = get_checksums()
         if server_scripts:
             create_dirs(server_scripts)
             download_scripts(server_scripts)
@@ -48,7 +47,7 @@ def get_prefs():
     return required_prefs
 
 
-def get_checksum():
+def get_checksums():
     """Downloads the checksum of existing scripts.
 
     Returns:
@@ -56,17 +55,24 @@ def get_checksum():
         or None if no external scripts are used.
     """
     preflight_url = f"{sal.pref('ServerURL')}/preflight-v2/"
-    stdout, stderr = sal.send_report(preflight_url, form_data={'os_family': 'Darwin'})
+    sal_client = sal.SalClient()
+    error_msg = None
+    try:
+        response = sal_client.post(preflight_url, data={'os_family': 'Darwin'})
+    except Exception as error:
+        error_msg = str(error)
+    if response.status_code != 200:
+        error_msg = f'Request failed with HTTP {response.status_code}'
+
+    if "<h1>Page not found</h1>" not in response.text:
+        munkicommon.display_debug2(response.text)
 
-    if stderr:
-        munkicommon.display_debug2(stderr)
-    stdout_list = stdout.split("\n")
-    if "<h1>Page not found</h1>" not in stdout_list:
-        munkicommon.display_debug2(stdout)
+    if error_msg:
+        munkicommon.display_debug2(error_msg)
 
     try:
-        return json.loads(stdout)
-    except:
+        return response.json()
+    except ValueError:
         munkicommon.display_debug2("Didn't receive valid JSON.")
         return None
 
@@ -94,16 +100,17 @@ def download_and_write_script(server_script):
     script_url = (
         f"{sal.pref('ServerURL')}/preflight-v2/get-script/"
         f"{server_script['plugin']}/{server_script['filename']}/")
-    stdout, stderr = sal.curl(script_url)
-    if stderr:
+    sal_client = sal.SalClient()
+    response = sal_client.get(script_url)
+    if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
-        munkicommon.display_debug2(stderr)
+        munkicommon.display_debug2(response.text)
 
     script = open(
         os.path.join(EXTERNAL_SCRIPTS_DIR, server_script['plugin'], server_script['filename']),
         'w')
     try:
-        data = json.loads(stdout)
+        data = response.json()
     except:
         munkicommon.display_debug2('Did not receive valid JSON when requesting script content.')
         return False

From dedd23eab8b159e536cafc7badc5dea07f731016 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:16:42 -0400
Subject: [PATCH 47/79] Update machine_checkin model lookup to use requests.

Also, this prevents a bug I discovered that would have a null model
being written to disk, and then no further attempts to check later.
---
 .../sal/checkin_modules/machine_checkin.py      | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index a6fe562..173b08d 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -86,7 +86,8 @@ def get_friendly_model(serial):
     """Return friendly model name"""
     if not MODEL_PATH.exists():
         model = cleanup_model(query_apple_support(serial))
-        MODEL_PATH.write_text(model)
+        if model:
+            MODEL_PATH.write_text(model)
     else:
         try:
             model = MODEL_PATH.read_text().strip()
@@ -116,17 +117,13 @@ def get_model_code(serial):
 def query_apple_support(serial):
     model_code = get_model_code(serial)
     tree = ElementTree.ElementTree()
+    session = macsesh.KeychainSession()
+    response = session.get(f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US")
     try:
-        response = subprocess.check_output(
-            ['curl', f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US"],
-            text=True)
-    except subprocess.CalledProcessError:
-        pass
-    try:
-        tree = ElementTree.fromstring(response)
+        tree = ElementTree.fromstring(response.text)
     except ElementTree.ParseError:
-        pass
-    return tree.findtext("configCode")
+        tree = None
+    return tree.findtext("configCode") if tree else None
 
 
 def cleanup_model(model):

From 73cf7453ec10807bcdab64e29302e4458485074c Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:18:01 -0400
Subject: [PATCH 48/79] Update sal-submit to use requests.

---
 payload/usr/local/sal/bin/sal-submit | 79 ++++++++++++----------------
 1 file changed, 35 insertions(+), 44 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 5c34f54..0a6c5c4 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -53,9 +53,9 @@ def main():
         submission = sal.get_checkin_results()
         logging.debug('Checkin submission:')
         logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
-    _, errors = send_checkin(server_url)
+    response = send_checkin(server_url)
 
-    if not errors:
+    if response.status_code == 200:
         sal.clean_results()
 
     # Speed up manual runs by skipping these potentially slow-running,
@@ -214,16 +214,8 @@ def send_checkin(server_url):
     checkinurl = os.path.join(server_url, 'checkin', '')
     logging.debug(f"Sending report to {checkinurl}")
     logging.debug("Checkin Response:")
-    out, error = sal.send_report(checkinurl, json_path=sal.RESULTS_PATH)
-    log(out, error)
-    return out, error
-
-
-def log(out, error):
-    if out:
-        logging.debug(out.strip())
-    if error:
-        logging.debug(error.strip())
+    sal_client = sal.SalClient()
+    return sal_client.post(checkinurl, json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
 def send_inventory(server_url, serial):
@@ -239,18 +231,17 @@ def send_inventory(server_url, serial):
 
     if inventory := inventory_plist.read_bytes():
         inventory_hash = sal.get_hash(inventory_plist)
+        logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
-        serverhash, stderr = sal.curl(hash_url)
-        if stderr:
-            return
-        if serverhash != inventory_hash:
+        sal_client = sal.SalClient()
+        response = sal_client.get(hash_url)
+        if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
             logging.debug("Inventory report response:")
-            out, error = sal.send_report(inventory_submit_url, form_data=inventory_submission)
-            log(out, error)
+            sal_client.post(inventory_submit_url, data=inventory_submission)
 
 
 def send_catalogs(server_url, machine_group_key):
@@ -275,29 +266,31 @@ def send_catalogs(server_url, machine_group_key):
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    response, stderr = sal.send_report(hash_url, form_data=hash_submission)
-
-    if stderr is not None:
-        try:
-            remote_data = plistlib.loads(response.encode())
-        except plistlib.InvalidFileException:
-            remote_data = []
-
-        for catalog in check_list:
-            if catalog not in remote_data:
-                contents = (pathlib.Path(catalog_dir) / catalog['name']).read_bytes()
-                catalog_submission = {
-                    'key': machine_group_key,
-                    'base64bz2catalog': sal.submission_encode(contents),
-                    'name': catalog['name'],
-                    'sha256hash': catalog['sha256hash']}
+    sal_client = sal.SalClient()
+    try:
+        response = sal_client.post(hash_url, data=hash_submission)
+    except:
+        return
 
-                logging.debug("Submitting Catalog: %s", catalog['name'])
-                try:
-                    out, error = sal.send_report(catalog_submit_url, form_data=catalog_submission)
-                    log(out, error)
-                except OSError:
-                    logging.warning("Error while submitting Catalog: %s", catalog['name'])
+    try:
+        remote_data = plistlib.loads(response.content)
+    except plistlib.InvalidFileException:
+        remote_data = []
+
+    for catalog in check_list:
+        if catalog not in remote_data:
+            contents = (pathlib.Path(catalog_dir) / catalog['name']).read_bytes()
+            catalog_submission = {
+                'key': machine_group_key,
+                'base64bz2catalog': sal.submission_encode(contents),
+                'name': catalog['name'],
+                'sha256hash': catalog['sha256hash']}
+
+            logging.debug("Submitting Catalog: %s", catalog['name'])
+            try:
+                sal_client.post(catalog_submit_url, data=catalog_submission)
+            except OSError:
+                logging.warning("Error while submitting Catalog: %s", catalog['name'])
 
 
 def send_profiles(server_url, serial):
@@ -319,10 +312,8 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-
-    logging.debug("Profiles Response:")
-    out, error = sal.send_report(profile_submit_url, form_data=profile_submission)
-    log(out, error)
+    sal_client = sal.SalClient()
+    sal_client.post(profile_submit_url, data=profile_submission)
 
 
 if __name__ == "__main__":

From 27dc7b8134cdec5ee96be4a1f202f2e15983ca6b Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 1 May 2020 13:40:43 -0400
Subject: [PATCH 49/79] Skip trying to execute py3 cache, and set up place to
 skip others.

---
 sal_python_pkg/sal/utils.py | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 8859924..0d27d7d 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -251,25 +251,28 @@ def serializer(obj):
     return obj
 
 
-def run_scripts(dir_path, cli_args=None):
+def run_scripts(dir_path, cli_args=None, error=False):
     results = []
-    for script in os.listdir(dir_path):
-        script_stat = os.stat(os.path.join(dir_path, script))
-        if not script_stat.st_mode & stat.S_IWOTH:
-            cmd = [os.path.join(dir_path, script)]
-            if cli_args:
-                cmd.append(cli_args)
-            try:
-                subprocess.check_call(cmd, stdin=None)
-                results.append("'{}' ran successfully")
-            except (OSError, subprocess.CalledProcessError):
-                errormsg = "'{}' had error during execution!".format(script)
-                if not error:
-                    results.append(errormsg)
-                else:
-                    raise RuntimeError(errormsg)
-        else:
+    skip_names = {'__pycache__'}
+    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
+    for script in scripts:
+        if script.stat().st_mode & stat.S_IWOTH:
             results.append(f"'{script}' is not executable or has bad permissions")
+            continue
+
+        cmd = [script]
+        if cli_args:
+            cmd.append(cli_args)
+        try:
+            subprocess.check_call(cmd)
+            results.append(f"'{script}' ran successfully")
+        except (OSError, subprocess.CalledProcessError):
+            errormsg = f"'{script}' had errors during execution!"
+            if not error:
+                results.append(errormsg)
+            else:
+                raise RuntimeError(errormsg)
+
     return results
 
 

From 3f839dd1b00d7da1470d4990f443852749077cc3 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:24:37 -0400
Subject: [PATCH 50/79] Handle json errors in checkin results by using an empty
 dict.

Just start again!
---
 sal_python_pkg/sal/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 0d27d7d..3580b60 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -209,7 +209,10 @@ def add_plugin_results(plugin, data, historical=False):
 def get_checkin_results():
     if os.path.exists(RESULTS_PATH):
         with open(RESULTS_PATH) as results_handle:
-            results = json.load(results_handle)
+            try:
+                results = json.load(results_handle)
+            except json.decoder.JSONDecodeError:
+                results = {}
     else:
         results = {}
 

From 4837a260304da5a29f0a9074ac75dc7a5ac2c3a9 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:54:20 -0400
Subject: [PATCH 51/79] Move SalClient into its own module.

---
 sal_python_pkg/sal/__init__.py   |  1 +
 sal_python_pkg/sal/client.py     | 58 ++++++++++++++++++++++++++++++++
 sal_python_pkg/sal/exceptions.py |  2 ++
 sal_python_pkg/sal/utils.py      | 42 -----------------------
 4 files changed, 61 insertions(+), 42 deletions(-)
 create mode 100644 sal_python_pkg/sal/client.py
 create mode 100644 sal_python_pkg/sal/exceptions.py

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index 789f79b..e987577 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,2 +1,3 @@
+from sal.client import SalClient
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
new file mode 100644
index 0000000..8cfad40
--- /dev/null
+++ b/sal_python_pkg/sal/client.py
@@ -0,0 +1,58 @@
+import logging
+
+import macsesh
+
+from sal.utils import pref
+
+
+class SalClient():
+
+    basic_timeout = (3.05, 4)
+    post_timeout = (3.05, 8)
+    base_url = ''
+
+    def __init__(self):
+        sesh = macsesh.KeychainSession()
+        # sesh = macsesh.SecureTransportSession()
+
+        base_url = pref('ServerURL')
+        self.base_url = base_url if not base_url.endswith('/') else base_url[:-1]
+
+        ca_cert = pref('CACert')
+        if ca_cert:
+            sesh.verify = ca_cert
+
+        basic_auth = pref('BasicAuth')
+        if basic_auth:
+            key = pref('key', '')
+            sesh.auth = ('sal', key)
+
+        # TODO: Handle keychain-based certs.
+        ssl_client_cert = pref('SSLClientCertificate')
+        ssl_client_key = pref('SSLClientKey')
+        if ssl_client_cert:
+            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
+
+        self.sesh = sesh
+
+    def get(self, url):
+        url = self.build_url(url)
+        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
+
+    def post(self, url, data=None, json=None):
+        url = self.build_url(url)
+        kwargs = {'timeout': self.post_timeout}
+        if json:
+            kwargs['json'] = json
+        else:
+            kwargs['data'] = data
+        return self.log_response(self.sesh.post(url, **kwargs))
+
+    def log_response(self, response):
+        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
+        return response
+
+    def build_url(self, url):
+        url = url[1:] if url.startswith('/') else url
+        url = url[:-1] if url.endswith('/') else url
+        return '/'.join((self.base_url, url)) + '/'
diff --git a/sal_python_pkg/sal/exceptions.py b/sal_python_pkg/sal/exceptions.py
new file mode 100644
index 0000000..32cd949
--- /dev/null
+++ b/sal_python_pkg/sal/exceptions.py
@@ -0,0 +1,2 @@
+class SalClientError(Exception):
+    pass
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 3580b60..c5534ce 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -142,48 +142,6 @@ def get_hash(file_path):
     return hashlib.sha256(text).hexdigest()
 
 
-class SalClient():
-
-    basic_timeout = (3.05, 4)
-    post_timeout = (3.05, 8)
-
-    def __init__(self):
-        sesh = macsesh.KeychainSession()
-
-        ca_cert = pref('CACert')
-        if ca_cert:
-            sesh.verify = ca_cert
-
-        basic_auth = pref('BasicAuth')
-        if basic_auth:
-            key = pref('key', '')
-            sesh.auth = ('sal', key)
-
-        # TODO: Handle keychain-based certs.
-        ssl_client_cert = pref('SSLClientCertificate')
-        ssl_client_key = pref('SSLClientKey')
-        if ssl_client_cert:
-            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
-
-        self.sesh = sesh
-
-    def get(self, url):
-        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
-
-
-    def post(self, url, data=None, json=None):
-        kwargs = {'timeout': self.post_timeout}
-        if json:
-            kwargs['json'] = json
-        else:
-            kwargs['data'] = data
-        return self.log_response(self.sesh.post(url, **kwargs))
-
-    def log_response(self, response):
-        logging.debug(f'Response HTTP {response.status_code}: {response.text}')
-        return response
-
-
 def add_plugin_results(plugin, data, historical=False):
     """Add data to the shared plugin results plist.
 

From 8dce76bb5fce5717e207f844be53a8b54e481682 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:54:42 -0400
Subject: [PATCH 52/79] Move URL building to SalClient for sal-submit.

---
 payload/usr/local/sal/bin/sal-submit | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 0a6c5c4..0b1cfe3 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -211,16 +211,13 @@ def sanitize_submission():
 
 
 def send_checkin(server_url):
-    checkinurl = os.path.join(server_url, 'checkin', '')
-    logging.debug(f"Sending report to {checkinurl}")
-    logging.debug("Checkin Response:")
+    logging.debug("Sending report")
     sal_client = sal.SalClient()
-    return sal_client.post(checkinurl, json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
 def send_inventory(server_url, serial):
     logging.info('Processing inventory...')
-    hash_url = os.path.join(server_url, 'inventory/hash', serial, '')
     inventory_submit_url = os.path.join(server_url, 'inventory/submit', '')
 
     managed_install_dir = (
@@ -234,7 +231,7 @@ def send_inventory(server_url, serial):
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
         sal_client = sal.SalClient()
-        response = sal_client.get(hash_url)
+        response = sal_client.get(f'inventory/hash/{serial}/')
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
@@ -246,7 +243,6 @@ def send_inventory(server_url, serial):
 
 def send_catalogs(server_url, machine_group_key):
     logging.info('Processing catalogs...')
-    hash_url = os.path.join(server_url, 'catalog/hash', '')
     catalog_submit_url = os.path.join(server_url, 'catalog/submit', '')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -268,7 +264,7 @@ def send_catalogs(server_url, machine_group_key):
         'catalogs': sal.submission_encode(catalog_check_plist)}
     sal_client = sal.SalClient()
     try:
-        response = sal_client.post(hash_url, data=hash_submission)
+        response = sal_client.post('catalog/hash/', data=hash_submission)
     except:
         return
 
@@ -295,8 +291,6 @@ def send_catalogs(server_url, machine_group_key):
 
 def send_profiles(server_url, serial):
     logging.info('Processing profiles...')
-    profile_submit_url = os.path.join(server_url, 'profiles/submit', '')
-
     temp_dir = tempfile.mkdtemp()
     profile_out = pathlib.Path(temp_dir) / 'profiles.plist'
 
@@ -313,7 +307,7 @@ def send_profiles(server_url, serial):
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
     sal_client = sal.SalClient()
-    sal_client.post(profile_submit_url, data=profile_submission)
+    sal_client.post('profiles/submit/', data=profile_submission)
 
 
 if __name__ == "__main__":

From 7578ddcb6d9644a7f26257a42340b467618f88c8 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 15:59:06 -0400
Subject: [PATCH 53/79] Use SalClient URL builder for preflight.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 0b227a6..6ad71b5 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -54,11 +54,10 @@ def get_checksums():
         A dict with the script name, plugin name and hash of the script
         or None if no external scripts are used.
     """
-    preflight_url = f"{sal.pref('ServerURL')}/preflight-v2/"
     sal_client = sal.SalClient()
     error_msg = None
     try:
-        response = sal_client.post(preflight_url, data={'os_family': 'Darwin'})
+        response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
     except Exception as error:
         error_msg = str(error)
     if response.status_code != 200:
@@ -97,11 +96,9 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    script_url = (
-        f"{sal.pref('ServerURL')}/preflight-v2/get-script/"
-        f"{server_script['plugin']}/{server_script['filename']}/")
     sal_client = sal.SalClient()
-    response = sal_client.get(script_url)
+    response = sal_client.get(
+        f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
     if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)

From 8e9b9308e178397e0eccba3003c024648dd39b4f Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:31:50 -0400
Subject: [PATCH 54/79] Fix missed URL update to new SalClient.

---
 payload/usr/local/sal/bin/sal-submit | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 0b1cfe3..a7fc04e 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -218,8 +218,6 @@ def send_checkin(server_url):
 
 def send_inventory(server_url, serial):
     logging.info('Processing inventory...')
-    inventory_submit_url = os.path.join(server_url, 'inventory/submit', '')
-
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
         '/Library/Managed Installs')
@@ -237,8 +235,7 @@ def send_inventory(server_url, serial):
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
-            logging.debug("Inventory report response:")
-            sal_client.post(inventory_submit_url, data=inventory_submission)
+            sal_client.post('inventory/submit/', data=inventory_submission)
 
 
 def send_catalogs(server_url, machine_group_key):

From 627823a4e694805a374580392672e1e0f383088e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:32:22 -0400
Subject: [PATCH 55/79] Add module-private SalClient "singleton" for session
 reuse purposes.

---
 payload/usr/local/munki/preflight.d/sal-preflight |  4 ++--
 payload/usr/local/sal/bin/sal-submit              |  8 ++++----
 sal_python_pkg/sal/__init__.py                    |  2 +-
 sal_python_pkg/sal/client.py                      | 10 ++++++++++
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 6ad71b5..46b9267 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -54,7 +54,7 @@ def get_checksums():
         A dict with the script name, plugin name and hash of the script
         or None if no external scripts are used.
     """
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     error_msg = None
     try:
         response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
@@ -96,7 +96,7 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     response = sal_client.get(
         f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
     if response.status_code != 200:
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index a7fc04e..1478a8d 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -212,7 +212,7 @@ def sanitize_submission():
 
 def send_checkin(server_url):
     logging.debug("Sending report")
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
 
 
@@ -228,7 +228,7 @@ def send_inventory(server_url, serial):
         inventory_hash = sal.get_hash(inventory_plist)
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
-        sal_client = sal.SalClient()
+        sal_client = sal.get_sal_client()
         response = sal_client.get(f'inventory/hash/{serial}/')
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
@@ -259,7 +259,7 @@ def send_catalogs(server_url, machine_group_key):
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
     except:
@@ -303,7 +303,7 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-    sal_client = sal.SalClient()
+    sal_client = sal.get_sal_client()
     sal_client.post('profiles/submit/', data=profile_submission)
 
 
diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index e987577..b49c5eb 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,3 @@
-from sal.client import SalClient
+from sal.client import SalClient, get_sal_client
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 8cfad40..db7e9f4 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -5,6 +5,9 @@
 from sal.utils import pref
 
 
+_client_instance = None
+
+
 class SalClient():
 
     basic_timeout = (3.05, 4)
@@ -56,3 +59,10 @@ def build_url(self, url):
         url = url[1:] if url.startswith('/') else url
         url = url[:-1] if url.endswith('/') else url
         return '/'.join((self.base_url, url)) + '/'
+
+
+def get_sal_client():
+    global _client_instance
+    if _client_instance is None:
+        _client_instance = SalClient()
+    return _client_instance

From 969d4e2e585c270aea2b8f3f7583c87ef2f2aba7 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 16:36:31 -0400
Subject: [PATCH 56/79] Remove unused import.

---
 payload/usr/local/sal/checkin_modules/munki_checkin.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/payload/usr/local/sal/checkin_modules/munki_checkin.py b/payload/usr/local/sal/checkin_modules/munki_checkin.py
index 1a8244d..e7c4b1b 100755
--- a/payload/usr/local/sal/checkin_modules/munki_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/munki_checkin.py
@@ -2,7 +2,6 @@
 
 
 import datetime
-import os
 import pathlib
 import plistlib
 import sys

From 0b63f15eea37e0852a61194e34f3caf48dd1adc0 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 4 May 2020 17:26:33 -0400
Subject: [PATCH 57/79] Do a better job of handling requests exceptions.

---
 .../usr/local/munki/preflight.d/sal-preflight | 31 ++++++++------
 payload/usr/local/sal/bin/sal-submit          | 42 ++++++++++++++-----
 2 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 46b9267..92a2e56 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -12,6 +12,7 @@ import shutil
 import sys
 import urllib
 
+import requests.exceptions
 import sal
 sys.path.append('/usr/local/munki')
 from munkilib import munkicommon
@@ -58,16 +59,15 @@ def get_checksums():
     error_msg = None
     try:
         response = sal_client.post('preflight-v2/', data={'os_family': 'Darwin'})
-    except Exception as error:
-        error_msg = str(error)
+    except requests.exceptions.RequestException as error:
+        munkicommon.display_debug2(str(error_msg))
+        return
     if response.status_code != 200:
-        error_msg = f'Request failed with HTTP {response.status_code}'
-
-    if "<h1>Page not found</h1>" not in response.text:
+        munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
+        return
+    if response and "<h1>Page not found</h1>" not in response.text:
         munkicommon.display_debug2(response.text)
-
-    if error_msg:
-        munkicommon.display_debug2(error_msg)
+        return
 
     try:
         return response.json()
@@ -96,9 +96,14 @@ def download_scripts(server_scripts):
 
 def download_and_write_script(server_script):
     """Gets script from the server and makes it execuatble."""
-    sal_client = sal.get_sal_client()
-    response = sal_client.get(
-        f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
+    try:
+        response = sal.get_sal_client().get(
+            f"preflight-v2/get-script/{server_script['plugin']}/{server_script['filename']}/")
+    except requests.exceptions.RequestException as error:
+        munkicommon.display_debug2('Error received downloading script:')
+        munkicommon.display_debug2(str(error))
+        return
+
     if response.status_code != 200:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)
@@ -108,9 +113,9 @@ def download_and_write_script(server_script):
         'w')
     try:
         data = response.json()
-    except:
+    except ValueError:
         munkicommon.display_debug2('Did not receive valid JSON when requesting script content.')
-        return False
+        return
 
     script.write(data[0]['content'])
     script.close()
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 1478a8d..bcdb09b 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -17,6 +17,7 @@ import subprocess
 import tempfile
 
 from Foundation import CFPreferencesCopyAppValue
+import requests.exceptions
 
 import sal
 
@@ -55,7 +56,7 @@ def main():
         logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
     response = send_checkin(server_url)
 
-    if response.status_code == 200:
+    if response and response.status_code == 200:
         sal.clean_results()
 
     # Speed up manual runs by skipping these potentially slow-running,
@@ -212,8 +213,14 @@ def sanitize_submission():
 
 def send_checkin(server_url):
     logging.debug("Sending report")
-    sal_client = sal.get_sal_client()
-    return sal_client.post('checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    try:
+        response = sal.get_sal_client().post(
+            'checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to send report')
+        logging.debug(error)
+        response = None
+    return response
 
 
 def send_inventory(server_url, serial):
@@ -229,13 +236,22 @@ def send_inventory(server_url, serial):
         logging.debug(f"Inventory hash: {inventory_hash}")
         serverhash = None
         sal_client = sal.get_sal_client()
-        response = sal_client.get(f'inventory/hash/{serial}/')
+        try:
+            response = sal_client.get(f'inventory/hash/{serial}/')
+        except requests.exceptions.RequestException as error:
+            logging.error('Failed to get inventory hash')
+            logging.debug(error)
+            return
         if response.status_code == 200 and response.text != inventory_hash:
             logging.info("Inventory is out of date; submitting...")
             inventory_submission = {
                 'serial': serial,
                 'base64bz2inventory': sal.submission_encode(inventory)}
-            sal_client.post('inventory/submit/', data=inventory_submission)
+            try:
+                sal_client.post('inventory/submit/', data=inventory_submission)
+            except requests.exceptions.RequestException as error:
+                logging.error('Failed to submit inventory')
+                logging.debug(error)
 
 
 def send_catalogs(server_url, machine_group_key):
@@ -262,7 +278,9 @@ def send_catalogs(server_url, machine_group_key):
     sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
-    except:
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to get catalog hashes')
+        logging.debug(error)
         return
 
     try:
@@ -282,8 +300,9 @@ def send_catalogs(server_url, machine_group_key):
             logging.debug("Submitting Catalog: %s", catalog['name'])
             try:
                 sal_client.post(catalog_submit_url, data=catalog_submission)
-            except OSError:
-                logging.warning("Error while submitting Catalog: %s", catalog['name'])
+            except requests.exceptions.RequestException as error:
+                logging.error("Error while submitting Catalog: %s", catalog['name'])
+                logging.debug(error)
 
 
 def send_profiles(server_url, serial):
@@ -303,8 +322,11 @@ def send_profiles(server_url, serial):
     profile_out.unlink()
 
     profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
-    sal_client = sal.get_sal_client()
-    sal_client.post('profiles/submit/', data=profile_submission)
+    try:
+        sal.get_sal_client().post('profiles/submit/', data=profile_submission)
+    except requests.exceptions.RequestException as error:
+        logging.error('Failed to submit profiles')
+        logging.debug(error)
 
 
 if __name__ == "__main__":

From ab5706087bf4c3bfa3a636236aedc1fdb2edda65 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:36:11 -0400
Subject: [PATCH 58/79] Remove now unused arg.

---
 payload/usr/local/sal/bin/sal-submit | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index bcdb09b..f4029ab 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -49,12 +49,11 @@ def main():
     remove_skipped_facts()
     sanitize_submission()
 
-    server_url, _, machine_group_key = sal.get_server_prefs()
+    report = sal.get_checkin_results()
     if logging.getLogger().level <= 10:
-        submission = sal.get_checkin_results()
         logging.debug('Checkin submission:')
-        logging.debug(json.dumps(submission, indent=4, default=sal.serializer))
-    response = send_checkin(server_url)
+        logging.debug(json.dumps(report, indent=4, default=sal.serializer))
+    response = send_checkin(report)
 
     if response and response.status_code == 200:
         sal.clean_results()
@@ -62,9 +61,9 @@ def main():
     # Speed up manual runs by skipping these potentially slow-running,
     # and infrequently changing tasks.
     if run_type != 'manual':
-        send_inventory(server_url, submission['Machine']['extra_data']['serial'])
-        send_catalogs(server_url, machine_group_key)
-        send_profiles(server_url, submission['Machine']['extra_data']['serial'])
+        send_inventory(submission['Machine']['extra_data']['serial'])
+        send_catalogs(machine_group_key)
+        send_profiles(submission['Machine']['extra_data']['serial'])
 
     pathlib.Path('/Users/Shared/.com.salopensource.sal.run').unlink(missing_ok=True)
 
@@ -211,11 +210,10 @@ def sanitize_submission():
     sal.save_results(json.loads(submission_str))
 
 
-def send_checkin(server_url):
+def send_checkin(report):
     logging.debug("Sending report")
     try:
-        response = sal.get_sal_client().post(
-            'checkin/', json=json.loads(pathlib.Path(sal.RESULTS_PATH).read_text()))
+        response = sal.get_sal_client().post('checkin/', json=report)
     except requests.exceptions.RequestException as error:
         logging.error('Failed to send report')
         logging.debug(error)
@@ -223,7 +221,7 @@ def send_checkin(server_url):
     return response
 
 
-def send_inventory(server_url, serial):
+def send_inventory(serial):
     logging.info('Processing inventory...')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -254,9 +252,8 @@ def send_inventory(server_url, serial):
                 logging.debug(error)
 
 
-def send_catalogs(server_url, machine_group_key):
+def send_catalogs(machine_group_key):
     logging.info('Processing catalogs...')
-    catalog_submit_url = os.path.join(server_url, 'catalog/submit', '')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
         '/Library/Managed Installs')
@@ -299,13 +296,13 @@ def send_catalogs(server_url, machine_group_key):
 
             logging.debug("Submitting Catalog: %s", catalog['name'])
             try:
-                sal_client.post(catalog_submit_url, data=catalog_submission)
+                sal_client.post('catalog/submit/', data=catalog_submission)
             except requests.exceptions.RequestException as error:
                 logging.error("Error while submitting Catalog: %s", catalog['name'])
                 logging.debug(error)
 
 
-def send_profiles(server_url, serial):
+def send_profiles(serial):
     logging.info('Processing profiles...')
     temp_dir = tempfile.mkdtemp()
     profile_out = pathlib.Path(temp_dir) / 'profiles.plist'

From 010c7a1f25d99ccdc9e36161006223d072c61abb Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:37:59 -0400
Subject: [PATCH 59/79] Rethink how `[In|Unin]stallResults` get merged into
 `ManagedItem`s.

---
 .../sal/checkin_modules/munki_checkin.py      | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/payload/usr/local/sal/checkin_modules/munki_checkin.py b/payload/usr/local/sal/checkin_modules/munki_checkin.py
index e7c4b1b..7e7ba69 100755
--- a/payload/usr/local/sal/checkin_modules/munki_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/munki_checkin.py
@@ -80,24 +80,30 @@ def main():
         munki_submission['managed_items'][item] = submission_item
 
     # Process InstallResults and RemovalResults into update history
-    for report_key, result_type in (('InstallResults', 'PRESENT'), ('RemovalResults', 'ABSENT')):
+    for report_key in ('InstallResults', 'RemovalResults'):
         for item in munki_report.get(report_key, []):
             # Skip Apple software update items.
             if item.get('applesus'):
                 continue
-            history = {}
-            # history = {'update_type': 'apple' if item.get('applesus') else 'third_party'}
-            history['status'] = 'ERROR' if item.get('status') != 0 else result_type
+            # Construct key; we pop the name off because we don't need
+            # to submit it again when we stuff `item` into `data`.
+            name = f'{item.pop("name")} {item["version"]}'
+            submission_item = munki_submission['managed_items'].get(name, {'name': name})
+            if item.get('status') != 0:
+                # Something went wrong, so change the status.
+                submission_item['status'] = 'ERROR'
+            if 'data' in submission_item:
+                submission_item['data'].update(item)
+            else:
+                submission_item['data'] = item
+            if 'type' not in submission_item['data']:
+                submission_item['data']['type'] = (
+                    'ManagedInstalls' if report_key == 'InstallResults' else 'ManagedUninstalls')
             # This UTC datetime gets converted to a naive datetime by
             # plistlib. Fortunately, we can just tell it that it's UTC.
-            history['date_managed'] = item['time'].replace(
+            submission_item['date_managed'] = item['time'].replace(
                 tzinfo=datetime.timezone.utc).isoformat()
-            history['data'] = {'version': item.get('version', '0')}
-            # Add over top of any pending items we may have already built.
-            if item['name'] in munki_submission['managed_items']:
-                munki_submission['managed_items'][item['name']].update(history)
-            else:
-                munki_submission['managed_items'][item['name']] = history
+            munki_submission['managed_items'][name] = submission_item
 
     sal.set_checkin_results('Munki', munki_submission)
 

From bb0459c7c6df3d291da2dd44b9dcb0b13e6a301a Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:54:03 -0400
Subject: [PATCH 60/79] Add property for setting auth.

---
 sal_python_pkg/sal/client.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index db7e9f4..3fbdc96 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -15,7 +15,7 @@ class SalClient():
     base_url = ''
 
     def __init__(self):
-        sesh = macsesh.KeychainSession()
+        self.sesh = macsesh.KeychainSession()
         # sesh = macsesh.SecureTransportSession()
 
         base_url = pref('ServerURL')
@@ -23,20 +23,26 @@ def __init__(self):
 
         ca_cert = pref('CACert')
         if ca_cert:
-            sesh.verify = ca_cert
+            self.sesh.verify = ca_cert
 
         basic_auth = pref('BasicAuth')
         if basic_auth:
             key = pref('key', '')
-            sesh.auth = ('sal', key)
+            self.sesh.auth = ('sal', key)
 
         # TODO: Handle keychain-based certs.
-        ssl_client_cert = pref('SSLClientCertificate')
-        ssl_client_key = pref('SSLClientKey')
-        if ssl_client_cert:
-            sesh.cert = (ssl_client_cert, ssl_client_key) if ssl_client_key else ssl_client_cert
-
-        self.sesh = sesh
+        cert = pref('SSLClientCertificate')
+        key = pref('SSLClientKey')
+        if cert:
+            self.sesh.cert = (cert, key) if key else cert
+
+    @property
+    def auth(self):
+        return self.sesh.auth
+
+    @auth.setter
+    def auth(self, creds):
+        self.sesh.auth = creds
 
     def get(self, url):
         url = self.build_url(url)

From 9a5f00eaaa0c17ed02ac8f4da3f788cd3cc5fa4e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 11:54:37 -0400
Subject: [PATCH 61/79] Add commandline args to allow you to override ServerURL
 and key.

This is used primarily for debugging to a local Sal instance, or
submitting to a staging/test server.
---
 payload/usr/local/sal/bin/sal-submit | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index f4029ab..cccd31e 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -26,8 +26,9 @@ CHECKIN_MODULES_DIR = '/usr/local/sal/checkin_modules'
 
 
 def main():
+    args = get_args()
     logging.basicConfig(
-        level=get_log_level(), format='%(asctime)s %(levelname)s %(message)s')
+        level=get_log_level(args), format='%(asctime)s %(levelname)s %(message)s')
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
 
     exit_if_not_root()
@@ -50,6 +51,15 @@ def main():
     sanitize_submission()
 
     report = sal.get_checkin_results()
+    if args.url:
+        sal.get_sal_client().base_url = args.url
+        logging.debug('Server URL overridden with %s', args.url)
+
+    if args.key:
+        sesh = sal.get_sal_client().auth = ('sal', args.key)
+        # Override the key in the report, since it's used for querying.
+        report['Sal']['extra_data']['key'] = args.key
+        logging.debug('Machine group key overridden with %s', args.key)
     if logging.getLogger().level <= 10:
         logging.debug('Checkin submission:')
         logging.debug(json.dumps(report, indent=4, default=sal.serializer))
@@ -62,7 +72,7 @@ def main():
     # and infrequently changing tasks.
     if run_type != 'manual':
         send_inventory(submission['Machine']['extra_data']['serial'])
-        send_catalogs(machine_group_key)
+        send_catalogs()
         send_profiles(submission['Machine']['extra_data']['serial'])
 
     pathlib.Path('/Users/Shared/.com.salopensource.sal.run').unlink(missing_ok=True)
@@ -70,10 +80,9 @@ def main():
     logging.info('Checkin complete.')
 
 
-def get_log_level():
+def get_log_level(args):
     """Set the verbosity based on options."""
     loglevel = logging.CRITICAL
-    args = get_args()
     if args.debug:
         loglevel = logging.DEBUG
     elif args.verbose:
@@ -88,6 +97,8 @@ def get_args():
         "-d", "--debug", default=False, action="store_true", help="Enable full debug output.")
     parser.add_argument(
         "-v", "--verbose", default=False, action="store_true", help="Enable verbose output.")
+    parser.add_argument("-u", "--url", default=None, help="Override the server URL for testing.")
+    parser.add_argument("-k", "--key", default=None, help="Override the machine group key.")
     return parser.parse_args()
 
 
@@ -252,7 +263,7 @@ def send_inventory(serial):
                 logging.debug(error)
 
 
-def send_catalogs(machine_group_key):
+def send_catalogs():
     logging.info('Processing catalogs...')
     managed_install_dir = (
         CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
@@ -269,10 +280,11 @@ def send_catalogs(machine_group_key):
 
         catalog_check_plist = plistlib.dumps(check_list)
 
+    sal_client = sal.get_sal_client()
+    machine_group_key = sal_client.auth[1]
     hash_submission = {
         'key': machine_group_key,
         'catalogs': sal.submission_encode(catalog_check_plist)}
-    sal_client = sal.get_sal_client()
     try:
         response = sal_client.post('catalog/hash/', data=hash_submission)
     except requests.exceptions.RequestException as error:

From 6e87985b0da3c1051525271e66eaf436f9097f27 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 12:41:32 -0400
Subject: [PATCH 62/79] Add prefs debug output.

---
 payload/usr/local/sal/bin/sal-submit | 13 +++++++++++--
 sal_python_pkg/sal/utils.py          | 18 +++++++++++++++---
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index cccd31e..7598fab 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -27,9 +27,18 @@ CHECKIN_MODULES_DIR = '/usr/local/sal/checkin_modules'
 
 def main():
     args = get_args()
-    logging.basicConfig(
-        level=get_log_level(args), format='%(asctime)s %(levelname)s %(message)s')
+    log_level = get_log_level(args)
+    logging.basicConfig(level=log_level, format='%(asctime)s %(levelname)s %(message)s')
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
+    if log_level == logging.DEBUG:
+        logging.debug("Sal client prefs:")
+        prefs = sal.prefs()
+        if args.url:
+            prefs['ServerURL'] = {'value': args.url, 'forced': 'commandline'}
+        if args.key:
+            prefs['key'] = {'value': args.key, 'forced': 'commandline'}
+        for k, v in prefs.items():
+            logging.debug(f'\t{k}: {v["value"]} ({"profile" if v["forced"] else "prefs"})')
 
     exit_if_not_root()
     if sal.wait_for_script('sal-submit'):
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index c5534ce..ddc97f7 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -17,9 +17,10 @@
 import urllib.parse
 
 import macsesh
-from Foundation import (kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
-                        CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, NSDate, NSArray,
-                        NSDictionary, NSData, NSNull)
+from Foundation import (
+    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
+    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
+    NSArray, NSDictionary, NSData, NSNull)
 
 import sal.version
 
@@ -85,6 +86,17 @@ def pref(pref_name, default=None):
     return unobjctify(pref_value)
 
 
+def prefs():
+    prefs = (
+        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
+        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
+    return {k: {'value': pref(k), 'forced': forced(k)} for k in prefs}
+
+
+def forced(pref):
+    return CFPreferencesAppValueIsForced(pref, BUNDLE_ID)
+
+
 def wait_for_script(scriptname, repeat=3, pause=1):
     """Tries a few times to wait for a script to finish."""
     count = 0

From 7221070c64c2be6ba9859b609bbc00e090d610eb Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 8 May 2020 17:16:47 -0400
Subject: [PATCH 63/79] Do some processing client side so we don't send so much
 profile data.

---
 payload/usr/local/sal/bin/sal-submit | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 7598fab..86cad29 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -336,10 +336,15 @@ def send_profiles(serial):
         logging.warning("Couldn't output profiles.")
         return
 
-    profiles = sal.submission_encode(profile_out.read_bytes())
+    profiles = plistlib.loads(profile_out.read_bytes())
     profile_out.unlink()
-
-    profile_submission = {'serial': serial, 'base64bz2profiles': profiles}
+    # Drop all of the payload info we're not going to actual store.
+    for profile in profiles['_computerlevel']:
+        cleansed_payloads = [_payload_cleanse(p) for p in profile.get('ProfileItems', [])]
+        profile['ProfileItems'] = cleansed_payloads
+    logging.debug(profiles)
+    profile_submission = {
+        'serial': serial, 'base64bz2profiles': sal.submission_encode(plistlib.dumps(profiles))}
     try:
         sal.get_sal_client().post('profiles/submit/', data=profile_submission)
     except requests.exceptions.RequestException as error:
@@ -347,5 +352,10 @@ def send_profiles(serial):
         logging.debug(error)
 
 
+def _payload_cleanse(payload):
+    stored = ('PayloadIdentifier', 'PayloadUUID', 'PayloadType')
+    return {k: payload[k] for k in stored}
+
+
 if __name__ == "__main__":
     main()

From 80374711367499717a842bdfed7767adfa7ffe7d Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 16:32:04 -0400
Subject: [PATCH 64/79] Fix bug with downloading plugin scripts.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 92a2e56..ecb0ee1 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -65,7 +65,7 @@ def get_checksums():
     if response.status_code != 200:
         munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
         return
-    if response and "<h1>Page not found</h1>" not in response.text:
+    if response and "<h1>Page not found</h1>" in response.text:
         munkicommon.display_debug2(response.text)
         return
 

From 1fd780799ad7ad34b07f866097aee1235f84bcf8 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 18:09:10 -0400
Subject: [PATCH 65/79] Fix executable script check in utils.

---
 sal_python_pkg/sal/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index ddc97f7..6ac477e 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -229,7 +229,7 @@ def run_scripts(dir_path, cli_args=None, error=False):
     skip_names = {'__pycache__'}
     scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
     for script in scripts:
-        if script.stat().st_mode & stat.S_IWOTH:
+        if not os.access(script, os.X_OK):
             results.append(f"'{script}' is not executable or has bad permissions")
             continue
 

From d914a0bb9bf3ec35a79f00e602b125393547743d Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 18:13:05 -0400
Subject: [PATCH 66/79] Remove unused imports.

---
 sal_python_pkg/sal/utils.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 6ac477e..269dbc7 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -7,23 +7,17 @@
 import datetime
 import hashlib
 import json
-import logging
 import os
 import pathlib
 import plistlib
-import stat
 import subprocess
 import time
-import urllib.parse
 
-import macsesh
 from Foundation import (
     kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
     CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
     NSArray, NSDictionary, NSData, NSNull)
 
-import sal.version
-
 
 BUNDLE_ID = 'com.github.salopensource.sal'
 RESULTS_PATH = '/usr/local/sal/checkin_results.json'

From 8fefc394da64b189fdd9182104986386f830a0de Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:07:16 -0400
Subject: [PATCH 67/79] Move preference setting code out of client and bump
 macsesh version.

---
 sal_python_pkg/sal/__init__.py |  2 +-
 sal_python_pkg/sal/client.py   | 91 +++++++++++++++++++++++-----------
 sal_python_pkg/setup.py        |  4 ++
 3 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index b49c5eb..d06b704 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,3 @@
-from sal.client import SalClient, get_sal_client
+from sal.client import MacKeychainClient, SalClient, get_sal_client
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 3fbdc96..78e868f 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -1,6 +1,11 @@
 import logging
+import os
 
-import macsesh
+try:
+    from macsesh import Session as MacSeshSession
+except ImportError:
+    MacSeshSession = None
+import requests
 
 from sal.utils import pref
 
@@ -8,45 +13,67 @@
 _client_instance = None
 
 
-class SalClient():
+class SalClient:
 
+    session_class = requests.Session
+    _base_url = ''
+    _auth = None
+    _cert = None
+    _verify = None
     basic_timeout = (3.05, 4)
     post_timeout = (3.05, 8)
-    base_url = ''
 
     def __init__(self):
-        self.sesh = macsesh.KeychainSession()
-        # sesh = macsesh.SecureTransportSession()
-
-        base_url = pref('ServerURL')
-        self.base_url = base_url if not base_url.endswith('/') else base_url[:-1]
-
-        ca_cert = pref('CACert')
-        if ca_cert:
-            self.sesh.verify = ca_cert
-
-        basic_auth = pref('BasicAuth')
-        if basic_auth:
-            key = pref('key', '')
-            self.sesh.auth = ('sal', key)
+        self.create_session()
+
+    def create_session(self):
+        self.session = self.session_class()
+        if self.auth:
+            self.session.auth = self._auth
+        if self.cert:
+            self.session.cert = self._cert
+        if self.verify:
+            self.session.verify = self._verify
+
+            # self.session.cert = (self._cert, self._key) if self._key else self._cert
+    @property
+    def base_url(self):
+        return self._base_url
 
-        # TODO: Handle keychain-based certs.
-        cert = pref('SSLClientCertificate')
-        key = pref('SSLClientKey')
-        if cert:
-            self.sesh.cert = (cert, key) if key else cert
+    @base_url.setter
+    def base_url(self, base_url):
+        self._base_url = base_url if not base_url.endswith('/') else base_url[:-1]
 
     @property
     def auth(self):
-        return self.sesh.auth
+        return self._auth
 
     @auth.setter
     def auth(self, creds):
-        self.sesh.auth = creds
+        self._auth = creds
+        self.create_session()
+
+    @property
+    def cert(self):
+        return self._cert
+
+    @cert.setter
+    def cert(self, cert, key=None):
+        self._cert = (cert, key) if key else cert
+        self.create_session()
+
+    @property
+    def verify(self):
+        return self._verify
+
+    @verify.setter
+    def verify(self, path):
+        self._verify = path
+        self.create_session()
 
     def get(self, url):
         url = self.build_url(url)
-        return self.log_response(self.sesh.get(url, timeout=self.basic_timeout))
+        return self.log_response(self.session.get(url, timeout=self.basic_timeout))
 
     def post(self, url, data=None, json=None):
         url = self.build_url(url)
@@ -55,7 +82,7 @@ def post(self, url, data=None, json=None):
             kwargs['json'] = json
         else:
             kwargs['data'] = data
-        return self.log_response(self.sesh.post(url, **kwargs))
+        return self.log_response(self.session.post(url, **kwargs))
 
     def log_response(self, response):
         logging.debug(f'Response HTTP {response.status_code}: {response.text}')
@@ -67,8 +94,14 @@ def build_url(self, url):
         return '/'.join((self.base_url, url)) + '/'
 
 
-def get_sal_client():
+class MacKeychainClient(SalClient):
+
+    session_class = MacSeshSession
+
+
+def get_sal_client(with_client_class=None):
     global _client_instance
-    if _client_instance is None:
-        _client_instance = SalClient()
+    if _client_instance is None or (
+            with_client_class is not None and not isinstance(_client_instance, with_client_class)):
+        _client_instance =  with_client_class() if with_client_class is not None else SalClient()
     return _client_instance
diff --git a/sal_python_pkg/setup.py b/sal_python_pkg/setup.py
index 797fa57..3af1887 100644
--- a/sal_python_pkg/setup.py
+++ b/sal_python_pkg/setup.py
@@ -10,4 +10,8 @@
     name='sal',
     version=namespace['__version__'],
     description='Sal client utilities',
+    install_requires=[
+        'pyobjc == 6.2 ; platform_system=="Darwin"',
+        'macsesh == 0.3.0 ; platform_system=="Darwin"',
+        'requests >= 2.23.0'],
     packages=['sal'])
\ No newline at end of file

From 01bcc00e8802bdebae6bec8fb6dcd2c0c84701cb Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:09:32 -0400
Subject: [PATCH 68/79] Use new session name.

---
 payload/usr/local/sal/checkin_modules/machine_checkin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index 173b08d..062e8ae 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -117,7 +117,7 @@ def get_model_code(serial):
 def query_apple_support(serial):
     model_code = get_model_code(serial)
     tree = ElementTree.ElementTree()
-    session = macsesh.KeychainSession()
+    session = macsesh.Session()
     response = session.get(f"https://support-sp.apple.com/sp/product?cc={model_code}&lang=en_US")
     try:
         tree = ElementTree.fromstring(response.text)

From df59a6ecaecb801b1c2ed087bcf1e6b4e85dd66d Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:30:52 -0400
Subject: [PATCH 69/79] Bump python version.

---
 build_python_framework.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_python_framework.sh b/build_python_framework.sh
index 786aa28..835bb04 100755
--- a/build_python_framework.sh
+++ b/build_python_framework.sh
@@ -1,7 +1,7 @@
 #!/bin/zsh
 # Build script for Python 3 framework for Sal scripts
 TOOLSDIR=$(dirname "$0")
-PYTHON_VERSION=3.8.2
+PYTHON_VERSION=3.8.3
 
 # build the framework
 /tmp/relocatable-python-git/make_relocatable_python_framework.py \

From 5550d0f0602d80c6cf3e0a41df38be7404d999e3 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:08 -0400
Subject: [PATCH 70/79] Add pref-based client setup code into new mac_utils
 module.

---
 sal_python_pkg/sal/__init__.py  |  1 +
 sal_python_pkg/sal/mac_utils.py | 44 +++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 sal_python_pkg/sal/mac_utils.py

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index d06b704..7591db4 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,3 +1,4 @@
 from sal.client import MacKeychainClient, SalClient, get_sal_client
+from sal.mac_utils import *
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
new file mode 100644
index 0000000..5734c3f
--- /dev/null
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -0,0 +1,44 @@
+import logging
+import os
+
+from sal.client import get_sal_client, MacKeychainClient
+from sal.utils import pref
+
+
+def setup_sal_client():
+    ca_cert = pref('CACert', '')
+    cert = pref('SSLClientCertificate', '')
+    key = pref('SSLClientKey', '')
+    exists = map(os.path.exists, (ca_cert, cert, key))
+    if any(exists):
+        if not all(exists):
+            logging.warning(
+                'Argument warning! If using the `CACert`, `SSLClientCertificate`, or '
+                '`SSLClientKey` prefs, they must all be either paths to cert files or the '
+                'common name of the certs to find in the keychain.')
+
+        # If any of the above have been passed as a path, we have to
+        # use a vanilla Session.
+        logging.debug('Using SalClient')
+        client = get_sal_client()
+    else:
+        # Assume that any passed certs are by CN since they don't
+        # exist as files anywhere.
+        # If we're going to use the keychain, we need to use a
+        # macsesh
+        logging.debug('Using MacKeychainClient')
+        client = get_sal_client(MacKeychainClient)
+
+    if ca_cert:
+        client.verify = ca_cert
+    if cert:
+        client.cert = (cert, key) if key else cert
+
+    basic_auth = pref('BasicAuth')
+    if basic_auth:
+        key = pref('key', '')
+        client.auth = ('sal', key)
+
+    client.base_url = pref('ServerURL')
+
+

From 58d56bcdcf598bc584ad1db8b4bf1dd0cfbf7e99 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:33 -0400
Subject: [PATCH 71/79] Use newer macsesh.

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d9634ba..f0e2ec0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ sal_python_pkg/
 pyobjc==6.2.2
 urllib3==1.25.10
 requests==2.23.0
-MacSesh==0.2.1
+MacSesh==0.3.0

From 681ee58935611299924f46239ec7d3fe982d44a7 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:31:38 -0400
Subject: [PATCH 72/79] Use client setup util.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 1 +
 payload/usr/local/sal/bin/sal-submit              | 1 +
 2 files changed, 2 insertions(+)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index ecb0ee1..03a40ae 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -23,6 +23,7 @@ EXTERNAL_SCRIPTS_DIR = '/usr/local/sal/external_scripts'
 
 def main():
     set_verbosity()
+    sal.setup_sal_client()
 
     if sal.pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 86cad29..871ab6d 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -60,6 +60,7 @@ def main():
     sanitize_submission()
 
     report = sal.get_checkin_results()
+    sal.setup_sal_client()
     if args.url:
         sal.get_sal_client().base_url = args.url
         logging.debug('Server URL overridden with %s', args.url)

From f03b92c2f52f26800bf2fa95ba6f2b7a9661e542 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:45:00 -0400
Subject: [PATCH 73/79] Create generic pref getting mac util.

---
 sal_python_pkg/sal/__init__.py  | 6 +++++-
 sal_python_pkg/sal/mac_utils.py | 5 +++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/sal_python_pkg/sal/__init__.py b/sal_python_pkg/sal/__init__.py
index 7591db4..2e584bd 100644
--- a/sal_python_pkg/sal/__init__.py
+++ b/sal_python_pkg/sal/__init__.py
@@ -1,4 +1,8 @@
 from sal.client import MacKeychainClient, SalClient, get_sal_client
-from sal.mac_utils import *
+try:
+    from sal.mac_utils import *
+except ImportError:
+    # Allow non-macOS to import safely.
+    pass
 from sal.utils import *
 from sal.version import __version__
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index 5734c3f..47ea805 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,6 +1,8 @@
 import logging
 import os
 
+from Foundation import CFPreferencesCopyAppValue
+
 from sal.client import get_sal_client, MacKeychainClient
 from sal.utils import pref
 
@@ -42,3 +44,6 @@ def setup_sal_client():
     client.base_url = pref('ServerURL')
 
 
+def mac_pref(domain, key, default=None):
+    val = CFPreferencesCopyAppValue(key, domain)
+    return val if val is not None else default

From 08d68426c3f941ab4c255f34bdeba82a9017d787 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 19:46:40 -0400
Subject: [PATCH 74/79] Use mac_pref util in sal-submit.

---
 payload/usr/local/sal/bin/sal-submit | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index 871ab6d..c772642 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -16,7 +16,6 @@ import stat
 import subprocess
 import tempfile
 
-from Foundation import CFPreferencesCopyAppValue
 import requests.exceptions
 
 import sal
@@ -244,9 +243,8 @@ def send_checkin(report):
 
 def send_inventory(serial):
     logging.info('Processing inventory...')
-    managed_install_dir = (
-        CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
-        '/Library/Managed Installs')
+    managed_install_dir = sal.mac_pref(
+        'ManagedInstalls', 'ManagedInstallDir', '/Library/Managed Installs')
     inventory_plist = pathlib.Path(managed_install_dir) / 'ApplicationInventory.plist'
     logging.debug('ApplicationInventory.plist Path: %s', inventory_plist)
 
@@ -275,9 +273,8 @@ def send_inventory(serial):
 
 def send_catalogs():
     logging.info('Processing catalogs...')
-    managed_install_dir = (
-        CFPreferencesCopyAppValue('ManagedInstallDir', 'ManagedInstalls') or
-        '/Library/Managed Installs')
+    managed_install_dir = sal.mac_pref(
+        'ManagedInstalls', 'ManagedInstallDir', '/Library/Managed Installs')
     catalog_dir = pathlib.Path(managed_install_dir) / 'catalogs'
 
     check_list = []

From 040626f95f7344d13255067eb75fcc8ee5be675e Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 20:23:01 -0400
Subject: [PATCH 75/79] Move Mac-specific sal pkg code to a dedicated mac_utils
 module.

---
 .../local/munki/postflight.d/sal-postflight   |   8 +-
 .../usr/local/munki/preflight.d/sal-preflight |   6 +-
 payload/usr/local/sal/bin/sal-submit          |   6 +-
 .../sal/checkin_modules/machine_checkin.py    |   2 +-
 .../local/sal/checkin_modules/sal_checkin.py  |   3 +-
 sal_python_pkg/sal/client.py                  |   2 -
 sal_python_pkg/sal/mac_utils.py               | 139 ++++++++++++++++-
 sal_python_pkg/sal/utils.py                   | 143 ------------------
 8 files changed, 143 insertions(+), 166 deletions(-)

diff --git a/payload/usr/local/munki/postflight.d/sal-postflight b/payload/usr/local/munki/postflight.d/sal-postflight
index e41baa8..a5043a3 100644
--- a/payload/usr/local/munki/postflight.d/sal-postflight
+++ b/payload/usr/local/munki/postflight.d/sal-postflight
@@ -57,21 +57,21 @@ def check_server_connection():
 
 def check_server_online():
     # is the offline report pref true?
-    if not sal.pref('SendOfflineReport'):
+    if not sal.sal_pref('SendOfflineReport'):
         return
     # read report
     report = munki_checkin.get_managed_install_report()
     # check for errors and warnings
     if not check_for_errors(report):
-        sal.set_pref('LastRunWasOffline', False)
+        sal.set_sal_pref('LastRunWasOffline', False)
         return
     # if they're there check is server is really offline
     if check_server_connection():
-        sal.set_pref('LastRunWasOffline', True)
+        sal.set_sal_pref('LastRunWasOffline', True)
         return
 
     # If we get here, it's online
-    sal.set_pref('LastRunWasOffline', False)
+    sal.set_sal_pref('LastRunWasOffline', False)
 
 def write_touch_file():
     if os.path.exists(TOUCH_FILE_PATH):
diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 03a40ae..6bde9f7 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -25,7 +25,7 @@ def main():
     set_verbosity()
     sal.setup_sal_client()
 
-    if sal.pref('SyncScripts') == True:
+    if sal.sal_pref('SyncScripts') == True:
         if not os.path.exists(EXTERNAL_SCRIPTS_DIR):
             os.makedirs(EXTERNAL_SCRIPTS_DIR)
         server_scripts = get_checksums()
@@ -40,8 +40,8 @@ def main():
 def get_prefs():
     # Check for mandatory prefs and bail if any are missing.
     required_prefs = {}
-    required_prefs["key"] = sal.pref('key')
-    required_prefs["ServerURL"] = sal.pref('ServerURL').rstrip('/')
+    required_prefs["key"] = sal.sal_pref('key')
+    required_prefs["ServerURL"] = sal.sal_pref('ServerURL').rstrip('/')
 
     for key, val in required_prefs.items():
         if not val:
diff --git a/payload/usr/local/sal/bin/sal-submit b/payload/usr/local/sal/bin/sal-submit
index c772642..8c85b11 100755
--- a/payload/usr/local/sal/bin/sal-submit
+++ b/payload/usr/local/sal/bin/sal-submit
@@ -31,7 +31,7 @@ def main():
     logging.info("%s Version: %s", os.path.basename(__file__), sal.__version__)
     if log_level == logging.DEBUG:
         logging.debug("Sal client prefs:")
-        prefs = sal.prefs()
+        prefs = sal.prefs_report()
         if args.url:
             prefs['ServerURL'] = {'value': args.url, 'forced': 'commandline'}
         if args.key:
@@ -168,7 +168,7 @@ def get_plugin_results(plugin_results_plist):
 
 
 def remove_blacklisted_messages():
-    patterns = sal.pref('MessageBlacklistPatterns', [])
+    patterns = sal.sal_pref('MessageBlacklistPatterns', [])
     if patterns:
         compiled = [re.compile(p) for p in patterns]
         update = False
@@ -196,7 +196,7 @@ def remove_blacklisted_messages():
 
 
 def remove_skipped_facts():
-    if skip_facts := sal.pref('SkipFacts'):
+    if skip_facts := sal.sal_pref('SkipFacts'):
         update = False
         submission = sal.get_checkin_results()
 
diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index 062e8ae..d4ec549 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -68,7 +68,7 @@ def process_system_profile():
 
 
 def get_hostname():
-    _, name_type, _ = sal.get_server_prefs()
+    name_type = sal.sal_pref('NameType', default='ComputerName')
     net_config = SCDynamicStoreCreate(None, "net", None, None)
     return get_machine_name(net_config, name_type)
 
diff --git a/payload/usr/local/sal/checkin_modules/sal_checkin.py b/payload/usr/local/sal/checkin_modules/sal_checkin.py
index f273d75..72e6638 100755
--- a/payload/usr/local/sal/checkin_modules/sal_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/sal_checkin.py
@@ -8,11 +8,10 @@
 
 
 def main():
-    _, _, bu_key = sal.get_server_prefs()
     sal_submission = {
         'extra_data': {
             'sal_version': sal.__version__,
-            'key': bu_key,},
+            'key': sal.sal_pref('key'),},
         'facts': {'checkin_module_version': __version__}}
     sal.set_checkin_results('Sal', sal_submission)
 
diff --git a/sal_python_pkg/sal/client.py b/sal_python_pkg/sal/client.py
index 78e868f..cae3fd1 100644
--- a/sal_python_pkg/sal/client.py
+++ b/sal_python_pkg/sal/client.py
@@ -7,8 +7,6 @@
     MacSeshSession = None
 import requests
 
-from sal.utils import pref
-
 
 _client_instance = None
 
diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index 47ea805..f774040 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,16 +1,23 @@
+import datetime
 import logging
 import os
 
-from Foundation import CFPreferencesCopyAppValue
+from Foundation import (
+    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
+    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
+    NSArray, NSDictionary, NSData, NSNull)
 
 from sal.client import get_sal_client, MacKeychainClient
-from sal.utils import pref
+
+
+BUNDLE_ID = 'com.github.salopensource.sal'
+ISO_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
 
 
 def setup_sal_client():
-    ca_cert = pref('CACert', '')
-    cert = pref('SSLClientCertificate', '')
-    key = pref('SSLClientKey', '')
+    ca_cert = sal_pref('CACert', '')
+    cert = sal_pref('SSLClientCertificate', '')
+    key = sal_pref('SSLClientKey', '')
     exists = map(os.path.exists, (ca_cert, cert, key))
     if any(exists):
         if not all(exists):
@@ -36,14 +43,130 @@ def setup_sal_client():
     if cert:
         client.cert = (cert, key) if key else cert
 
-    basic_auth = pref('BasicAuth')
+    basic_auth = sal_pref('BasicAuth')
     if basic_auth:
-        key = pref('key', '')
+        key = sal_pref('key', '')
         client.auth = ('sal', key)
 
-    client.base_url = pref('ServerURL')
+    client.base_url = sal_pref('ServerURL')
 
 
 def mac_pref(domain, key, default=None):
     val = CFPreferencesCopyAppValue(key, domain)
     return val if val is not None else default
+
+
+def set_sal_pref(pref_name, pref_value):
+    """Sets a Sal preference.
+
+    The preference file on disk is located at
+    /Library/Preferences/com.github.salopensource.sal.plist.  This should
+    normally be used only for 'bookkeeping' values; values that control
+    the behavior of munki may be overridden elsewhere (by MCX, for
+    example)
+    """
+    try:
+        CFPreferencesSetValue(
+            pref_name, pref_value, BUNDLE_ID, kCFPreferencesAnyUser, kCFPreferencesCurrentHost)
+        CFPreferencesAppSynchronize(BUNDLE_ID)
+    except Exception:
+        pass
+
+
+def sal_pref(pref_name, default=None):
+    """Return a preference value.
+
+    Since this uses CFPreferencesCopyAppValue, Preferences can be defined
+    several places. Precedence is:
+        - MCX
+        - /var/root/Library/Preferences/com.github.salopensource.sal.plist
+        - /Library/Preferences/com.github.salopensource.sal.plist
+        - default_prefs defined here.
+
+    Returned values are all converted to native python types through the
+    `unobjctify` function; e.g. dates are returned as aware-datetimes,
+    NSDictionary to dict, etc.
+    """
+    default_prefs = {
+        'ServerURL': 'http://sal',
+        'osquery_launchd': 'com.facebook.osqueryd.plist',
+        'SkipFacts': [],
+        'SyncScripts': True,
+        'BasicAuth': True,
+        'GetGrains': False,
+        'GetOhai': False,
+        'LastRunWasOffline': False,
+        'SendOfflineReport': False,
+    }
+
+    pref_value = mac_pref(BUNDLE_ID, pref_name, default)
+    if pref_value is None and pref_name in default_prefs:
+        # If we got here, the pref value was either set to None or never
+        # set, AND the default was also None. Fall back to auto prefs.
+        pref_value = default_prefs.get(pref_name)
+        # we're using a default value. We'll write it out to
+        # /Library/Preferences/<BUNDLE_ID>.plist for admin
+        # discoverability
+        set_sal_pref(pref_name, pref_value)
+
+    return unobjctify(pref_value)
+
+
+def forced(pref, bundle_identifier=BUNDLE_ID):
+    return CFPreferencesAppValueIsForced(pref, bundle_identifier)
+
+
+def prefs_report():
+    prefs = (
+        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
+        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
+    return {k: {'value': sal_pref(k), 'forced': forced(k)} for k in prefs}
+
+
+def unobjctify(element, safe=False):
+    """Recursively convert nested elements to native python datatypes.
+
+    Types accepted include str, bytes, int, float, bool, None, list,
+    dict, set, tuple, NSArray, NSDictionary, NSData, NSDate, NSNull.
+
+    element: Some (potentially) nested data you want to convert.
+
+    safe: Bool (defaults to False) whether you want printable
+        representations instead of the python equivalent. e.g.  NSDate
+        safe=True becomes a str, safe=False becomes a datetime.datetime.
+        NSData safe=True bcomes a hex str, safe=False becomes bytes. Any
+        type not explicitly handled by this module will raise an
+        exception unless safe=True, where it will instead replace the
+        data with a str of '<UNSUPPORTED TYPE>'
+
+        This is primarily for safety in serialization to plists or
+        output.
+
+    returns: Python equivalent of the original input.
+        e.g. NSArray -> List, NSDictionary -> Dict, etc.
+
+    raises: ValueError for any data that isn't supported (yet!) by this
+        function.
+    """
+    supported_types = (str, bytes, int, float, bool, datetime.datetime)
+    if isinstance(element, supported_types):
+        return element
+    elif isinstance(element, (dict, NSDictionary)):
+        return {k: unobjctify(v, safe=safe) for k, v in element.items()}
+    elif isinstance(element, (list, NSArray)):
+        return [unobjctify(i, safe=safe) for i in element]
+    elif isinstance(element, set):
+        return set([unobjctify(i, safe=safe) for i in element])
+    elif isinstance(element, tuple):
+        return tuple([unobjctify(i, safe=safe) for i in element])
+    elif isinstance(element, NSData):
+        return binascii.hexlify(element) if safe else bytes(element)
+    elif isinstance(element, NSDate):
+        return str(element) if safe else datetime.datetime.strptime(
+            element.description(), ISO_TIME_FORMAT)
+    elif isinstance(element, NSNull) or element is None:
+        return '' if safe else None
+    elif safe:
+        return '<UNSUPPORTED TYPE>'
+    raise ValueError(f"Element type '{type(element)}' is not supported!")
+
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 269dbc7..6296e62 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -13,82 +13,8 @@
 import subprocess
 import time
 
-from Foundation import (
-    kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
-    CFPreferencesAppSynchronize, CFPreferencesCopyAppValue, CFPreferencesAppValueIsForced, NSDate,
-    NSArray, NSDictionary, NSData, NSNull)
 
-
-BUNDLE_ID = 'com.github.salopensource.sal'
 RESULTS_PATH = '/usr/local/sal/checkin_results.json'
-ISO_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
-
-
-def set_pref(pref_name, pref_value):
-    """Sets a Sal preference.
-
-    The preference file on disk is located at
-    /Library/Preferences/com.github.salopensource.sal.plist.  This should
-    normally be used only for 'bookkeeping' values; values that control
-    the behavior of munki may be overridden elsewhere (by MCX, for
-    example)
-    """
-    try:
-        CFPreferencesSetValue(
-            pref_name, pref_value, BUNDLE_ID, kCFPreferencesAnyUser, kCFPreferencesCurrentHost)
-        CFPreferencesAppSynchronize(BUNDLE_ID)
-    except Exception:
-        pass
-
-
-def pref(pref_name, default=None):
-    """Return a preference value.
-
-    Since this uses CFPreferencesCopyAppValue, Preferences can be defined
-    several places. Precedence is:
-        - MCX
-        - /var/root/Library/Preferences/com.github.salopensource.sal.plist
-        - /Library/Preferences/com.github.salopensource.sal.plist
-        - default_prefs defined here.
-
-    Returned values are all converted to native python types through the
-    `unobjctify` function; e.g. dates are returned as aware-datetimes,
-    NSDictionary to dict, etc.
-    """
-    default_prefs = {
-        'ServerURL': 'http://sal',
-        'osquery_launchd': 'com.facebook.osqueryd.plist',
-        'SkipFacts': [],
-        'SyncScripts': True,
-        'BasicAuth': True,
-        'GetGrains': False,
-        'GetOhai': False,
-        'LastRunWasOffline': False,
-        'SendOfflineReport': False,
-    }
-
-    pref_value = CFPreferencesCopyAppValue(pref_name, BUNDLE_ID)
-    if pref_value is None and default is not None:
-        pref_value = default
-    elif pref_value is None and pref_name in default_prefs:
-        pref_value = default_prefs.get(pref_name)
-        # we're using a default value. We'll write it out to
-        # /Library/Preferences/<BUNDLE_ID>.plist for admin
-        # discoverability
-        set_pref(pref_name, pref_value)
-
-    return unobjctify(pref_value)
-
-
-def prefs():
-    prefs = (
-        'ServerURL', 'key', 'BasicAuth', 'SyncScripts', 'SkipFacts', 'CACert', 'SendOfflineReport',
-        'SSLClientCertificate', 'SSLClientKey', 'MessageBlacklistPatterns')
-    return {k: {'value': pref(k), 'forced': forced(k)} for k in prefs}
-
-
-def forced(pref):
-    return CFPreferencesAppValueIsForced(pref, BUNDLE_ID)
 
 
 def wait_for_script(scriptname, repeat=3, pause=1):
@@ -243,75 +169,6 @@ def run_scripts(dir_path, cli_args=None, error=False):
     return results
 
 
-def get_server_prefs():
-    """Get Sal preferences, bailing if required info is missing.
-
-    Returns:
-        Tuple of (Server URL, NameType, and key (business unit key)
-    """
-    # Check for mandatory prefs and bail if any are missing.
-    required_prefs = {
-        'key': pref('key'),
-        'server_url': pref('ServerURL').rstrip('/')}
-
-    for key, val in required_prefs.items():
-        if not val:
-            exit(f'Required Sal preference "{key}" is not set.')
-
-    # Get optional preferences.
-    name_type = pref('NameType', default='ComputerName')
-
-    return required_prefs["server_url"], name_type, required_prefs["key"]
-
-
-def unobjctify(element, safe=False):
-    """Recursively convert nested elements to native python datatypes.
-
-    Types accepted include str, bytes, int, float, bool, None, list,
-    dict, set, tuple, NSArray, NSDictionary, NSData, NSDate, NSNull.
-
-    element: Some (potentially) nested data you want to convert.
-
-    safe: Bool (defaults to False) whether you want printable
-        representations instead of the python equivalent. e.g.  NSDate
-        safe=True becomes a str, safe=False becomes a datetime.datetime.
-        NSData safe=True bcomes a hex str, safe=False becomes bytes. Any
-        type not explicitly handled by this module will raise an
-        exception unless safe=True, where it will instead replace the
-        data with a str of '<UNSUPPORTED TYPE>'
-
-        This is primarily for safety in serialization to plists or
-        output.
-
-    returns: Python equivalent of the original input.
-        e.g. NSArray -> List, NSDictionary -> Dict, etc.
-
-    raises: ValueError for any data that isn't supported (yet!) by this
-        function.
-    """
-    supported_types = (str, bytes, int, float, bool, datetime.datetime)
-    if isinstance(element, supported_types):
-        return element
-    elif isinstance(element, (dict, NSDictionary)):
-        return {k: unobjctify(v, safe=safe) for k, v in element.items()}
-    elif isinstance(element, (list, NSArray)):
-        return [unobjctify(i, safe=safe) for i in element]
-    elif isinstance(element, set):
-        return set([unobjctify(i, safe=safe) for i in element])
-    elif isinstance(element, tuple):
-        return tuple([unobjctify(i, safe=safe) for i in element])
-    elif isinstance(element, NSData):
-        return binascii.hexlify(element) if safe else bytes(element)
-    elif isinstance(element, NSDate):
-        return str(element) if safe else datetime.datetime.strptime(
-            element.description(), ISO_TIME_FORMAT)
-    elif isinstance(element, NSNull) or element is None:
-        return '' if safe else None
-    elif safe:
-        return '<UNSUPPORTED TYPE>'
-    raise ValueError(f"Element type '{type(element)}' is not supported!")
-
-
 def submission_encode(data: bytes) -> bytes:
     """Return a b64 encoded, bz2 compressed copy of text."""
     return base64.b64encode(bz2.compress(data))

From f789c1630a87993679197d3f3317db5551428ad3 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Wed, 27 May 2020 21:02:39 -0400
Subject: [PATCH 76/79] Move more mac-specific code to mac_utils.

---
 sal_python_pkg/sal/mac_utils.py | 80 ++++++++++++++++++++++++++++++-
 sal_python_pkg/sal/utils.py     | 85 +++------------------------------
 2 files changed, 85 insertions(+), 80 deletions(-)

diff --git a/sal_python_pkg/sal/mac_utils.py b/sal_python_pkg/sal/mac_utils.py
index f774040..4acfe76 100644
--- a/sal_python_pkg/sal/mac_utils.py
+++ b/sal_python_pkg/sal/mac_utils.py
@@ -1,6 +1,10 @@
+import binascii
 import datetime
 import logging
 import os
+import pathlib
+import subprocess
+import time
 
 from Foundation import (
     kCFPreferencesAnyUser, kCFPreferencesCurrentHost, CFPreferencesSetValue,
@@ -156,7 +160,7 @@ def unobjctify(element, safe=False):
     elif isinstance(element, (list, NSArray)):
         return [unobjctify(i, safe=safe) for i in element]
     elif isinstance(element, set):
-        return set([unobjctify(i, safe=safe) for i in element])
+        return {unobjctify(i, safe=safe) for i in element}
     elif isinstance(element, tuple):
         return tuple([unobjctify(i, safe=safe) for i in element])
     elif isinstance(element, NSData):
@@ -170,3 +174,77 @@ def unobjctify(element, safe=False):
         return '<UNSUPPORTED TYPE>'
     raise ValueError(f"Element type '{type(element)}' is not supported!")
 
+
+def script_is_running(scriptname):
+    """Returns Process ID for a running python script.
+
+    Not at all stolen from Munki. Honest.
+    """
+    cmd = ['/bin/ps', '-eo', 'pid=,command=']
+    proc = subprocess.Popen(
+        cmd, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    out, _ = proc.communicate()
+    mypid = os.getpid()
+    for line in out.splitlines():
+        try:
+            pid, process = line.split(maxsplit=1)
+        except ValueError:
+            # funky process line, so we'll skip it
+            pass
+        else:
+            args = process.split()
+            try:
+                # first look for Python processes
+                if 'MacOS/Python' in args[0] or 'python' in args[0]:
+                    # look for first argument being scriptname
+                    if scriptname in args[1]:
+                        try:
+                            if int(pid) != mypid:
+                                return True
+                        except ValueError:
+                            # pid must have some funky characters
+                            pass
+            except IndexError:
+                pass
+
+    # if we get here we didn't find a Python script with scriptname
+    # (other than ourselves)
+    return False
+
+
+def run_scripts(dir_path, cli_args=None, error=False):
+    results = []
+    skip_names = {'__pycache__'}
+    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
+    for script in scripts:
+        if not os.access(script, os.X_OK):
+            results.append(f"'{script}' is not executable or has bad permissions")
+            continue
+
+        cmd = [script]
+        if cli_args:
+            cmd.append(cli_args)
+        try:
+            subprocess.check_call(cmd)
+            results.append(f"'{script}' ran successfully")
+        except (OSError, subprocess.CalledProcessError):
+            errormsg = f"'{script}' had errors during execution!"
+            if not error:
+                results.append(errormsg)
+            else:
+                raise RuntimeError(errormsg)
+
+    return results
+
+
+def wait_for_script(scriptname, repeat=3, pause=1):
+    """Tries a few times to wait for a script to finish."""
+    count = 0
+    while count < repeat:
+        if script_is_running(scriptname):
+            time.sleep(pause)
+            count += 1
+        else:
+            return False
+    return True
+
diff --git a/sal_python_pkg/sal/utils.py b/sal_python_pkg/sal/utils.py
index 6296e62..51eaa8a 100644
--- a/sal_python_pkg/sal/utils.py
+++ b/sal_python_pkg/sal/utils.py
@@ -2,68 +2,17 @@
 
 
 import base64
-import binascii
 import bz2
 import datetime
 import hashlib
 import json
 import os
+import platform
 import pathlib
 import plistlib
-import subprocess
-import time
 
 
-RESULTS_PATH = '/usr/local/sal/checkin_results.json'
-
-
-def wait_for_script(scriptname, repeat=3, pause=1):
-    """Tries a few times to wait for a script to finish."""
-    count = 0
-    while count < repeat:
-        if script_is_running(scriptname):
-            time.sleep(pause)
-            count += 1
-        else:
-            return False
-    return True
-
-
-def script_is_running(scriptname):
-    """Returns Process ID for a running python script.
-
-    Not at all stolen from Munki. Honest.
-    """
-    cmd = ['/bin/ps', '-eo', 'pid=,command=']
-    proc = subprocess.Popen(
-        cmd, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    out, _ = proc.communicate()
-    mypid = os.getpid()
-    for line in out.splitlines():
-        try:
-            pid, process = line.split(maxsplit=1)
-        except ValueError:
-            # funky process line, so we'll skip it
-            pass
-        else:
-            args = process.split()
-            try:
-                # first look for Python processes
-                if 'MacOS/Python' in args[0] or 'python' in args[0]:
-                    # look for first argument being scriptname
-                    if scriptname in args[1]:
-                        try:
-                            if int(pid) != mypid:
-                                return True
-                        except ValueError:
-                            # pid must have some funky characters
-                            pass
-            except IndexError:
-                pass
-
-    # if we get here we didn't find a Python script with scriptname
-    # (other than ourselves)
-    return False
+RESULTS_PATH = {'Darwin': '/usr/local/sal/checkin_results.json'}.get(platform.system())
 
 
 def get_hash(file_path):
@@ -86,7 +35,10 @@ def add_plugin_results(plugin, data, historical=False):
         historical (bool): Whether to keep only one record (False) or
             all results (True). Optional, defaults to False.
     """
-    plist_path = pathlib.Path('/usr/local/sal/plugin_results.plist')
+    if platform.system() == 'Darwin':
+        plist_path = pathlib.Path('/usr/local/sal/plugin_results.plist')
+    else:
+        raise NotImplementedError('Please PR a plugin results path for your platform!')
     if plist_path.exists():
         plugin_results = plistlib.loads(plist_path.read_bytes())
     else:
@@ -144,31 +96,6 @@ def serializer(obj):
     return obj
 
 
-def run_scripts(dir_path, cli_args=None, error=False):
-    results = []
-    skip_names = {'__pycache__'}
-    scripts = (p for p in pathlib.Path(dir_path).iterdir() if p.name not in skip_names)
-    for script in scripts:
-        if not os.access(script, os.X_OK):
-            results.append(f"'{script}' is not executable or has bad permissions")
-            continue
-
-        cmd = [script]
-        if cli_args:
-            cmd.append(cli_args)
-        try:
-            subprocess.check_call(cmd)
-            results.append(f"'{script}' ran successfully")
-        except (OSError, subprocess.CalledProcessError):
-            errormsg = f"'{script}' had errors during execution!"
-            if not error:
-                results.append(errormsg)
-            else:
-                raise RuntimeError(errormsg)
-
-    return results
-
-
 def submission_encode(data: bytes) -> bytes:
     """Return a b64 encoded, bz2 compressed copy of text."""
     return base64.b64encode(bz2.compress(data))

From 2ebce5065f05ae4d9ea5c393818a989c1295e49d Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@users.noreply.github.com>
Date: Sat, 30 May 2020 15:21:28 -0400
Subject: [PATCH 77/79] Fix missing import.

---
 payload/usr/local/sal/checkin_modules/machine_checkin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/payload/usr/local/sal/checkin_modules/machine_checkin.py b/payload/usr/local/sal/checkin_modules/machine_checkin.py
index d4ec549..2784abc 100755
--- a/payload/usr/local/sal/checkin_modules/machine_checkin.py
+++ b/payload/usr/local/sal/checkin_modules/machine_checkin.py
@@ -8,6 +8,7 @@
 import sys
 from xml.etree import ElementTree
 
+import macsesh
 from SystemConfiguration import (
     SCDynamicStoreCreate, SCDynamicStoreCopyValue, SCDynamicStoreCopyConsoleUser)
 

From f5c200332f9aac7f2e1426ab08fbf8d385aea123 Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Mon, 13 Jul 2020 17:04:05 -0400
Subject: [PATCH 78/79] Use status code objects rather than int literal.

---
 payload/usr/local/munki/preflight.d/sal-preflight | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/payload/usr/local/munki/preflight.d/sal-preflight b/payload/usr/local/munki/preflight.d/sal-preflight
index 6bde9f7..fe53676 100755
--- a/payload/usr/local/munki/preflight.d/sal-preflight
+++ b/payload/usr/local/munki/preflight.d/sal-preflight
@@ -63,7 +63,7 @@ def get_checksums():
     except requests.exceptions.RequestException as error:
         munkicommon.display_debug2(str(error_msg))
         return
-    if response.status_code != 200:
+    if response.status_code != requests.status_codes.codes.okay:
         munkicommon.display_debug2(f'Request failed with HTTP {response.status_code}')
         return
     if response and "<h1>Page not found</h1>" in response.text:
@@ -105,7 +105,7 @@ def download_and_write_script(server_script):
         munkicommon.display_debug2(str(error))
         return
 
-    if response.status_code != 200:
+    if response.status_code != requests.status_codes.codes.okay:
         munkicommon.display_debug2('Error received downloading script:')
         munkicommon.display_debug2(response.text)
 

From ac77f0102d61f1f7529dc328c24f378ed9ed3adb Mon Sep 17 00:00:00 2001
From: Shea Craig <sheagcraig@gmail.com>
Date: Fri, 21 Aug 2020 10:46:58 -0400
Subject: [PATCH 79/79] Update requests.

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f0e2ec0..55b1cfa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 sal_python_pkg/
 pyobjc==6.2.2
 urllib3==1.25.10
-requests==2.23.0
+requests==2.24.0
 MacSesh==0.3.0