* Third attempt to fix lmod "module" for ipython ("fix3")

(dated: 2020-06-09).
* Supports both Turing and Wahab.
* Allows both addition and deletion of paths from sys.path.

* Author's note: Later on, I discovered that these complex steps do not appear
  to be necessary, somehow the lmod Python commands were able to take care
  of additions and deletions through the modification of os.environ['PYTHONPATH']
  alone, so I am not sure if this fix is absolutely needed. (?)

  Sigs:
    # -rw------- 1 wpurwant users 3298 2020-06-09 20:07 lmod_python_fix3.py
    # 78abc7bae90bf23a42aa40285175ca48  lmod_python_fix3.py

lmod/ipython/startup/lmod_python_fix.py

@@ -1,39 +1,88 @@
 # -*- python -*-
+# modified "module" command for Wahab
+
 #from __future__ import print_function
 #from subprocess import PIPE, Popen
 import os, sys 
 
-sys.path.append('/shared/apps/common/lmod/current/init')
+try:
-
+    with open("/etc/cluster", "r") as _cluster_F:
-from env_modules_python import module as lmod_module
+        _cluster = next(_cluster_F).strip()
-
+except:
-def module(command, *arguments):
+    sys.stderr.write("Unable to determine cluster, 'module' command will not be available.\n")
-    lmod_module(command, *arguments)
+else:
-    # BAD: This will cause alteration of PYTHONPATH in a way that may not be desirable.
+    if _cluster == "wahab":
-    sys_path_orig = sys.path
+        sys.path.append('/shared/apps/common/lmod/current/init')
-    PYTHONPATH = os.environ.get('PYTHONPATH')
+    elif _cluster == "turing":
-    sys_path_new = PYTHONPATH.split(':') if PYTHONPATH is not None else []
+        sys.path.append('/cm/shared/applications/lmod/lmod/init')
-    for p in sys_path_orig:
+    else:
-        if p not in sys_path_new:
+        sys.stderr.write("Unknown cluster, 'module' command will not be available.\n")
-            sys_path_new.append(p)
+    del _cluster
-    print(sys_path_new)
+
-    sys.path = sys_path_new
+try:
-
+    from env_modules_python import module as lmod_module
-    manual_ld_library_dir = os.environ.get('LMOD_MANUAL_LD_LIBRARY_PATH')
+except:
-    if manual_ld_library_dir is None:
+    pass
-        return
+else:
-
+    def module(command, *arguments):
-    for search_path in os.getenv('LD_LIBRARY_PATH').split(':')[::-1]:
+        split_path_str = lambda P: P.split(':') if P is not None else []
-        if not os.path.isdir(search_path):
+
-            continue
+        PYTHONPATH_ORIG = os.environ.get('PYTHONPATH')
-        if search_path == manual_ld_library_dir:
+        lmod_module(command, *arguments)
-            continue
+        PYTHONPATH = os.environ.get('PYTHONPATH')
-
+
-        for f in os.listdir(search_path):
+        # update sys.path
-            if '.so' in f:
+        # Must account the *changes* in PYTHONPATH and reflect that to sys.path
-                src = f'{search_path}/{f}'
+        #
-                tgt = f'{manual_ld_library_dir}/{f}'
+        # WARNING: This is not perfect!
-                if os.path.islink(tgt):
+        # Due to the algorithm used below, the updated PYTHONPATH
-                    os.unlink(tgt)
+        # will not be reflected 100% in its order in sys.path
-
+        #
-                os.symlink(src, tgt)
+        # 1) It assumes that additional paths are prepended.
+        # 2) It will completely delete a path that is remove
+        #    as a result of module command
+        # 3) If there are simultaneous deletion(s) and adition(s),
+        #    then the added paths will take precedence (unfortunately),
+        #    which may result in unexpected behavior in corner cases,
+        #    at least in principle.
+        #    This should not be the case for "well behaved" HPC packages
+        #    where order of loaded modules should NOT cause change in behavior.
+
+        # Save the original in case needed for debugging
+        global sys_path_orig
+        sys_path_orig = sys.path
+
+        python_path_orig = split_path_str(PYTHONPATH_ORIG)
+        python_path = split_path_str(PYTHONPATH)
+
+        sys_path_added = set(python_path) - set(python_path_orig)
+        sys_path_deleted = set(python_path_orig) - set(python_path)
+
+        # delete removed paths
+        sys_path_trimmed = [ p for p in sys_path_orig if p not in sys_path_deleted ]
+
+        # prepend new paths
+        sys_path_prepended = [ p for p in python_path if p in sys_path_added ]
+
+        sys.path = sys_path_prepended + sys_path_trimmed
+
+        # Extra: if LMOD_MANUAL_LD_LIBRARY_PATH is specified, then
+        # the shared library symlinks are updated.
+        manual_ld_library_dir = os.environ.get('LMOD_MANUAL_LD_LIBRARY_PATH')
+        if manual_ld_library_dir is None:
+            return
+
+        for search_path in os.getenv('LD_LIBRARY_PATH').split(':')[::-1]:
+            if not os.path.isdir(search_path):
+                continue
+            if search_path == manual_ld_library_dir:
+                continue
+
+            for f in os.listdir(search_path):
+                if '.so' in f:
+                    src = f'{search_path}/{f}'
+                    tgt = f'{manual_ld_library_dir}/{f}'
+                    if os.path.islink(tgt):
+                        os.unlink(tgt)
+
+                    os.symlink(src, tgt)