Source code for hispecdrp.utils.parallelization_utils

"""
Shared helpers for parallel CPU work: shared-memory views, Numba thread limits, and
Numba decorators for the pipeline.

The HISPEC DRP stack depends on Numba at install time; this module imports it
eagerly. :func:`numba_thread_scope` and :func:`resolve_numba_threads` share logic
that probes Numba for the maximum allowed thread count (cached). On some CI runners
Numba caps parallel threads at 2 even when ``os.cpu_count()`` is higher; clamping
avoids ``ValueError`` from ``set_num_threads``.
"""
import ctypes
import os
import numpy as np
from contextlib import contextmanager

from numba import get_num_threads, njit, prange, set_num_threads

_CTYPES_TO_NP = {
    ctypes.c_float:   np.float32,
    ctypes.c_double:  np.float64,
    ctypes.c_int32:   np.int32,
    ctypes.c_uint32:  np.uint32,
    ctypes.c_int16:   np.int16,
    ctypes.c_uint16:  np.uint16,
    ctypes.c_int8:    np.int8,
    ctypes.c_uint8:   np.uint8,
}

__all__ = [
    "_arraytonumpy",
    "njit",
    "numba_thread_scope",
    "prange",
    "resolve_numba_threads",
]


[docs] def _arraytonumpy(shared_array, shape=None, dtype=None): """ Covert a shared array to a numpy array Originally from pyklip Wang, J. J., Ruffio, J.-B., De Rosa, R. J., et al. 2015, ASCL, ascl:1506.001 https://bitbucket.org/pyKLIP/pyklip/src/main/pyklip/parallelized.py Args: shared_array: a multiprocessing.Array array shape: a shape for the numpy array. otherwise, will assume a 1d array dtype: data type of the arrays. Should be either ctypes.c_float(default) or ctypes.c_double Returns: numpy_array: numpy array for vectorized operation. still points to the same memory! returns None is shared_array is None """ if dtype is None: dtype = ctypes.c_float np_dtype = _CTYPES_TO_NP.get(dtype) # if you passed in nothing you get nothing if shared_array is None: return None buf = shared_array.get_obj() if hasattr(shared_array, "get_obj") else shared_array numpy_array = np.frombuffer(buf, dtype=np_dtype) # numpy_array = np.frombuffer(shared_array.get_obj(), dtype=dtype) if shape is not None: numpy_array.shape = shape return numpy_array
# Cached cap: largest n accepted by set_num_threads on this build (CI often ≤ 2). _NUMBA_MAX_PARALLEL_THREADS: int | None = None def _numba_max_parallel_threads() -> int: """ Probe and cache the maximum thread count Numba allows for ``prange`` / ``set_num_threads``. Some wheels and CI VMs reject ``set_num_threads(n)`` for n above a small cap (e.g. 2) even when ``os.cpu_count()`` is larger. Result is cached for the process. """ global _NUMBA_MAX_PARALLEL_THREADS if _NUMBA_MAX_PARALLEL_THREADS is not None: return _NUMBA_MAX_PARALLEL_THREADS cpu = max(1, int(os.cpu_count() or 1)) saved = int(get_num_threads()) hi = min(cpu, 128) best = 1 try: for candidate in range(hi, 0, -1): try: set_num_threads(candidate) best = candidate break except ValueError: continue finally: set_num_threads(saved) _NUMBA_MAX_PARALLEL_THREADS = max(1, best) return _NUMBA_MAX_PARALLEL_THREADS def _effective_numba_threads(requested: int | None) -> int: """Map requested parallelism to a safe count (uses cached Numba max-thread probe).""" cap = _numba_max_parallel_threads() cpu = max(1, int(os.cpu_count() or 1)) if requested is None or requested == 0: return max(1, min(cpu, cap)) return max(1, min(int(requested), cap))
[docs] def resolve_numba_threads(max_cores: int | None) -> int: """ Map a requested parallelism level to a thread count safe for this host and Numba build. Uses the same maximum-thread probe and clamping as :func:`numba_thread_scope`. - ``None`` or ``0``: use all logical CPUs, capped to Numba's maximum. - Positive ``max_cores``: use at most that many threads, capped the same way. This prevents ``ValueError`` from ``set_num_threads`` on runners where Numba allows fewer threads than ``os.cpu_count()`` (e.g. GitHub Actions). """ return _effective_numba_threads(max_cores)
[docs] @contextmanager def numba_thread_scope(num_threads: int | None = None): """ Temporarily set Numba's thread pool, then restore the previous setting. On entry, this uses the same logic as :func:`resolve_numba_threads`: it determines the **maximum number of threads Numba allows** on this build (probed once and cached; often matches CPU count but may be **as low as 2 on CI**), then sets the active count to ``num_threads`` clamped to that maximum and to ``os.cpu_count()``. This addresses environments where ``set_num_threads(6)`` raises because Numba enforces ``Nthreads <= 2``. Parameters ---------- num_threads : int or None, optional Explicit thread count, or ``None`` / ``0`` for all logical CPUs (capped). Default is ``None``. Examples -------- >>> with numba_thread_scope(4): # noqa: SIM117 ... ... # Numba parallel regions here """ effective = _effective_numba_threads(num_threads) original_num_threads = get_num_threads() set_num_threads(effective) try: yield finally: set_num_threads(original_num_threads)