Method launch_pinned_threads

src/utils/threads/thread_pool.cpp:109–187

FIXME (trb 08/03/2019): Setting thread affinity is not a portable pthread operation (hence the _np suffix); indeed, OSX does not support it. Unfortunately the case on OSX is even more dire -- they seem to want to prevent you from messing with their scheduler at all. The MACH kernel API for doing this is marked "deprecated" and its use is not advised for code that is not tied to a specific OSX vers

// As a result of the above, this will, in fact, *not* launch pinned
// threads when the locally-supported pthread API does not support it.
void thread_pool::launch_pinned_threads(size_type num_threads, int PU_offset)
{

#if defined(LBANN_TOPO_AWARE)
  threads_.reserve(num_threads);
  m_work_group.reserve(num_threads);
  m_thread_id_to_local_id_map.reserve(num_threads);

  hwloc_topology_t topo;
  int err;
  /* initialize a topology context */
  err = hwloc_topology_init(&topo);
  if (err) {
    LBANN_ERROR("hwloc_topology_init failed");
  }
  /* build the topology created and configured above */
  err = hwloc_topology_load(topo);
  if (err) {
    LBANN_ERROR("hwloc_topology_load failed");
  }
  // Get the number of PUs per core
  // hwloc_obj_t core = hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0);
  m_threads_offset = PU_offset;

  check_hwloc_api_version();
  //  hwloc_cpuset_t current_cpuset = get_local_cpuset_for_current_thread(topo);
  int i;
  //  hwloc_obj_t obj;
  // Try to launch each worker thread and pin it to a single core
  try {
    hwloc_cpuset_t allocated_cpuset = get_local_cpuset_for_current_thread(topo);
    hwloc_cpuset_t excluded_cpuset = hwloc_bitmap_alloc();
    // int cpuset_idx = 0;
    int skipped_indices = 0;
    // Skip PUs to match the thread offset
    hwloc_bitmap_foreach_begin(i, allocated_cpuset)
    {
      if (skipped_indices < m_threads_offset) {
        skipped_indices++;
        hwloc_bitmap_set(excluded_cpuset, i);
      }
    }
    hwloc_bitmap_foreach_end();

    hwloc_cpuset_t iot_cpuset = hwloc_bitmap_dup(allocated_cpuset);
    hwloc_bitmap_andnot(iot_cpuset, iot_cpuset, excluded_cpuset);
    if (hwloc_bitmap_iszero(iot_cpuset)) {
      LBANN_WARNING(
        "Insufficient number of allocated cores to respect I/O CPU offset");
      hwloc_bitmap_free(iot_cpuset);
      // Reset the cpuset back to all of the allowed cores
      iot_cpuset = hwloc_bitmap_dup(allocated_cpuset);
    }

    for (size_type cnt = 0; cnt < num_threads; ++cnt) {
      hwloc_cpuset_t ht_cpuset = hwloc_bitmap_dup(iot_cpuset);
      hwloc_topology_t ht_topo;
      err = hwloc_topology_dup(&ht_topo, topo);