ó
à™xTc           @   s�  d  d l  m Z d  d l m Z d  d l m Z d  d l m Z d  d l m	 Z	 d  d l
 Z
 d  d l Z d  d l Z e j d  dK k Z e a d	 „  Z e d d
 „ Z d „  Z d  d l m Z e ƒ  Z d e f d „  ƒ  YZ d  d l m Z e ƒ  Z y d  d l m Z Wn e  k
 r#e Z n Xd e f d „  ƒ  YZ d „  Z! d e f d „  ƒ  YZ" d e f d „  ƒ  YZ# d e f d „  ƒ  YZ$ d e f d „  ƒ  YZ% d d dL e d d d d e d e& d d d „ Z' [ d Z( d d  d! d" d# d$ g Z) d% d& d' d( d) d* g Z* e( d+ j+ e) d, g ƒ d+ j+ e* d- g ƒ f Z, e( d+ j+ e) ƒ d+ j+ e* ƒ f Z- d. e f d/ „  ƒ  YZ. d d0 d1 d e& d e& e/ e& d2 „	 Z0 d3 „  Z1 d4 „  Z2 d5 e f d6 „  ƒ  YZ3 e/ d7 „ Z4 d8 „  Z5 e/ d9 „ Z6 d: e f d; „  ƒ  YZ7 i e7 d< d% d= e1 d> e4 ƒ d1 6e7 d< d- d= e2 d> e5 ƒ d, 6e7 d< d& d= e3 d? d  ƒ d> e6 ƒ d  6e7 d< d' d= e3 d? d! ƒ d> e6 ƒ d! 6e7 d< d( d= e3 d? d" ƒ d> e6 ƒ d" 6e7 d< d@ d= e3 d? dA ƒ d> e6 ƒ dA 6e7 d< d) d= e3 d? d# ƒ d> e6 ƒ d# 6e7 d< d* d= e3 d? d$ ƒ d> e6 ƒ d$ 6Z8 dB e f dC „  ƒ  YZ9 dD e f dE „  ƒ  YZ: dF e f dG „  ƒ  YZ; e& dH „ Z< e& d dI „ Z= d d0 d1 d e& d e& e/ e& dJ „	 Z> d S(M   iÿÿÿÿ(   t   division(   t   show_string(   t   ifloor(   t   Auto(   t   StringIONi   i   c          C   sR   t  t k rN d d l }  |  j d k r0 d a  qN d d l } | j j ƒ  a  n  t  S(   sJ   
  Identify situations where multiprocessing will not work as required.
  iÿÿÿÿNt   nts4   libtbx.easy_mp: Windows is not a supported platform.(   t   _problem_cacheR   t   ost   namet   libtbx.utilst   utilst   detect_multiprocessing_problem(   R   t   libtbx(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   detect_problem   s    	c         C   sÆ   |  d k s |  d k r d S| d k rD d d l m } | ƒ  } n  t ƒ  } | d k	 rª | t k	 rª |  t k s} |  d k r� d Sd d l m } | d t | ƒ ƒ ‚ n | d t |  ƒ IJ|  Sd S(	   s8  
  Switch for using multiple CPUs with the pool_map function, usually called at
  the beginning of an app.  If nproc is Auto or None and we are running
  Windows, it will be reset to 1.

  :param nproc: default number of processors to use
  :returns: number of processors to use (None or Auto means automatic)
  i   i    iÿÿÿÿ(   t   null_out(   t   Sorrys&   %s.  Please use nproc=1 or nproc=Auto.sW  
 ******************************************************************
 INFO: Some parts of this job will make use of multiple processors:
 ******************************************************************

   nproc = %s

 Please ask your system administrator for advice about this, in particular if
 you run this job through a queuing system.
N(   t   NoneR	   R   R   R   R   t   str(   t   nproct   logR   t   problemsR   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt"   enable_multiprocessing_if_possible   s    			c         C   s¡   |  d t g k r‹ t j d k s0 t j d	 k  r4 d Sd d l m } |  t k } | j ƒ  }  | r� t	 t
 |  t j ƒ  d ƒ d ƒ }  q� n |  d k s� t ‚ |  S(
   sì   
  Determine number of processes dynamically: number of CPUs minus the current
  load average (with a minimum of 1).

  :param processes: default number of processes (may be None or Auto)
  :returns: actual number of processes to use
  R   i   i   i   iÿÿÿÿ(   t   introspectioni    N(   i   i   (   R   R   R   R   t   syst   version_infoR   R   t   number_of_processorst   maxR   t
   getloadavgt   AssertionError(   t	   processesR   t   auto_adjust(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   get_processes?   s    )(   t   WeakValueDictionaryt   fixed_func_proxyc           B   s    e  Z d  Z d „  Z d „  Z RS(   s   Implementation detailc         C   s   | |  _  | t | <d  S(   N(   t   keyt   fixed_func_registry(   t   selfR"   t   func(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   __init__X   s    	c         C   s/   |  j  } t | } | d  k	 s% t ‚ | | ƒ S(   N(   R"   R#   R   R   (   R$   t   argR"   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   __call__\   s    	
(   t   __name__t
   __module__t   __doc__R&   R(   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR!   V   s   	(   t   count(   t   PoolR-   c           B   s2   e  Z d  Z d d d d d d „ Z d d „ Z RS(   s>   Subclass of multiprocessing.Pool, used internally by pool_map.c   	   	   C   sí   t  t k r6 t ƒ  } | d  k	 s' t ‚ t | ƒ ‚ n  t | ƒ |  _ | d  k	 rr t j	 ƒ  } t
 | | ƒ |  _
 n	 d  |  _
 t t |  ƒ j } | t k r¢ d  } n  | d  k rÊ | d | d | d | ƒ n | d | d | d | d | ƒ d  S(   NR   t   initializert   initargst   maxtasksperchild(   t   multiprocessing_Poolt   objectR   R   R   t   RuntimeErrorR   R   t!   fixed_func_registry_key_generatort   nextR!   t   superR-   R&   R   (	   R$   R   R.   R/   R0   t
   fixed_funct
   mp_problemR"   t   init(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   l   s,    			
c         C   s4   |  j  d k	 s t ‚ |  j d |  j  d | d | ƒ S(   sÍ   
    Uses fixed_func as passed to the constructor.
    Avoids repeated pickling/unpickling of func, which can be rate-limiting
    if func is large and the amount of work per call is relatively small.
    R%   t   iterablet	   chunksizeN(   R!   R   R   t   map(   R$   R:   R;   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   map_fixed_func‹   s
    		N(    (   R)   R*   R+   R   R&   R=   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR-   j   s   c         C   sn   d |  GHy t  | ƒ } Wn n5 Xt | ƒ d k rN | d  d | d } n  d G| GHt j d t j ƒ d  S(   Ns    CAUGHT EXCEPTION: (argument #%d)i   i   s   ...i‚ÿÿÿs   ARGUMENT LEADING TO EXCEPTION:t   file(   t   reprt   lent	   tracebackt	   print_excR   t   stdout(   t   indexR'   t   r(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   show_caught_exception—   s    		t   func_wrapper_simple_implc           B   s    e  Z d  Z d „  Z d „  Z RS(   s   Implementation detailc         C   s   | |  _  | |  _ d  S(   N(   t   optionsR%   (   t   ORH   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   ¦   s    	c         C   sš   t  | ƒ d k s t ‚ | \ } } |  j j rJ t ƒ  t _ t _ } n  y |  j | ƒ } Wn d  } t
 | | ƒ n X|  j j r– | j ƒ  | f S| S(   Ni   (   R@   R   RH   t   buffer_stdout_stderrR   R   t   stderrRC   R%   R   RF   t   getvalue(   RI   t   index_and_argRD   R'   t   siot   result(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR(   ª   s    (   R)   R*   R+   R&   R(   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRG   £   s   	t   func_wrapper_simplec           B   s#   e  Z d  Z e d „ Z d „  Z RS(   s   Implementation detailc         C   s   | |  _  d  S(   N(   RJ   (   RI   RJ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   »   s    c         C   s   t  d |  d | ƒ S(   NRH   R%   (   RG   (   RI   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   wrap¾   s    (   R)   R*   R+   t   FalseR&   RQ   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRP   ¸   s   t!   func_wrapper_sub_directories_implc           B   s    e  Z d  Z d „  Z d „  Z RS(   s   Implementation detailc         C   s   | |  _  | |  _ d  S(   N(   RH   R%   (   RI   RH   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   Ä   s    	c   
         sÍ  t  | ƒ d k s t ‚ | \ } } |  j j | ‰ t j ‰  ˆ  j ˆ ƒ r` d t ˆ ƒ d  f Sy t j	 ˆ d |  j j
 ƒWn d t ˆ ƒ d  f SXˆ  j ˆ ƒ s» d t ˆ ƒ d  f St j ƒ  } zë y t j ˆ ƒ Wn d t ˆ ƒ d  f SX‡  ‡ f d †  } y t d d	 ƒ } Wn d
 | ƒ  d  f SXt j } t j } zL | t _ t _ y |  j | ƒ }	 Wn" t | | ƒ d | ƒ  d  f SXWd  | t _ | t _ | j ƒ  XWd  t j | ƒ Xd  |	 f S(   Ni   s    sub-directory exists already: %st   modes   cannot create sub-directory: %ss"   failure creating sub-directory: %ss!   cannot chdir to sub-directory: %sc              s   t  ˆ  j ˆ d ƒ ƒ S(   NR   (   R   t   join(    (   t   opt   sub_name(    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   sub_logã   s    R   t   ws   cannot open file: %ss   CAUGHT EXCEPTION: %s(   R@   R   RH   t   sub_name_formatR   t   patht   existsR   R   t   makedirst   makedirs_modet   isdirt   getcwdt   chdirt   openR   RC   RK   R%   RF   t   close(
   RI   RM   RD   R'   t   initial_cwdRX   R   t   initial_outt   initial_errRO   (    (   RV   RW   s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR(   È   sR    					(   R)   R*   R+   R&   R(   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRS   Á   s   	t   func_wrapper_sub_directoriesc           B   s&   e  Z d  Z d d d „ Z d „  Z RS(   s   Implementation details   mp%03diÿ  c         C   s  t  | t ƒ s t ‚ | } | j d ƒ d k  r· | j d ƒ } | d k r› | j d | ƒ } | | t | ƒ k r’ | |  d t | ƒ d } q› d } n  | d k  r· | d 7} q· n  | } t | ƒ d k sÕ t ‚ t | d ƒ d k sñ t ‚ | |  _ | |  _ d  S(   Nt   %i    t   #s   %0t   diÿÿÿÿs   %03d(   t
   isinstanceR   R   t   findR,   R@   RZ   R^   (   RI   RZ   R^   t   st   it   c(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   ü   s     		c         C   s   t  d |  d | ƒ S(   NRH   R%   (   RS   (   RI   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRQ     s    (   R)   R*   R+   R&   RQ   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRg   ù   s   t   simplec         C   sÕ  | | g j  d ƒ d k s! t ‚ | | g j  d ƒ d k sB t ‚ | d k sc t | d ƒ sc t ‚ t |	 t ƒ r(|	 d k rŠ t ƒ  }	 q(|	 d k r¨ t d t ƒ }	 nM |	 d k rÀ t ƒ  }	 n5 |	 j	 d ƒ rå t d |	 d ƒ }	 n t
 d	 |	 ƒ ‚ | t k rt rd } n  | t k r(d } q(n  |	 d k	 r‹t |	 d
 d ƒ } | d k rat
 d ƒ ‚ n  | d k	 r|| | ƒ } q‹| | ƒ } n  t |  ƒ }  t j d k sµt j d k  r¾d }  n  | d k	 r÷| } |  d k	 r÷t |  t | ƒ ƒ }  q÷n  |
 rt | ƒ } n  | d k	 rk| d I|  IJt | d d ƒ } | d k	 rP| ƒ  n  d d l } | j ƒ  } n  d } |  d k sŒt j d k røg  } x | D]X } | d k	 rÁ| j | | ƒ ƒ n | j | | ƒ ƒ | d k	 r™| | d ƒ q™q™Wn� t d |  d | d | d | d | ƒ } | t k r4d } n  zI | d k	 rd| j d | d | d | ƒ } n | j d | d | ƒ } Wd | j ƒ  | j ƒ  X| d k	 rÑd d l m } | d | j ƒ  | d | ƒ n  | S(   sn  
  Parallelized map() using subclassed multiprocessing.Pool.  If func is not
  None, this function essentially calls the Pool's own map method; this means
  that both func and iterable/args must be pickle-able.  If fixed_func is not
  None, it will not be pickled but instead saved as an attribute of the Pool,
  which will be preserved after the fork() call.  Additional features include
  optional redirection of output and automatic process number determination.

  Note that because of the reliance on fork(), this function will run in serial
  on Windows, regardless of how many processors are available.

  :param processes: number of processes to spawn; if None or Auto, the
    get_processes() function will be used.
  :param func: target function (will be pickled)
  :param fixed_func: "fixed" target function, which will be be propagated to
    the child process when forked (instead of pickling)
  :param iterable: argument list
  :param args: same as iterable (alternate keyword)
  :param chunksize: number of arguments to process at once

  Examples
  --------
  >>> def f (x) :
  ...   return some_long_running_method(x)
  ...
  >>> args = range(1000)
  >>> result = easy_mp.pool_map(
  ...   func=f,
  ...   args=args)
  ...
  >>> print len(result)
  ... 1000

  >>> class f_caller (object) :
  ...   def __init__ (self, non_pickleable_object) :
  ...     self._obj = non_pickleable_object
  ...   def __call__ (self, x) :
  ...     return some_long_running_method(x, self._obj)
  ...
  >>> args = range(1000)
  >>> f = f_caller(processed_pdb_file)
  >>> result = easy_mp.pool_map(
  ...   fixed_func=f,
  ...   args=args)
  ...
  i   R(   Rp   RJ   t   sub_directoriess   sub_directories:RZ   i   s    Unknown func_wrapper keyword: %sRQ   s(   func_wrapper must have a .wrap() method.R   i   i   s   multiprocessing pool size:t   flushiÿÿÿÿNR   R.   R/   R0   R7   R%   R:   R;   (   t   show_wall_clock_timet   secondst   out(   i   i   (   R,   R   R   t   hasattrRk   R   RP   t   TrueRg   t
   startswithR3   R   t   _have_maxtasksperchildt   getattrR   R   R   R   R   t   minR@   t	   enumeratet   timet   appendR-   R<   R=   Rc   RU   R	   Rs   (   R   R.   R/   R0   R%   R7   R:   t   argsR;   t   func_wrappert
   index_argsR   t   call_back_for_serial_runRQ   Rr   R}   t
   time_startRO   t   poolRs   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   pool_map  sŠ    <!!		
		!
 s  
nproc = 1
  .type = int
  .short_caption = Number of processes
  .style = bold renderer:draw_nproc_widget
technology = %s
  .type = choice
  .short_caption = Parallelization method
  .caption = %s
qsub_command = None
  .type = str
  .short_caption = qsub command
s   *multiprocessingt   sget   lsft   pbst   condort   slurmt   Multiprocessingt   Sun_Grid_Enginet   LSFt   PBSt   Condort   SLURMt    t	   threadingt	   Threadingt   python_exception_safe_runc           B   s    e  Z d  Z d „  Z d „  Z RS(   s1   
  Thin wrapper for catching Python exceptions
  c         C   s   | |  _  d  S(   N(   R%   (   R$   R%   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   Á  s    c         O   s^   d d l  m } y" | j d |  j | | Ž  ƒ } Wn% t k
 rY } | j d | ƒ } n X| S(   Niÿÿÿÿ(   RO   t   valuet	   exception(   t   libtbx.queuing_system_utilsRO   t   SuccessR%   t	   Exceptiont   AnyException(   R$   R   t   kwargsRO   RE   t   e(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR(   Æ  s    "(   R)   R*   R+   R&   R(   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR”   ¼  s   	i   t   multiprocessingc   "   	      sÉ  | d k	 r* | j } | j } | j } n  d d l m } d d l m ‰  | d k sq | ˆ  j k sq t | ƒ ‚ | d k s’ t	 | d ƒ s’ t ‚ g  } | t
 d g k rÝ | d k rÎ t d t | ƒ ƒ } qÝ t | ƒ } n  | d k rR| d k rRt j j d	 t ƒ rRx@ | D]8 } |  | ƒ } | d k	 r=| | ƒ n  | j | ƒ qW| Sd d
 l m } d d l m } d  \ } } | d k rëd d l } |	 r¸d d l m } | j } n	 | j } |
 rß| j ƒ  } | j } qŸ| j } n´ | d k r(d d l m } d d l } | j } | j } nw ˆ  j | \ } } y& | d | d | d |	 d |	 ƒ } Wn# | j k
 r†} | d | ‚ n X| j } ‡  f d †  } g  t | ƒ D]- } | j  d | d | j! d | ƒ  ƒ ƒ ^ q¬} | j d | ƒ } | r| j" } n	 | j# } t$ d |  ƒ ‰ y„ | j% d ‡ f d †  | Dƒ d | ƒ } xU | d | ƒ D]D \ } }  |  ƒ  }! |! ƒ  } | j | ƒ | d k	 rV| | ƒ qVqVWWn# | j k
 rÄ} | d | ‚ n X| S(!   sì  
  Generic parallel map() implementation for a variety of platforms, including
  the multiprocessing module and supported queuing systems, via the module
  libtbx.queuing_system_utils.scheduling.  This is less flexible than pool_map
  above, since it does not provide a way to use a non-pickleable target
  function, but it provides a consistent API for programs where multiple
  execution methods are desired.  It will also work on Windows (if the method
  is multiprocessing or threading).

  Note that for most applications, the threading method will be constrained
  by the Global Interpreter Lock, therefore multiprocessing is prefered for
  parallelizing across a single multi-core system.

  See Computational Crystallography Newsletter 3:37-42 (2012) for details of
  the underlying method.

  :param func: target function (must be pickleable)
  :param iterable: list of arguments for func
  :param processes: number of processes/threads to start
  :param method: parallelization method (multiprocessing|threading|sge|lsf|pbs)
  :param qsub_command: command to submit queue jobs (optional)
  :param asynchronous: run queue jobs asynchronously
  :param preserve_exception_message: keeps original exception message
  :returns: a list of result objects
  iÿÿÿÿ(   t
   scheduling(   t
   processingR�   R’   R(   i   i   t   LIBTBX_FORCE_PARALLEL(   t   errors(   R   N(   t   scheduling_helperst   commandt   asynchronoust
   save_errort   display_stderrs   Queue error: %sc              s   ˆ  j  d d ƒ S(   Nt
   identifiert   parallel_map(   t   Queue(    (   RŸ   (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   <lambda>@  s    t   factoryt	   processort   queuet   unitsR%   t   calculationsc         3   s!   |  ] } ˆ  | f i  f Vq d  S(   N(    (   t   .0R   (   t   safe_run_func(    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>S  s    t   managert   parallel_for(   s   multiprocessingR’   (   R’   s   multiprocessing(   R’   s   multiprocessing(   NN(&   R   t
   technologyR   t   qsub_commandR—   Rž   RŸ   t   INTERFACE_FORR   Rv   R   R{   R@   R   R   t   environt   getRR   R~   t&   libtbx.queuing_system_utils.processingR¡   R	   R   R�   R¢   t   Processt   ManagerR©   t   Threadt   BatchQueueErrort   Jobt   ranget   ExecutionUnitt   RetrieveProcessort   SubmissionOrdert   FinishingOrderR”   t   ParallelForIterator("   R%   R:   t   paramsR   t   methodRµ   R¤   t   callbackt   preserve_ordert   preserve_exception_messaget   use_managerRž   t   resultsR   RO   R¡   R   R«   t   queue_factoryR�   R¢   t
   mp_managerR©   t   qhandler_functiont	   evaluatort   qhandlerRœ   Rn   R®   R²   t   ordererR³   t   result_wrappert   result_proxy(    (   RŸ   R±   s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR¨   Ó  sŽ    %		!			=					c          K   s   d d l  m } | j |  �  S(   Niÿÿÿÿ(   R¢   (   R—   R¢   t   ProcessFactory(   R›   R¢   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   multiprocessing_getterd  s    c          K   s   d d l  m } | j |  �  S(   Niÿÿÿÿ(   R¢   (   R—   R¢   t   ThreadFactory(   R›   R¢   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   threading_getterj  s    t
   job_getterc           B   s   e  Z d  „  Z d „  Z RS(   c         C   s   | |  _  d  S(   N(   t   platform(   R$   RÙ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   r  s    c         K   s&   d d l  m } | j d |  j | � S(   Niÿÿÿÿ(   RŸ   RÙ   (   R—   RŸ   t
   JobFactoryRÙ   (   R$   R›   RŸ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR(   w  s    (   R)   R*   R&   R(   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRØ   p  s   	c         K   s+   d d l  m } |  r  | j ƒ  S| j Sd  S(   Niÿÿÿÿ(   R¢   (   R—   R¢   t   MPManagerQFactoryt
   MPQFactory(   RÊ   R›   R¢   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   multiprocessing_queue_getter}  s    
c          K   s   d d l  m } | j S(   Niÿÿÿÿ(   R¢   (   R—   R¢   t	   QQFactory(   R›   R¢   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   threading_queue_getterˆ  s    c         K   s|   |  r? d d l  m } t d „  | j ƒ  Dƒ ƒ } | j | �  Sd d l  m } t d „  | j ƒ  Dƒ ƒ } | j | �  Sd  S(   Niÿÿÿÿ(   t   socket_queuec         s   s4   |  ]* \ } } | j  d  ƒ r | d | f Vq d S(   t   socketq_i   N(   Rx   (   R°   t   kt   v(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>“  s    (   t
   file_queuec         s   s4   |  ]* \ } } | j  d  ƒ r | d | f Vq d S(   t   fileq_i   N(   Rx   (   R°   Râ   Rã   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>š  s    (   R—   Rà   t   dictt   itemst   QFactoryRä   (   t   socket_basedR›   Rà   t   relevantRä   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   job_queue_getterŽ  s    t   parallelization_infoc           B   s)   e  Z d  Z d „  Z d „  Z d „  Z RS(   s(   
  Data for setting up a parallel run
  c         C   s   | |  _  | |  _ | |  _ d  S(   N(   t   captiont   gettert   qgetter(   R$   Rí   Rî   Rï   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   ¤  s    		c         K   s   |  j  | �  S(   N(   Rî   (   R$   R›   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   jfactory«  s    c         K   s   |  j  | �  S(   N(   Rï   (   R$   R›   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   qfactory°  s    (   R)   R*   R+   R&   Rð   Rñ   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRì   Ÿ  s   		Rí   Rî   Rï   RÙ   t   PBSProt   pbsprot   process_pool_holderc           B   s    e  Z d  Z d „  Z d „  Z RS(   s.   
  Holds the pool and shuts it down at exit
  c         C   sq   | |  _  | j ƒ  |  _ | j ƒ  |  _ d d l m } | j d |  j d |  j d | d | d | j ƒ |  _ d  S(   Niÿÿÿÿ(   Rž   t   inqueuet   outqueuet   job_factoryt   loadmanagert   lifemanager(	   Rñ   t   createRõ   Rö   R—   Rž   t   ProcessPoolt   ImmortalR„   (   R$   Rð   Rñ   Rø   Rž   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&   æ  s    				c         C   sD   |  j  j ƒ  |  j  j ƒ  |  j j |  j ƒ |  j j |  j ƒ d  S(   N(   R„   t	   terminateRU   Rñ   t   destroyRõ   Rö   (   R$   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   __del__ö  s    (   R)   R*   R+   R&   Rÿ   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRô   á  s   	t   mainthread_pool_holderc           B   s   e  Z d  Z d „  Z RS(   s   
  Holds a mainthread pool
  c         C   s#   d d l  m } | j ƒ  |  _ d  S(   Niÿÿÿÿ(   Rž   (   R—   Rž   t   MainthreadPoolR„   (   R$   Rž   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&     s    (   R)   R*   R+   R&   (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR   ÿ  s   t   process_pool_user_interfacec           B   sw   e  Z d  Z e e e d e d e e e d d d d d d „ Z d „  Z d	 d d e d
 „ Z d „  Z	 e
 d „  ƒ Z RS(   s$   
  Customize needs by application
  R�   t   tmpt   .gš™™™™™¹?i    i   c         C   sñ   g  |  _  | r" |  j  j d ƒ n  | r; |  j  j d ƒ n  | rf |  j  j d d d d d d g ƒ n  | d  k	 rŠ | t k rŠ t d	 ‚ n  | |  _ | |  _ | |  _ | |  _	 | |  _
 |	 |  _ |
 |  _ | |  _ | |  _ | |  _ | |  _ d  S(
   NR�   R’   R†   R‡   Rˆ   Ró   R‰   RŠ   s0   Incorrect default for parallelization technology(   t	   jkeywordsR~   t   extendR   t   parallelization_method_namedt
   ValueErrort   defaultt   enable_socket_queuet   factory_namet"   factory_preserve_exception_messaget   factory_job_use_target_filet   qfactory_mp_use_managert   qfactory_fileq_prefixt   qfactory_fileq_foldert   qfactory_fileq_waittimet   qfactory_socketq_portt   qfactory_socketq_keylength(   R$   t   enable_multiprocessingt   enable_threadingt   enable_queuing_systemsR	  R
  R  R  R  R  R  R  R  R  R  (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR&     s(    	%										c         C   s`   d i |  j  d |  j d |  j ƒ d 6d j d „  |  j Dƒ ƒ d 6} |  j r\ | d 7} n  d  S(	   Ns9  
    nproc = 1
      .type = int
      .short_caption = Number of processes
    technology = %(technologies)s
      .type = choice
      .short_caption = Parallelization method
      .caption = %(captions)s
      .optional = False
    qsub_command = None
      .type = str
      .short_caption = qsub command
    t   keywordsR	  t   technologiesR‘   c         s   s   |  ] } t  | j Vq d  S(   N(   R  Rí   (   R°   Râ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>W  s    t   captionss“   network_based_channel = False
        .type = bool
        .help = Use network (as opposed to filesystem) to communicate with submitted jobs
      (   t   phil_choiceR  R	  RU   R
  (   R$   t   base(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   phil_strC  s    			i   c   
      C   s  | d k r& t  j j d ƒ r& t ƒ  St | } d d l m } d d l m } d d l	 m
 } y’ t d | j d |  j d	 |  j d
 | d |  j ƒ d | j d |  j d | d |  j d |  j d |  j d |  j d |  j ƒ d | j d | ƒ ƒ SWn# | j k
 r}	 | d |	 ‚ n Xd  S(   Ni   R    iÿÿÿÿ(   Rž   (   R¡   (   R   Rð   R   RÉ   R£   t   use_target_fileRñ   RÊ   Ré   t   fileq_prefixt   fileq_foldert   fileq_waittimet   socketq_portt   socketq_keylengthRø   t   capacitys   Queue error: %s(   R   R·   R¸   R   R  R—   Rž   R¹   R¡   R	   R   Rô   Rð   R  R  R  Rñ   R  R  R  R  R  R  t   ConstantStrategyR½   (
   R$   R   R´   Rµ   t   network_based_channelt   pinfoRž   R¡   R   Rœ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   create_from_argumentsb  s0    
									c      	   C   s=   |  j  d | j d | j d | j d |  j r6 | j n t ƒ S(   NR   R´   Rµ   R%  (   R'  R   R´   Rµ   R
  R%  RR   (   R$   t   extract(    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   create_from_philŠ  s
    				c            s   d j  ‡  f d †  |  Dƒ ƒ S(   NR‘   c         3   s+   |  ]! } | ˆ  k r d  | n | Vq d S(   t   *N(    (   R°   Râ   (   R	  (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>˜  s    (   RU   (   R  R	  (    (   R	  s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR  •  s    N(   R)   R*   R+   Rw   RR   R   R&   R  R'  R)  t   staticmethodR  (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyR  
  s,    	!#	c            sq   d d l  m } t d | ƒ ‰  | j d ‡  f d †  | Dƒ d |  ƒ } | r] | j d | ƒ S| j d | ƒ Sd  S(   Niÿÿÿÿ(   Rž   R%   R¯   c         3   s!   |  ] } ˆ  | f i  f Vq d  S(   N(    (   R°   R   (   R±   (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pys	   <genexpr>   s    R²   R³   (   R—   Rž   R”   RÄ   RÂ   RÃ   (   R„   R%   R:   RÈ   Rž   t   pfor(    (   R±   s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   pool_based_parallel_iterator›  s    		c      	   C   sÍ   | d  k r d „  } n  d d l m } d d l m } g  } ye t d |  d | d | d | ƒ } x= | D]5 \ }	 }
 |
 ƒ  } | ƒ  } | j | ƒ | | ƒ qi WWn# | j k
 rÈ } | d	 | ‚ n X| S(
   Nc         S   s   d  S(   N(   R   (   RO   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyRª   ®  s    iÿÿÿÿ(   R¡   (   R   R„   R%   R:   RÈ   s   Queue error: %s(   R   R¹   R¡   R	   R   R-  R~   R½   (   R„   R%   R:   RÈ   RÇ   R¡   R   RË   t   piterRÅ   RÒ   RÓ   RO   Rœ   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   pool_based_parallel_map«  s&    			c         C   sž   t  d t d |	 d |
 ƒ } | d  k	 rE | j } | j } | j } n  | j d | d | d | ƒ } t d | j d |  d	 | d
 | d | ƒ } | j j	 ƒ  | S(   NR  R  R  R   R´   Rµ   R„   R%   R:   RÈ   RÇ   (
   R  Rw   R   R´   R   Rµ   R'  R/  R„   RU   (   R%   R:   RÅ   R   RÆ   Rµ   R¤   RÇ   RÈ   RÉ   RÊ   t   uit   holderRË   (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   replacement_parallel_mapÉ  s(    							(   i   i   (    (?   t
   __future__R    t   libtbx.str_utilsR   t   libtbx.math_utilsR   R   R   t	   cStringIOR   RA   R   R   R   Ry   R   R   R   R   R   t   weakrefR    t   _R#   R2   R!   t	   itertoolsR,   R4   t   multiprocessing.poolR-   R1   R™   RF   RG   RP   RS   Rg   Rw   R…   t   parallel_phil_str_baset   parallel_methodst   parallel_captionsRU   t   parallel_phil_strt   parallel_phil_str_no_threadingR”   RR   R¨   RÕ   R×   RØ   RÝ   Rß   Rë   Rì   R  Rô   R   R  R-  R/  R2  (    (    (    s<   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/easy_mp.pyt   <module>   sâ   	#			
-		8€†			






‘!