ó
ë™xTc        
   @   s™  d  Z  d d l m Z d d l m Z d d l Z d d l Z d d l Z d d l Z d d l	 Z	 d d l
 m Z d e f d „  ƒ  YZ d e f d	 „  ƒ  YZ d
 e f d „  ƒ  YZ d e f d „  ƒ  YZ
 d e f d „  ƒ  YZ d e f d „  ƒ  YZ d „  Z d d e d d d d e e d „	 Z d d e d d d d e e d „	 Z d d e d d d d e e d „	 Z d d e d d d d e e d „	 Z d d e d d d d e e d „	 Z d d e d d d d e e d „	 Z d „  Z d „  Z d „  Z d „  Z  d „  Z! d „  Z" d  „  Z# i e e f d! 6e e f d" 6e e  f d# 6e e! f d$ 6e e" f d% 6e e# f d& 6Z$ d d! d d d' „ Z% d( e f d) „  ƒ  YZ& d S(*   så   
Generic module to provide parallel job execution on queuing systems

Provides drop-in replacement classes to those defined in the multiprocessing
module (Queue and Process), with certain restrictions placed by the pickle
module
iÿÿÿÿ(   t   division(   t   with_statementN(   t   Emptyt   InstantTimeoutc           B   s   e  Z d  Z d „  Z RS(   s   
  Timeout immediately
  c         C   s   t  d ‚ d  S(   Ns   No data found in queue(   t   QueueEmptyException(   t   selft   waittime(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   delay   s    (   t   __name__t
   __module__t   __doc__R   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR      s   t   TimedTimeoutc           B   s    e  Z d  Z d „  Z d „  Z RS(   s   
  Timeout after given time
  c         C   s   | |  _  d  S(   N(   t	   max_delay(   R   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   __init__!   s    c         C   sM   d |  j  k  r@ t |  j  | ƒ } |  j  | 8_  t j | ƒ n	 t d ‚ d  S(   Ni    s%   No data found in queue within timeout(   R   t   mint   timet   sleepR   (   R   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   &   s
    (   R   R	   R
   R   R   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR      s   	t	   NoTimeoutc           B   s   e  Z d  Z d „  Z RS(   s   
  No timeout
  c         C   s   t  j | ƒ d  S(   N(   R   R   (   R   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   6   s    (   R   R	   R
   R   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   1   s   t   Queuec           B   s;   e  Z d  Z d d „ Z d „  Z e d d „ Z d „  Z RS(   sÖ   
  Queue object to receive data from jobs running on remote machines

  Data transfer is achieved via files. It is safe to use any number of
  Queue objects in the same directory, even with a matching identifier
  i   c         C   s>   | |  _  d | t j ƒ  t |  ƒ f |  _ t j ƒ  |  _ d  S(   Ns   %s_%d_%d(   R   t   ost   getpidt   idt   roott	   itertoolst   count(   R   t
   identifierR   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   C   s    	"c         C   s£   |  j  j ƒ  } d |  j | f } t j j | ƒ s; t ‚ t | d ƒ � } t j	 | | ƒ Wd  QXd |  j | f } t j j | ƒ s� t ‚ t j
 | | ƒ d  S(   Ns	   tmp_%s.%dt   wbs   %s.%d(   R   t   nextR   R   t   patht   existst   AssertionErrort   opent   picklet   dumpt   rename(   R   t   objt   indext   tmp_namet   ofilet   target_name(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   putJ   s    c         C   s…   | s t  ƒ  } n' | d  k	 r0 t d | ƒ } n	 t ƒ  } xE t r€ y |  j ƒ  } Wn' t k
 r{ | j d |  j ƒ q< n XPq< W| S(   NR   R   (	   R   t   NoneR   R   t   TrueR   t   StopIterationR   R   (   R   t   blockt   timeoutt	   predicatet   data(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   getY   s    		c         C   sÎ   g  } xt t  j  d |  j ƒ D]\ } t j j | ƒ \ } } y t | d ƒ } Wn t k
 re q n X| j | | f ƒ q W| sŒ t ‚ n  t	 | d d „  ƒd } t
 j t | d ƒ ƒ } t j | ƒ | S(   Ns   %s.*i   t   keyc         S   s   |  d S(   Ni    (    (   t   p(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   <lambda>„   s    t   rb(   t   globR   R   R   t   splitextt   intt
   ValueErrort   appendR+   R   R    t   loadR   t   remove(   R   t   waitingt   fnameR   t   extt   numbert   selectedR/   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   r   s    	N(	   R   R	   R
   R   R(   R*   R)   R0   R   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   ;   s
   	t   Jobc           B   sh   e  Z d  Z d	 i  d „ Z e d „  ƒ Z e d „  ƒ Z d „  Z d „  Z d „  Z	 d „  Z
 d „  Z RS(
   sœ   
  Job object to execute function calls on remote machines accessible via
  queuing systems

  Restrictions: target, args and kwargs has to be pickleable
  c         C   s1   | |  _  | |  _ | |  _ | |  _ d  |  _ d  S(   N(   t
   qinterfacet   targett   argst   kwargsR)   t   status(   R   RB   RC   RD   RE   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   ’   s
    				c         C   s   d |  j  j t |  ƒ f S(   Ns   %s_%d(   RB   R   R   (   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   name�   s    c         C   s   t  |  j d d  ƒ S(   Nt   jobid(   t   getattrRF   R)   (   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRH   £   s    c         C   s§   |  j  d  k	 r t d ‚ n  |  j j d |  j d |  j d |  j d |  j ƒ } |  j j	 d |  j d |  j j
 d |  j j | j ƒ  d |  j j d	 | j ƒ  ƒ |  _  d  S(
   Ns   start called second timeRG   RC   RD   RE   t
   executablet   scriptt   includet   cleanup(   RF   R)   t   RuntimeErrorRB   t   inputRG   RC   RD   RE   t	   submitterRJ   RK   RL   t   files(   R   R/   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   start¨   s    				c         C   s)   |  j  d  k r t d ‚ n  |  j  j ƒ  S(   Ns   job has not been submitted yet(   RF   R)   RN   t   is_finished(   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   is_alive½   s    c         C   s­   x |  j  ƒ  r t j d ƒ q W|  j j ƒ  \ } } |  _ | rI | GHn  | rc |  j j rc | GHn  |  j j rœ |  j d k r� t	 | ƒ |  _
 qœ d  |  _
 n  |  j j ƒ  d  S(   Ngš™™™™™¹?i    (   RT   R   R   RF   t   resultst   exitcodeRB   t   display_stderrt
   save_errorRN   t   errR)   RM   (   R   t   stdoutt   stderr(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   joinÅ   s    c         C   s   |  j  j ƒ  d  S(   N(   RF   t	   terminate(   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR]   Ü   s    c         C   s   d |  j  j |  j f S(   Ns   %s(name = '%s')(   t	   __class__R   RG   (   R   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   __str__á   s    (    (   R   R	   R
   R   t   propertyRG   RH   RR   RT   R\   R]   R_   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRA   Š   s   				t   QueueHandlerc           B   s8   e  Z d  Z d Z d e e e d „ Z d i  d „ Z RS(   s<   
  Handles submission requests for common queuing systems
  s6   import cPickle as pickle
%s
target( *args, **kwargs )
s   libtbx.pythonc	   	      C   s\   | |  _  d | t j ƒ  f |  _ | |  _ | |  _ | |  _ | |  _ | |  _ | |  _	 d  S(   Ns   %s%s(
   RP   R   R   R   RO   RL   RJ   RK   RX   RW   (	   R   RP   RO   RL   R   RJ   RK   RX   RW   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   ñ   s    						c      	   C   s   t  d |  d | d | d | ƒ S(   NRB   RC   RD   RE   (   RA   (   R   RC   RD   RE   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRA   	  s
    (    (   R   R	   R
   t   SCRIPTt   FalseR*   R   RA   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRa   æ   s   c          C   s   d d  l  }  |  j j d ƒ S(   Niÿÿÿÿs   setpaths.sh(   t   libtbx.load_envt   envt   under_build(   t   libtbx(    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   get_libtbx_env_setpaths  s    t   libtbx_pythonc	         C   s(  t  d | d d g ƒ } d d l m }	 | r¨ | d  k rV d d l m }
 |
 j } n  | d  k r„ d d l m } | j j ƒ  } n  |	 j	 j d | d	 | d | ƒ } n |	 j
 j d | ƒ } | d  k rå d d
 l m } | j } n  | d  k rý t ƒ  } n  t d | d | d | d |  d | d | ƒ S(   Nt   commandt   defaultt   qsubiÿÿÿÿ(   t
   submission(   RF   (   t   pollingt   pollert   handler(   t   transferRP   RO   RL   R   RX   RW   (   t   process_command_linet&   libtbx.queuing_system_utils.processingRm   R)   RF   t   StdStreamStrategyRn   t   CentralPollert   SGEt   AsynchronousCmdLinet   SynchronousRq   t   TemporaryFileRh   Ra   (   RG   Rj   t   asynchronousRO   RL   Ro   Rp   RX   RW   Rm   RF   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRv     s4    c	         C   sú   t  d | d d g ƒ } d d l m }	 | rz | d  k r\ d d l m }
 |
 j j ƒ  } n  |	 j j d | d | ƒ } n |	 j j d | ƒ } | d  k r· d d l m	 } | j
 } n  | d  k rÏ t ƒ  } n  t d	 | d
 | d | d |  d | d | ƒ S(   NRj   Rk   t   bsubiÿÿÿÿ(   Rm   (   Rn   Ro   (   Rq   RP   RO   RL   R   RX   RW   (   Rr   Rs   Rm   R)   Rn   Ru   t   LSFRw   Rx   Rq   Ry   Rh   Ra   (   RG   Rj   Rz   RO   RL   Ro   Rp   RX   RW   Rm   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR|   L  s,    c	         C   sî   t  d | d d g ƒ } d d l m }	 | rz | d  k r\ d d l m }
 |
 j j ƒ  } n  |	 j j d | d | ƒ } n	 t d ‚ | d  k r« d d	 l m	 } | j
 } n  | d  k rÃ t ƒ  } n  t d
 | d | d | d |  d | d | ƒ S(   NRj   Rk   Rl   iÿÿÿÿ(   Rm   (   Rn   Ro   s+   PBS does not support synchronous submission(   Rq   RP   RO   RL   R   RX   RW   (   Rr   Rs   Rm   R)   Rn   Ru   t   PBSRw   RN   Rq   Ry   Rh   Ra   (   RG   Rj   Rz   RO   RL   Ro   Rp   RX   RW   Rm   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR}   z  s,    	c	         C   sî   t  d | d d g ƒ } d d l m }	 | rz | d  k r\ d d l m }
 |
 j j ƒ  } n  |	 j j d | d | ƒ } n	 t d ‚ | d  k r« d d	 l m	 } | j
 } n  | d  k rÃ t ƒ  } n  t d
 | d | d | d |  d | d | ƒ S(   NRj   Rk   Rl   iÿÿÿÿ(   Rm   (   Rn   Ro   s2   Synchronous submission for PBSPro is not supported(   Rq   RP   RO   RL   R   RX   RW   (   Rr   Rs   Rm   R)   Rn   Ru   t   PBSProRw   RN   Rq   Ry   Rh   Ra   (   RG   Rj   Rz   RO   RL   Ro   Rp   RX   RW   Rm   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR~   ¨  s,    	c	         C   sî   t  d | d d g ƒ } d d l m }	 | rz | d  k r\ d d l m }
 |
 j j ƒ  } n  |	 j j d | d | ƒ } n	 t d ‚ | d  k r« d d	 l m	 } | j
 } n  | d  k rÃ t ƒ  } n  t d
 | d | d | d |  d | d | ƒ S(   NRj   Rk   t   condor_submitiÿÿÿÿ(   Rm   (   Rn   Ro   s.   Condor does not support synchronous submission(   Rq   RP   RO   RL   R   RX   RW   (   Rr   Rs   Rm   R)   Rn   Ru   t   Condort   AsynchronousScriptRN   Rq   Ry   Rh   Ra   (   RG   Rj   Rz   RO   RL   Ro   Rp   RX   RW   Rm   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR€   Ö  s,    	c	         C   s  d d l  m }	 | rz t d | d d g ƒ } | d  k r\ d d l  m }
 |
 j j ƒ  } n  |	 j j d | d | ƒ } n* t d | d d g ƒ } |	 j d | ƒ } | d  k rÌ d d	 l  m	 } | j
 } n  | d  k rä t ƒ  } n  t d
 | d | d | d |  d | d | ƒ S(   Niÿÿÿÿ(   Rm   Rj   Rk   t   sbatch(   Rn   Ro   t   srun(   Rq   RP   RO   RL   R   RX   RW   (   Rs   Rm   Rr   R)   Rn   Ru   t   SlurmRw   t   SlurmStreamRq   Ry   Rh   Ra   (   RG   Rj   Rz   RO   RL   Ro   Rp   RX   RW   Rm   Rn   RP   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR„     s.    c         C   s-   |  d  k r | Sd d  l } | j |  ƒ Sd  S(   Niÿÿÿÿ(   R)   t   shlext   split(   Rj   Rk   R†   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRr   3  s    c         C   s>   d d l  m } | j d t d |  d d d g ƒ d | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   t   qstats   -jt   evaluate(   Rs   Rn   t   SinglePollerRr   t   sge_single_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   sge_evaluate=  s    	c         C   s;   d d l  m } | j d t d |  d d g ƒ d | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   t   bjobsR‰   (   Rs   Rn   RŠ   Rr   t   lsf_single_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   lsf_evaluateG  s    	c         C   s>   d d l  m } | j d t d |  d d d g ƒ d | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   Rˆ   s   -fR‰   (   Rs   Rn   RŠ   Rr   t   pbs_single_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   pbs_evaluateQ  s    	c         C   s;   d d l  m } | j d t d |  d d g ƒ d | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   Rˆ   R‰   (   Rs   Rn   Ru   Rr   t   pbspro_text_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   pbspro_evaluate[  s    	c         C   s>   d d l  m } | j d t d |  d d d g ƒ d | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   t   condor_qs   -xmlR‰   (   Rs   Rn   Ru   Rr   t   condor_xml_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   condor_evaluatee  s    	c         C   sG   d d l  m } | j d t d |  d d d d d d	 g ƒ d
 | j ƒ S(   Niÿÿÿÿ(   Rn   Rj   Rk   t   squeues   -os   %ts   -hs   -jR‰   (   Rs   Rn   RŠ   Rr   t   slurm_single_evaluate(   Rj   Rn   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   slurm_evaluater  s    	t   sget   lsft   pbst   pbsprot   condort   slurmc      	   C   s{   t  |  d ƒ s t ‚ | t k r1 t d | ‚ n  t | \ } } | d | d | d | d | ƒ d t ƒ } | j d |  ƒ S(   Nt   __call__s   Unknown platform: %sRG   Rj   Ro   Rz   RC   (   t   hasattrR   t   INTERFACE_FORRN   R*   RA   (   RC   RG   t   platformRj   t   polling_commandt   factoryt   poller_factoryRB   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyRl   ˆ  s    	t
   JobFactoryc           B   s5   e  Z d  Z d d e e e d „ Z d i  d „ Z RS(   s#   
  Creator for Queue.Job objects
  c   	      K   sp   d d l  m } t | d d | d  k r0 d n | d | d | d | rQ | j n | j d	 | d
 t ƒ |  _ d  S(   Niÿÿÿÿ(   Rq   i    RG   Ri   Rj   Rz   RO   RX   RW   (   Rs   Rq   R¢   R)   Ry   t   StdinRc   RB   (	   R   R£   RG   Rj   Rz   t   use_target_filet   preserve_exception_messageRE   Rq   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR   ¦  s    
c         C   s   |  j  j d | d | d | ƒ S(   NRC   RD   RE   (   RB   RA   (   R   RC   RD   RE   (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR    »  s    N(    (   R   R	   R
   R)   R*   R   R    (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyR§   ¡  s   ('   R
   t
   __future__R    R   t   cPickleR    R   R   R   R5   R   R   R   t   objectR   R   R   RA   Ra   Rh   R)   R*   Rc   Rv   R|   R}   R~   R€   R„   Rr   RŒ   R�   R‘   R“   R–   R™   R¢   Rl   R§   (    (    (    s]   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/queuing_system_utils/processing/__init__.pyt   <module>   s°   

O\-	+&&&&&	
	
	
	
	
		