ó
ã™xTc           @   sP  d  d l  m Z d  d l m Z d  d l m Z d  d l Z d  d l m Z m Z m	 Z	 d  d l m
 Z
 m Z d  d l m Z m Z d  d l m Z d  d l Z d  d l Z d  d l Z d  d l Z d	 Z d
 Z d Z d Z e d „ Z d „  Z e d d „ Z d „  Z d e f d „  ƒ  YZ  d „  Z! d „  Z" e# d k rLe" ƒ  d GHn  d S(   iÿÿÿÿ(   t   division(   t   easy_run(   t
   test_utilsN(   t   import_python_objectt   Sorryt	   multi_out(   t
   group_argst   Auto(   t   Poolt	   cpu_count(   t   StringIOi    i   i   iÈ   c      	   C   sB  t  j j |  ƒ } | d  k r1 t d |  ƒ ‚ n4 t j j t j j | d ƒ ƒ se t d |  ƒ ‚ n  t	 d d |  d d d d d	 d ƒ j
 } t | t ƒ s± t | t ƒ s± t ‚ t  j j |  ƒ } | d  k	 rÛ | d  k	 sá t ‚ g  } t d
 t d t d | ƒ } x9 t j d | d | d | d | ƒ D] } | j | ƒ q'W| S(   Ns!   '%s' is not a valid CCTBX module.s   run_tests.pys   %s/run_tests.py does not exist.t   import_paths   %s.run_tests.tst_listt   error_prefixt    t   target_must_bet	   where_strt   verboset   quickt   valgrindt   cot	   build_dirt   dist_dirt   tst_list(   t   libtbxt   envt	   dist_patht   NoneR   t   ost   patht   isfilet   joinR   t   objectt
   isinstancet   tuplet   listt   AssertionErrort   under_buildR   t   Falset   TrueR   t   iter_tests_cmdt   append(   t   module_nameR   R   R   t
   build_patht   commandsR   t   cmd(    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   get_module_tests   s2    !
$	c         C   sµ   t  j j |  ƒ s% t d |  ƒ ‚ n  g  } xƒ t  j |  ƒ D]r \ } } } x` | D]X } t  j j | ƒ \ } } | j d ƒ rQ | d k rQ | j t  j j | | ƒ ƒ qQ qQ Wq; W| S(   Ns   '%s' is not a directory.t   tsts   .pys   .cshs   .sh(   s   .pys   .cshs   .sh(	   R   R   t   isdirt   RuntimeErrort   walkt   splitextt
   startswithR(   R   (   t   dir_namet	   all_testst   roott   dirnamest	   filenamest	   file_namet   baset   ext(    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt
   find_tests0   s    'c         C   s«   | d  k r t j } n  yH t j ƒ  } t j d |  ƒ } t j ƒ  | | _ t | ƒ | _ | SWnD t	 k
 r¦ d j
 t j t j ƒ  d ƒ ƒ } t j j | ƒ d  SXd  S(   Nt   commands   
i   (   R   t   syst   stdoutt   timeR   t   fully_bufferedt	   wall_timet   evaluate_outputt   error_linest   KeyboardInterruptR   t	   tracebackt	   format_tbt   exc_infot   write(   R=   t	   verbosityt   outt   t0t
   cmd_resultt   traceback_str(    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   run_command;   s    			"c         C   s�   t  j j d ƒ } g  } | rŒ d d l m } |  j } |  j } xI t | | ƒ D]4 \ } } | j | | j	 ƒ  ƒ rQ | j
 | ƒ qQ qQ Wn  | S(   Nt   phenix_regressioniÿÿÿÿ(   t   find_errors_and_warnings(   R   R   t
   has_modulet   phenix_regression.command_lineRQ   t   stdout_linest   stderr_linest	   enumeratet   is_error_or_warningt   lowerR(   (   RM   t   have_regressiont	   bad_linesRQ   RK   t   errt   it   line(    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyRC   T   s    		t   run_command_listc           B   s5   e  Z d  e j d e e d „ Z d „  Z d „  Z	 RS(   i   c         C   s'  | d  k r t ƒ  } n  t ƒ  |  _ | |  _ |  j j d | ƒ |  j j d | ƒ | |  _ | d k |  _ g  |  _ xA | D]9 } | |  j k r£ |  j j	 | ƒ q{ |  j d | IJq{ W| t
 k rÐ t ƒ  } n  t | t |  j ƒ ƒ } |  j d IJ|  j d I| IJ|  j d It |  j ƒ IJt j ƒ  } | d k rNt d	 | ƒ }	 n  g  |  _ xg |  j D]\ }
 | d k r˜|	 j t |
 | | g d
 |  j ƒqat |
 d | d | ƒ} |  j | ƒ qaW| d k r5z= y |	 j ƒ  Wn( t k
 r|  j d IJ|	 j ƒ  n XWd  |	 j ƒ  X|  j d t |  j ƒ IJn  t j ƒ  } |  j d IJ|  j d | | IJ|  j d IJd |  _ d |  _ d } d |  _ g  } g  } g  } g  } | rÎd d l m } m } g  } n  xm|  j D]b} |  j d 7_ | j	 | j ƒ | j d k r+|  j d 7_ | j	 | ƒ nV t | j  ƒ d k r_|  j d 7_ | j	 | ƒ n  t | j! ƒ d k r�| d 7} n  | j t" k r³| j	 | j# ƒ | j	 | j ƒ n  | rØ| d | j# d | j# d | j d d j | j$ ƒ d d j | j! ƒ ƒ } | j d k r*| j% d d | j ƒ n  | j	 | ƒ qØqØW| r�| d d | ƒ} t& d d ƒ �# } | | j' | g d t( ƒIJWd  QXn  t) j* j+ d ƒ r	d d  l, m- } |  j d! IJ| j. | j/ | ƒ d" d# ƒ} | j0 d$ |  j d% d& d' d( ƒ |  j d IJn  t | ƒ d k rž|  j d IJ|  j d) t" IJt1 t2 | | ƒ ƒ } | j3 d* „  ƒ x- | D]% \ } } |  j d& | d+ | IJqeW|  j d, IJn  t | ƒ d k rw|  j d IJ|  j d- IJx� | D]… } |  j d IJ|  j | j# d. | j IJx" | j! D] } |  j d& | IJq	Wx" | j  D] } |  j d& | IJq.W|  j d IJqÑW|  j d/ IJ|  j d IJn  |  j d0 IJ|  j d1 I|  j IJ|  j d2 I|  j IJ|  j d3 I|  j IJ|  j d4 I| IJ|  j t |  j ƒ k r#|  j d5 d6 IJ|  j d7 IJ|  j d5 d6 IJn  d  S(8   NR?   t   logi    s     test %s repeated, skippings   
  Starting command lists       NProcs :s       Cmds   :i   t	   processest   callbackRJ   RK   s%   Caught KeyboardInterrupt, terminatings   
Processes have joined : %d
R   s   Elapsed time: %.2fsiÿÿÿÿ(   t	   TestSuitet   TestCaset   namet	   classnamet   elapsed_secs   
t   stderrt   messages   exit code %ds   libtbx.run_tests_parallelt
   test_casess
   output.xmlt   wbt   prettyprintt   scitbx(   t   flexs   Distribution of test runtimes:t   n_slotsi   t   ft   prefixs     t   format_cutoffss   %.1fss5   WARNING: the following jobs took at least %d seconds:c         S   s   t  |  d | d ƒ S(   Ni   (   t   cmp(   t   xt   y(    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   <lambda>Ì   s    s    : %.1fssI   Please try to reduce overall runtime - consider splitting up these tests.sS   ERROR: the following jobs returned non-zero exit codes or suspicious stderr output:s   (exit code %d):s#   Please verify these tests manually.s   Summary:s      Tests run                    :s      Failures                     :s      Warnings (possible failures) :s      Stderr output (discouraged)  :t   *iP   s"     WARNING: NOT ALL TESTS FINISHED!(4   R   t   null_outR   RK   R_   t   registerRJ   t   quiett   cmd_listR(   R   R	   t   mint   lenR@   R   t   resultst   apply_asyncRO   t   save_resultt   closeRE   t	   terminateR   t   finishedt   warningt   failuret	   junit_xmlRb   Rc   RB   t   return_codeRD   RU   t   max_timeR=   RT   t   add_failure_infot   opent   to_xml_stringR&   R   R   RR   t   scitbx.array_familyRm   t	   histogramt   doublet   showR"   t   zipt   sort(   t   selfRz   t   nprocsRK   R_   RJ   t   output_junit_xmlR,   t   t_startt   poolR=   t   rct   t_endt   extra_stderrt   failurest	   long_jobst   long_runtimest   runtimesRb   Rc   Ri   t   resultt   tct   tsRo   Rm   t   histt   jobs_and_timingst   runtimeR]   (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   __init__a   sî     										%
c         C   s>   | d  k r |  j d IJd  S|  j j | ƒ |  j | ƒ d  S(   Ns   ERROR: job returned None(   R   RK   R}   R(   t   display_result(   R‘   R�   (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyR   è   s
    c         C   s�  t  } | j d k s* t | j ƒ d k r3 t } n  |  j r[ t | j ƒ d k r[ t } n  | sp |  j t k r| |  j } n	 |  j	 } | d IJ| d | j
 IJ| d | j IJ|  j t k rÙ | d j | j ƒ IJn  d } x3 | j D]( } | j d ƒ d k ré | d 7} qé qé W| d	 | IJt | j ƒ d k rº| d
 d IJ| d j | j ƒ IJ|  j t k rºt j d I| j
 IJt j d j | j ƒ IJt j d IJt j j ƒ  qºn  t | j ƒ d k rò| d IJ| d j | j ƒ IJn  | d | j IJ| j d k rs| d d IJ| d Id d Id IJ| d d IJ| d Id d Id IJ| d d IJn  | j ƒ  d  S(   Ni    R   s   
#command : "%s"s   #return_code : %ss   
t   OKiÿÿÿÿi   s   Found %d "OK"s   #stderr-i
   t   #s   possible errors:s   time : %5.2fsi<   t   !iN   s   !!s   WARNING i	   s   !!  %-71s !!s   TEST TAKES MORE THAN A MINUTE(   R%   R†   R|   RD   R&   Ry   RJ   t   EXTRA_VERBOSERK   R_   R=   R   RT   t   findRU   t   QUIETR>   Rg   t   flushRB   (   R‘   R�   t   alertRK   t   oksR]   (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyR¤   ï   sJ    $			

N(
   t   __name__t
   __module__R>   R?   R   t   DEFAULT_VERBOSITYR%   R£   R   R¤   (    (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyR^   `   s   �	c         C   s*  g  } g  } g  } xµ |  D]­ } | j  d ƒ rW d | } | | k rÆ | j | ƒ qÆ q | j  d ƒ su | j  d ƒ r¹ t j | t j ƒ sš | j | ƒ qÆ | | k rÆ | j | ƒ qÆ q | j | ƒ q Wt | ƒ d k rø t d d j | ƒ ƒ ‚ n  t | ƒ d k r&t d d j | ƒ ƒ ‚ n  | S(	   Ns   .pys   libtbx.python "%s"s   .shs   .cshi    s¯   The following files could not be recognized as programs:

  %s

Please use the extensions .py, .sh, or .csh for all tests.  Shell scripts will
also need to be made executable.s   
  sd   The following shell scripts are not executable:

  %s

Please enable execution of these to continue.(   t   endswithR(   R   t   accesst   X_OKR|   R0   R   (   t   filesR+   t   non_executablet   unrecognizedR9   R,   (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   make_commands  s*    
c          C   s-  t  d d ƒ }  t d d ƒ } | j d ƒ | j ƒ  t j d ƒ t d d ƒ } | j d ƒ | j ƒ  t d d ƒ } | j d	 ƒ | j ƒ  t ƒ  } t d
 d d d g d d d |  d | ƒ|  j ƒ  d | j ƒ  k sá t	 ‚ d | j ƒ  k sù t	 ‚ d | j ƒ  k st	 ‚ d | j ƒ  k s)t	 ‚ d  S(   Nt   test_utils_parallel_zlogRj   s   test_parallel.cshs$   #!/bin/csh

echo "hello world"

exits   chmod 755 test_parallel.cshs   fails.pys    print "this will crash"
assert 0s   works.pys   print "hello world"
print "OK"
s   libtbx.python fails.pys   libtbx.python works.pys   csh test_parallel.cshs   ./test_parallel.cshR’   i   R_   RK   s   ERROR - return code 1s"     Tests run                    : 4s"     Failures                     : 1s"     Warnings (possible failures) : 0(
   t   fileR‰   RI   R€   R   t   callR
   R^   t   getvalueR#   (   R_   t   f0t   f1t   f2RK   (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   exercise8  s8    


		
t   __main__R¥   ($   t
   __future__R    R   R   R   t   libtbx.load_envt   libtbx.utilsR   R   R   R   R   t   multiprocessingR   R	   t	   cStringIOR
   RF   R@   R   R>   Rª   R°   R¨   R‡   R%   R-   R<   R   RO   RC   R   R^   R·   R¿   R®   (    (    (    sH   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/libtbx/test_utils/parallel.pyt   <module>   s4   		·	!	&