ó
a#nSc           @   sT  d  Z  d d l m Z d d l Z d d l m Z m Z d d l m Z m	 Z	 d d l
 m Z d d l m Z d d l m Z d d l Z d d l Z d d l Z d d l Z e j ƒ  Z d	 Z d
 Z d „  Z d d
 d d d „ Z d „  Z d d d „ Z d d „ Z d „  Z d d „ Z d „  Z e d k rPe  e j! ƒ d k  rQe ƒ  n  d j" e j! ƒ e _# e e ƒ e e ƒ e d d
 ƒ e e ƒ Z$ e  e$ ƒ Z% e% d
 k r�xŸe& e% ƒ D]ÎZ' e j( j) d e' d
 e% f ƒ e e e$ e' d d
 ƒZ* e j+ re j, re e e* ƒ Z- n	 e* g Z- x e- D] Z. e. j/ ƒ  q)We j0 ræe  e j1 ƒ Z2 e2 d
 k r~e j0 Z0 e- d j3 e0 ƒ qæxe e& e2 ƒ D]T Z' e4 d e j1 e' ƒ Z5 e- e' Z6 d e j0 j7 d ƒ d e5 f Z0 e6 j3 e0 ƒ q‹Wn  e j8 r¸e j9 r¸e d e j9 e j8 f ƒ e  e j1 ƒ d
 k r4e d ƒ n  xO e& e  e- ƒ ƒ D]8 Z' e j: e- e' ƒ e _6 e j; d e j9 d  e j8 ƒ qGWq¸q¸WqPe% d
 k rCe e e$ d d d
 ƒZ* e j+ rÖe j, rÖe e e* ƒ Z- n	 e* g Z- x e- D] Z. e. j/ ƒ  qæWe j0 r£e  e j1 ƒ Z2 e2 d
 k r;e j0 Z0 e- d j3 e0 ƒ q£xe e& e2 ƒ D]T Z' e4 d e j1 e' ƒ Z5 e- e' Z6 d e j0 j7 d ƒ d e5 f Z0 e6 j3 e0 ƒ qHWn  e j8 rMe j9 rMe d e j9 e j8 f ƒ e  e- ƒ d
 k rîe d ƒ n  xO e& e  e- ƒ ƒ D]8 Z' e j: e- e' ƒ e _6 e j; d e j9 d  e j8 ƒ qWqMqPe d! ƒ n  d S("   s°   command line utility for growing composite models

**Usage**

  _GrowComposite [optional args] filename_

**Command Line Arguments**

  - -n *count*: number of new models to build

  - -C *pickle file name*:  name of file containing composite upon which to build.

  - --inNote *note*: note to be used in loading composite models from the database
      for growing

  - --balTable *table name*:  table from which to take the original data set
     (for balancing)

  - --balWeight *weight*: (between 0 and 1) weighting factor for the new data
     (for balancing). OR, *weight* can be a list of weights

  - --balCnt *count*: number of individual models in the balanced composite
     (for balancing)

  - --balH: use only the holdout set from the original data set in the balancing
     (for balancing)

  - --balT: use only the training set from the original data set in the balancing
     (for balancing)

  - -S: shuffle the original data set
     (for balancing)

  - -r: randomize the activities of the original data set
     (for balancing)

  - -N *note*: note to be attached to the grown composite when it's saved in the
     database

  - --outNote *note*: equivalent to -N

  - -o *filename*: name of an output file to hold the pickled composite after
     it has been grown.
     If multiple balance weights are used, the weights will be added to
     the filenames.

  - -L *limit*: provide an (integer) limit on individual model complexity
  
  - -d *database name*: instead of reading the data from a QDAT file,
     pull it from a database.  In this case, the _filename_ argument
     provides the name of the database table containing the data set.

  - -p *tablename*: store persistence data in the database
     in table *tablename*

  - -l: locks the random number generator to give consistent sets
     of training and hold-out data.  This is primarily intended
     for testing purposes.

  - -g: be less greedy when training the models.

  - -G *number*: force trees to be rooted at descriptor *number*.

  - -D: show a detailed breakdown of the composite model performance
     across the training and, when appropriate, hold-out sets.
     
  - -t *threshold value*: use high-confidence predictions for the final
     analysis of the hold-out data.

  - -q *list string*:  Add QuantTrees to the composite and use the list
     specified in *list string* as the number of target quantization
     bounds for each descriptor.  Don't forget to include 0's at the
     beginning and end of *list string* for the name and value fields.
     For example, if there are 4 descriptors and you want 2 quant bounds
     apiece, you would use _-q "[0,2,2,2,2,0]"_.
     Two special cases:
       1) If you would like to ignore a descriptor in the model building,
          use '-1' for its number of quant bounds.
       2) If you have integer valued data that should not be quantized
          further, enter 0 for that descriptor.

  - -V: print the version number and exit

iÿÿÿÿ(   t   RDConfigN(   t	   DataUtilst	   SplitData(   t   ScreenCompositet   BuildComposite(   t   AdjustComposite(   t	   DbConnect(   t   CompositeRuns   0.5.0i   c         C   s!   t  r t j j d |  ƒ n  d S(   s¥    emits messages to _sys.stdout_
    override this in modules which import this one to redirect output

    **Arguments**

      - msg: the string to be displayed

  s   %s
N(   t   _verboset   syst   stdoutt   write(   t   msg(    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   messagen   s    	 i    c   +      C   s©  t  j ƒ  |  _ | d$ k rÄ |  j j ƒ  } |  j d k rI | d |  _ n  |  j d k rj t j	 | ƒ } qÄ |  j
 g  k r‘ | |  _ |  j ƒ  } qÄ t j |  j | d |  j d |  j d |  j ƒ} n  | j ƒ  } | j } t j | ƒ g  }	 |  j d k rt j | d d d |  ƒn+ |  j d k rBt j | d d	 d |  ƒn  | j ƒ  }
 |
 } t | ƒ } t d
 | ƒ t d t | d	 ƒ d ƒ | j ƒ  } | j } t d | d ƒ } |  j r»d d l m } m } |  j
 g  k rüd d l m  } | j! } n d d l m" } | j# } | j$ } | j } | rC| j% | j& ƒ  ƒ n  | j' | | d	 g | d | d | d |  j( d |  j) d |  j* d d	 d | d |  j
 d |  j+ d |  j, d | d t- ƒnH d d l. m } | j$ } | j' | | d	 g | d |  j( d | d d	 ƒ| j/ ƒ  | j0 ƒ  | j1 ƒ  \ } } } t2 j3 | ƒ } t2 j3 | ƒ } | j& ƒ  | _4 x. t5 t | ƒ ƒ D] } | | j6 | j4 ƒ qlW| | } t7 | ƒ t7 | ƒ } | | } | | } t2 j8 | | ƒ } t7 | ƒ t7 | ƒ } t- rt d d | d | f ƒ n  |  j9 r+| j: | d  d	 ƒn  g  } |  j; sÇt- rMt d! ƒ n  t< j= | |
 | ƒ } t- r¥t d" t | ƒ d t> t | ƒ ƒ t> t |
 ƒ ƒ f ƒ n  t> t | ƒ ƒ t |
 ƒ t? _@ n  |  j; r¥t- rãt d# ƒ n  tA jB t | j ƒ  ƒ | | | d |  jC ƒ }  |  \ }! }" }# }$ }% }& }' t |
 ƒ }( |! |" }) t> |" ƒ |) t? _@ |$ t? _D |% t? _E tF |' ƒ t? _G |) |( }* |* d	 k r¥t> |* ƒ |( t? _H q¥n  | S(%   s�   does the actual work of building a composite model

    **Arguments**

      - details:  a _CompositeRun.CompositeRun_ object containing details
        (options, parameters, etc.) about the run

      - composite: the composite model to grow
      
      - progressCallback: (optional) a function which is called with a single
        argument (the number of models built so far) after each model is built.

      - saveIt: (optional) if this is nonzero, the resulting model will be pickled
        and dumped to the filename specified in _details.outName_

      - setDescNames: (optional) if nonzero, the composite's _SetInputOrder()_ method
        will be called using the results of the data set's _GetVarNames()_ method;
        it is assumed that the details object has a _descNames attribute which
        is passed to the composites _SetDescriptorNames()_ method.  Otherwise
        (the default), _SetDescriptorNames()_ gets the results of _GetVarNames()_.

      - data: (optional) the data set to be used.  If this is not provided, the
        data set described in details will be used.
        
    **Returns**

      the enlarged composite model


  t    s   .pklt	   quantNamet   usert   passwordi   t   shufflet
   runDetailsi    s   Training with %d exampless   	%d descriptorsi   iÿÿÿÿ(   t   CrossValidatet	   PruneTree(   t   BuildQuantTree(   t   ID3t   buildDrivert   prunert   nTriest   pruneItt
   lessGreedyt   needsQuantizationt   treeBuildert   nQuantBoundst   startAtt   maxDeptht   progressCallbackt   silent(   R   s>   # Overall Average Error: %%% 5.2f, Average Deviation: %%% 6.2fg      Y@t   verboses   Testing all exampless)   %d examples (%% %5.2f) were misclassifieds   
Entire data set:N(I   t   timet   asctimet   rundatet   Nonet	   tableNamet   stript   outNamet   dbNameR   t   BuildQuantDataSett   qBoundst
   GetDataSett   DBToQuantDatat
   qTableNamet   dbUsert
   dbPasswordt   GetNPtst   _randomSeedt   InitRandomNumberst   shuffleActivitiest   RandomizeActivitiest   randomActivitiest   GetNamedDatat   lenR   t   GetNVarst   nPossibleValst   ranget   useTreest   rdkit.ML.DecTreeR   R   R   t   QuantTreeBootR   t   ID3Boott   CrossValidationDrivert   SetInputOrdert   GetVarNamest   Growt   nModelsR   R   R    t
   limitDepthR   t   rdkit.ML.Neuralt   AverageErrorst
   SortModelst
   GetAllDatat   numpyt   arrayt	   _varNamest   xranget	   NameModelt   sumt   sqrtt
   bayesModelt   Traint   detailedResR   t   testallt   floatt   _runDetailst   overall_errorR   t   ShowVoteResultst	   thresholdt   overall_correct_conft   overall_incorrect_conft   reprt   overall_result_matrixt   overall_fraction_dropped(+   t   detailst	   compositeR"   t   saveItt   setDescNamest   datat   fNamet	   nExamplest   seedt   testExamplest   namedExamplest   trainExamplest   nVarsR=   t   attrsR   R   R   t   builderR   t   driverR   t	   modelListt   countst   avgErrst   it   weightedErrst
   averageErrt   devst   avgDevt   badExamplest   wrongt   resTupt   nGoodt   nBadt   nSkipt   avgGoodt   avgBadt   avgSkipt   voteTabt   nPtst   nClasst   nRej(    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   GrowIty   s¾     										"




		="	
		
c         C   s¸   g  } |  j  r† |  j r† t |  j |  j  ƒ } | j d d d d |  j ƒ } xe | D], } | d } | j t j t | ƒ ƒ ƒ qS Wn. |  j	 r´ | j t j
 t |  j	 d ƒ ƒ ƒ n  | S(   Nt   fieldst   MODELt   wheres   where note='%s'i    t   rb(   t   persistTblNamet   inNoteR   R,   t   GetDatat   appendt   cPicklet   loadst   strt   composFileNamet   loadt   open(   Rb   t   rest   connt   mdlst   rowt   rawD(    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   GetComposites  s    
#	%c         C   sñ  |  j  s |  j  t | ƒ k r# | St d ƒ | d k rš t d ƒ |  j j ƒ  } |  j } | |  _ |  j } |  j |  _ |  j	 ƒ  } | |  _ | |  _ n  | d k rª | S| j
 |  _ | j |  _ t j |  j ƒ |  j d k rý t j | d d d |  ƒn+ |  j d k r(t j | d d d |  ƒn  | j ƒ  } |  j sF|  j rCt j t | ƒ |  j d d ƒ\ } }	 g  | D] }
 | |
 ^ qt} g  |	 D] }
 | |
 ^ q‘} |  j d k r't j | |  j |  j d	 d
 d ƒ\ } } g  | D] }
 | |
 ^ qç} | g  | D] }
 | |
 ^ q7} | } n  |  j rI| | } } qIn | } | } g  | j ƒ  D] }
 |
 j ƒ  ^ q\} d } | d k rŸt d ƒ |  j	 ƒ  } n  | d k r¯| S| j
 |  _ | j |  _ t j |  j ƒ |  j d k rt j | d d d |  ƒn+ |  j d k r-t j | d d d |  ƒn  | j ƒ  } g  | j ƒ  D] }
 |
 j ƒ  ^ qF} d } g  } |  j } t | ƒ t j  t j! f k r�| f } n  xM | D]E } t d | ƒ | j" t# j$ | | | | |  j  d | d | ƒƒ q¤W| S(   s¥   balances the composite using the parameters provided in details

   **Arguments**

     - details a _CompositeRun.RunDetails_ object

     - composite: the composite model to be balanced

     - data1: (optional) if provided, this should be the
       data set used to construct the original models

     - data2: (optional) if provided, this should be the
       data set used to construct the new individual models

  s   Balancing Composites   	Reading First Data Seti   R   R   i    R#   g        iÿÿÿÿt   indicesOnlys   	Reading Second Data Sets   	Balancing with Weight: %.4ft   names1t   names2N(%   t   balCntR;   R   R(   t   balTableR*   R)   R,   t   balDbR/   t
   _splitFract	   splitFracR5   t
   randomSeedR   R6   R7   R8   R9   R:   t   balDoHoldoutt
   balDoTrainR   t   SplitIndicest
   filterFract
   FilterDatat	   filterValRE   t   uppert	   balWeightt   typet   typest	   TupleTypet   ListTypeRŽ   R   t   BalanceComposite(   Rb   Rc   t   data1t   data2Rg   t   tmpR,   Rk   t   trainIdxt   testIdxt   xRl   Rj   t   tempt   dataSet1t   cols1t   dataSet2t   cols2R•   t   weightst   weight(    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyR°     s†    

				!		%
%		c         C   s8   d t  GH|  r4 d d l } d GHd j | j ƒ GHn  d S(   s    prints the version number

  s#   This is GrowComposite.py version %siÿÿÿÿNs   command line was:t    (   t   __VERSION_STRINGR	   t   joint   argv(   t   includeArgsR	   (    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   ShowVersionw  s
    	c          C   s"   d d l  }  t GH|  j d ƒ d S(   sM    provides a list of arguments for when this is used from the command line

  iÿÿÿÿN(   R	   t   __doc__t   exit(   R	   (    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   Usage�  s    c         C   s"   |  d k r t }  n  t j |  ƒ S(   s    initializes a details object with default values

      **Arguments**

        - details:  (optional) a _CompositeRun.CompositeRun_ object.
          If this is not provided, the global _runDetails will be used.

      **Returns**

        the initialized _CompositeRun_ object.


  N(   R(   RY   R   t   SetDefaults(   R   (    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyRÇ   ‰  s     	c         C   sH  d d l  } | j  t j d d d d d d d	 d
 d d g ƒ \ } } d |  _ d |  _ d |  _ d. |  _ d |  _ d |  _ d |  _	 d |  _
 xŽ| D]†\ } } | d k r¿ t | ƒ |  _ q• | d k r× | |  _ q• | d k rï | |  _ q• | d k r@t | ƒ |  _ t |  j ƒ t j t j f k r|  j f |  _ qq• | d k r^t | ƒ |  _ q• | d k rvd |  _ q• | d k rŽd |  _	 q• | d k r¦| |  _
 q• | d k r¾| |  _ q• | d k sÖ| d k râ| |  _ q• | d k rú| |  _ q• | d k r| |  _ q• | d k r*d |  _ q• | d k rBd |  _ q• | d k rXt ƒ  q• | d  k rpd |  _ q• | d! k rˆd |  _ q• | d" k r¦t | ƒ |  _ q• | d# k r¾| |  _ q• | d$ k rÖd |  _ q• | d% k rôt | ƒ |  _ q• | d& k rd |  _ q• | d' k r*t | ƒ |  _  q• | d( k r�t | ƒ } t | ƒ t j t j f k slt! d) ƒ ‚ | |  _" | |  _# q• | d* k rÞt | ƒ } t | ƒ t g  ƒ t d/ ƒ g k sÉt! d+ ƒ ‚ | |  _$ | |  _% q• | d, k rt& ƒ  t j' d ƒ q• t j( d- I| IJt ƒ  q• W| d |  _) |  j
 sD|  j |  _
 n  d S(0   sŽ    parses command line arguments and updates _runDetails_

      **Arguments**

        - runDetails:  a _CompositeRun.CompositeRun_ object.

  iÿÿÿÿNi   s)   P:o:n:p:b:sf:F:v:hlgd:rSTt:Q:q:DVG:L:C:N:s   inNote=s   outNote=s	   balTable=s
   balWeight=s   balCnt=t   balHt   balTs   balDb=R   g      à?i    s   -ns   -Cs
   --balTables   --balWeights   --balCnts   --balHs   --balTs   --balDbs   --inNotes   -Ns	   --outNotes   -os   -ps   -rs   -Ss   -hs   -ls   -gs   -Gs   -ds   -Ts   -ts   -Ds   -Ls   -qs4   bad argument type for -q, specify a list as a strings   -Qs4   bad argument type for -Q, specify a list as a strings   -Vs   bad argument:(   g      à?(    (*   t   getoptR	   RÁ   RŒ   R’   RŸ   R«   Rž   R¤   R¥   R    t   intRG   t   evalR¬   R­   R®   R¯   t   noteR+   R‹   R9   R7   RÆ   t
   lockRandomR   R    R,   R?   RX   R\   RV   RH   t   AssertionErrort   qBoundCountR.   t   activityBoundst   activityBoundsValsRÃ   RÅ   t   stderrR)   (   R   RÊ   t   argst   extrat   argt   valR.   (    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt	   ParseArgsš  sž    								!
*	0		t   __main__i   R¾   RÂ   sT   ---------------------------------
	Doing %d of %d
---------------------------------
Re   id   s	   %s.%d.pkls   .pkls   Updating results table %s:%ssD   WARNING: updating results table with models having different weightst   dbt   tables   No models found(<   RÄ   t   rdkitR    RM   t   rdkit.ML.DataR   R   t   rdkit.MLR   R   t   rdkit.ML.CompositeR   t   rdkit.Dbase.DbConnectionR   R   R	   R�   R%   R­   RY   R¿   R   R   R(   R†   Rš   R°   RÃ   RÆ   RÇ   RØ   t   __name__R;   RÁ   RÀ   t   cmdt
   initModelsRG   R>   Rt   RÓ   R   Rc   RŸ   Rž   t
   compositest   mdlt   ClearModelExamplesR+   R«   t   nWeightst   PickleRË   R½   t   modelt   splitR‹   R,   t   dumpst   Store(    (    (    sX   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/python2.7/site-packages/rdkit/ML/GrowComposite.pyt   <module>_   s    0	‘	`
		X


			
 *			
 &