ó
«™xTc           @   s!  d  d l  m Z d  d l m Z d  d l m Z d  d l m Z d  d l m Z d  d l m	 Z	 m
 Z
 d  d l Z d  d l m Z d  d	 l m Z d  d
 l m Z d  d l Z d  d l m Z d  d l m Z d  d l m Z d  d l Z d  d l Z e j j d ƒ Z d „  Z d d „ Z d S(   iÿÿÿÿ(   t   division(   t   crystal(   t   sgtbx(   t   xray(   t   flex(   t   Sorryt	   multi_outN(   t   reflection_file_utils(   t   crystal_symmetry_from_any(   t   pdb(   t   outlier_rejection(   t   f_model(   t   StringIOs  outlier_utils{
  input{
    unit_cell=None
    .type=unit_cell
    space_group=None
    .type=space_group
    xray_data{
      file_name=None
      .type=path
      obs_labels=None
      .type=str
      free_flags=None
      .type=str
    }
    model{
      file_name=None
      .type=path
    }
  }
  outlier_detection{
    protocol=basic *extreme beamstop model
    .type=choice
    parameters{
      basic_wilson{
        level=1E-6
        .type=float
      }
      extreme_wilson{
        level=0.01
        .type=float
      }
      beamstop{
        level=0.001
        .type=float
        d_min=10.0
        .type=float
      }
      model_based{
        level=0.01
        .type=float
      }
    }
  }
  additional_parameters{
    free_flag_generation{
      fraction = 0.10
      .type=float
      max_number = 2000
      .type=float
      lattice_symmetry_max_delta=5.0
      .type=float
      use_lattice_symmetry=true
      .type=bool
    }
  }

  output{
    logfile=outlier_tools.log
    .type=str
    hklout=None
    .type=path
  }
}
c         C   s   d t  ƒ  GHd  S(   Nsá  
usage: %(command_name)s <options>

Options are defined by the following phil scope:

outlier_utils{
  input{
    unit_cell=None                   << Unit cell. Needed for CNS and SHELX formatted Fobs/Iobs files
    space_group=None                 << space group.  Needed for CNS and SHELX formatted Fobs/Iobs files
    xray_data{
      file_name=None                 << reflection file in any format.
      obs_labels=None                << The labels of the the observed labels (CNS and MTZ)
      free_flags=None                << The labels of the Free Flags (CNS and MTZ)
    }
    model{
      file_name=None                 << A PDB file of the model corresponding to the provided xray data
    }
  }
  outlier_detection{
    protocol=basic *extreme beamstop model   << outlier protocol. See below.
    parameters{
      basic_wilson{
        level=1E-6                   << Outlier rejection level for protocol basic
      }
      extreme_wilson{
        level=0.01                   << Outlier rejection level for protocol extreme
      }
      model_based{
        level=0.01                   << outlier rejection level for protocol model
      }
    }
  }
  additional_parameters{
    free_flag_generation{            << If no free flags are provided, they will be generated
      fraction = 0.10                << Fraction of free flags
      max_number = 2000              << Maximum number of free flags
      lattice_symmetry_max_delta=5.0 << Lattice symmetry maximum delta (no need to touch this)
      use_lattice_symmetry=true      << whether or not to use the lattice symmetry in free flag generation
    }
  }

  output{
    logfile=outlier_tools.log        << The output logfile. Has a copy of the screen dump and (possibly) added plots
    hklout=None                      << an mtz file with observations that are not outliers according to the selected protocol
  }
}


A basic run looks like:

%(command_name)s data.file=my_precious_data.mtz \
  data.obs=FOBS data.free=TEST \
  model.file=my_baby.pdb proto=model hklout=new.mtz

A short description of the 3 outlier protocols are described below

1. protocol: basic
   Outliers are reflection for which 1 - P(E^2=>E_obs^2) < level
   For the default value, this results in (approximately) designating
   reflections with E value larger then 3.8 as an outlier.
    see Read, Acta Cryst. (1999). D55, 1759-1764 for details.

2. protocol: extreme
   Outliers are reflections for which 1 - [P(E^2=>E_obs^2)]^Nobs < level
   Nobs is the number of observations. In this manner, the size of the dataset
   is taken into account in the decision if something is an outlier.
   The basic and extrenme protocol usually result in similar outliers.

3. protocol: model
   Calculated amplitudes and estimated values of alpha and beta
   are used to compute the log-likelihood of the observed amplitude.
   The method is inspired by Read, Acta Cryst. (1999). D55, 1759-1764.
   Outliers are rejected on the basis of the assumption that the log
   likelihood differnce log[P(Fmode)]-log[P(Fobs)] is distributed
   according to a Chi-square distribution
   (see http://en.wikipedia.org/wiki/Likelihood-ratio_test ).
   The outlier threshold of relates to the p-value of the
   extreme value distribution of the chi-square distribution.

(   t   vars(   t   command_name(    (    sK   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/scaling/remove_outliers.pyt
   print_helpU   s    Ps   phenix.remove_outliersc   #      C   s  t  |  ƒ d k s6 d |  k s6 d |  k s6 d |  k rF t d | ƒ n¹t ƒ  } d  } d |  k r} | j d d d	 t j ƒ n  t ƒ  } t ƒ  } | j d d
 d	 | ƒ g  } t j	 d d ƒ } xF|  D]>} d  }	 t
 }
 | d k rë t }
 n  t j j | ƒ rby; t j j d | ƒ }	 |	 d  k	 r7| j |	 ƒ t }
 n  WqÁt k
 rN‚  qÁt k
 r^qÁXn_ y8 | j d | ƒ }	 |	 d  k	 r™| j |	 ƒ t }
 n  Wn$ t k
 r°‚  n t k
 rÀn X|
 sÄ | d IJ| d I| IJ| d IJ| Jt d | ƒ ‚ qÄ qÄ Wt j d | ƒ } | j ƒ  } t j j | j j j j ƒ sat d | j j j j ƒ ‚ n  | j j j j d  k	 r¹t j j | j j j j ƒ s¹t d | j j j j ƒ ‚ q¹n  d  } | j j j j d  k	 røt j  d | j j j j ƒ } n  d  } | j j j j d  k	 r7t j  d | j j j j ƒ } n  t! j" d | j j j# d | j j j$ ƒ } | j% ƒ  | j% ƒ  t! j& d  | | g | g ƒ } | j# ƒ  | j j _# t' j( d | j$ ƒ  ƒ | j j _$ t j) d | ƒ } | j* d | ƒ | j j j# d  k rt d ƒ ‚ n  | j j j$ d  k r3t d ƒ ‚ n  | j j j j d  k rZt d ƒ ‚ n  | j j j j d  k rzd GHn  t! j" d | j j j# d | j j j$ ƒ } t+ j, d | d t d g  ƒ } d  } | j- d | j j j j d  | j j j j. d! t d" d# d$ d% ƒ } | j/ ƒ  } | j0 ƒ  } | j1 | j2 ƒ  dW k ƒ } | j3 ƒ  ss| j4 d& | j5 ƒ  d' d  ƒ j6 | ƒ } n  | j1 | j5 ƒ  d k ƒ } | j7 ƒ  d  k	 r¾| j1 | j7 ƒ  d k ƒ } n  | j8 ƒ  rÙ| j9 ƒ  } n | j: ƒ  rôt; | ƒ } n  | j< | ƒ t
 } | j= ƒ  r1| j> ƒ  j6 | ƒ } t } n  | j0 ƒ  } d  } | j j j j? d  k r²| j@ d( | j jA jB jC d) | j jA jB jD d* | j jA jB jE d+ | j jA jB jF ƒ } n¥ | j- d | j j j j d  | j j j j? d! t d" d# d$ d, ƒ } | j= ƒ  r| j> ƒ  } t } n  | j4 d& tG jH | j5 ƒ  d- k ƒ ƒ } | j0 ƒ  } | jI | ƒ } | J| d. IJ| d/ IJ| j% d0 | ƒ | r’| d1 IJn  | J| d2 IJ| d3 IJ| JtJ jK | | d | ƒ} d  } d  } d  } | jL d4 | j jM jN jO jP d5 t ƒ } | jQ d6 | j jM jN jR jP d5 t ƒ } | jS d7 | j jM jN jT jP d8 | j jM jN jT jU d5 t ƒ } | j j j j d  k	 r^	tV j d | j j j j ƒ jW d | ƒ } | d9 IJ| d: IJ| j% d0 | ƒ | JtX jY d; | d< | d= | ƒ } | d> IJ| d? IJ| d@ IJ| J| jZ d | ƒ t ƒ  } | j[ | jX ƒ  d7 | j jM jN j\ jP d5 t dA | ƒ} n  | j j] j^ d  k	 rQ| j jM j_ dB k rÉ	| j j j j d  k rÉ	| dC IJ| dD IJdE | j jM __ qÉ	n  d  } | J| j jM j_ dF k r
| dG IJ| dH IJ| } | jI | ƒ } n  | j jM j_ dE k rU
| dI IJ| dH IJ| } | jI | ƒ } n  | j jM j_ dB k r­
| dJ IJ| dK IJ| | j j] j^ IJ| J| } | jI | ƒ } n  | j jM j_ dL k r| dM IJ| dN IJ| | j j] j^ IJ| J| } | jI | ƒ } n  | j` dO dP ƒ }  |  ja dQ | dO dR ƒ }  |  jb ƒ  jc d | j j] j^ ƒ n  | j j] jd d  k	 rÿt ƒ  }! |! | je ƒ  IJ|! J| d  k	 r¢|! | je ƒ  IJn  tf | j j] jd dS ƒ }" |" jc |! je ƒ  ƒ | J| dT | j j] jd IJ| dU IJ| dV IJn  d  S(X   Ni    s   --helps   --hs   -hR   s   --quiett   labelt   stdoutt   file_objectt
   log_buffert
   home_scopet   outlier_detectiont	   file_namet   args2   ##----------------------------------------------##s   ## Unknown file or keyword:s   Unknown file or keyword: %st   sourcess   File %s can not be foundt	   unit_cellt   space_group_infot   groupt   python_objectt   outs   unit cell not specifieds   space group not specifieds   Xray data not specifieds=   PDB file not specified. Basic wilson outlier rejections only.t   crystal_symmetryt   force_symmetryt   reflection_filest   labelst   ignore_all_zerost   parameter_scopes   outlier_utils.input.xray_datat   parameter_namet
   obs_labelst   datat   sigmast   fractiont   max_freet   lattice_symmetry_max_deltat   use_lattice_symmetryt
   free_flagsi   s   Summary info of observed datas   =============================t   fsC   For outlier detection purposes, the Bijvoet pairs have been merged.s   Constructing an outlier managers   ===============================t   p_basic_wilsont   return_datat   p_extreme_wilsont   levelt   d_mins   Atomic model summarys   ====================t   f_obst   r_free_flagst   xray_structures    Bulk solvent scaling of the datas    ================================s(   Maximum likelihood bulk solvent scaling.t   plot_outt   models8   Model based rejections requested. No model was supplied.sM   Switching to writing out rejections based on extreme value Wilson statistics.t   extremet   basics1   Non-outliers found by the basic wilson statisticss   protocol will be written out.s9   Non-outliers found by the extreme value wilson statisticss%   Non-outliers found by the model baseds)   protocol will be written out to the file:t   beamstops&   Outliers found for the beamstop shadows8   problems detection protocol will be written to the file:t   column_root_labelt   FOBSt   miller_arrayt   Free_R_Flagt   ws   A logfile named %s was created.s+   This logfile contains the screen output ands)   (possibly) some ccp4 style loggraph plots(   i    i    i    (g   t   lenR   R   t   Nonet   registert   sysR   R   t   master_paramst!   command_line_argument_interpretert   Falset   Truet   ost   patht   isfilet   iotbxt   philt   parset   appendt   KeyboardInterruptt	   Exceptiont   processR   t   fetcht   extractt   existst   outlier_utilst   inputt	   xray_dataR   R7   R   t   extract_fromR   t   symmetryR   t   space_groupt   show_summaryt   select_crystal_symmetryR   R   t   formatt   showR   t   reflection_file_servert   get_xray_dataR%   t   infot
   map_to_asut   selectt   indicest   sigmas_are_sensiblet   customized_copyR&   t   set_observation_typeR'   t   is_xray_intensity_arrayt	   f_sq_as_ft   is_complex_arrayt   abst   set_infot   anomalous_flagt   average_bijvoet_matesR,   t   generate_r_free_flagst   additional_parameterst   free_flag_generationR(   t
   max_numberR*   R+   R   t   boolt
   common_setR
   t   outlier_managert   basic_wilson_outliersR   t
   parameterst   basic_wilsonR1   t   extreme_wilson_outlierst   extreme_wilsont   beamstop_shadow_outliersR:   R2   R	   t   xray_structure_simpleR   t   managert   update_solvent_and_scalet   model_based_outlierst   model_basedt   outputt   hkloutt   protocolt   as_mtz_datasett   add_miller_arrayt
   mtz_objectt   writet   logfilet   getvaluet   open(#   t   argsR   t   logR6   t   string_buffert   string_buffer_plotst   phil_objectst   argument_interpreterR   t   command_line_paramst   arg_is_processedt   effective_paramst   paramst   hkl_xst   pdb_xst   phil_xst   combined_xst
   new_paramst   xray_data_serverR=   Ra   t   merged_anomalousR,   Ru   t   basic_arrayt   extreme_arrayt   model_based_arrayt   beamstop_arrayR7   t   f_model_objectt   output_arrayt   new_set_of_free_flagst   mtz_datasett	   final_logt   outfile(    (    sK   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/scaling/remove_outliers.pyt   run§   sÖ   6						    

"			

!													



							!	

		


	









					
(   t
   __future__R    t   cctbxR   R   R   t   cctbx.array_familyR   t   libtbx.utilsR   R   t
   iotbx.philRK   R   R   R	   t   mmtbx.scalingt   mmtbxR
   R   t   libtbx.str_utilsR   RC   RH   RL   RM   RD   R   R¦   (    (    (    sK   /public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/scaling/remove_outliers.pyt   <module>   s$   	@		R