xTc@ sddlmZddlZddlZddlmZddlm Z ddl m Z ddlm Z ddlm Z ddlZddlZddlZddlZejjdd eZed Zd efd YZd ZdZdZdZdedZdZidd6dd6dd6Z dedZ!ee"dZ#dZ$defdYZ%d efd!YZ&d"efd#YZ'e(d$kreej)d%ndS(&i(tdivisionN(t make_header(tifloor(tSorry(t easy_pickle(tadopt_init_argss  simulate_data { pdb_file = None .type = path .help = Model file. If no reflections file is given, data will be generated starting from F(model). Otherwise it will only be used to report scaling statistics. hkl_file = None .type = path .help = Data file. If defined, the extracted amplitudes will be truncated. Any R-free flags will be propagated to the output file. data_label = None .type = str .help = Label for amplitudes or intensities in hkl_file (optional). d_min = 3.0 .type = float .help = Resolution cutoff of output data. random_seed = 90125714 .type = int output_file = None .type = path write_modified_pdb = True .type = bool r_free_flags { file_name = None .type = path .help = File containing R-free flags. Can be left blank if hkl_file is defined and contains R-free flags. label = None .type = str .help = Label for R-free flags in r_free_file (or hkl_file). fraction = 0.05 .type = float .help = Percent of reflections to flag for R-free (ignored if already available). missing_flags = *extend discard .type = choice(multi=False) .help = Handling of reflections for which R-free flags are not present. } crystal_symmetry { space_group = None .type = space_group .help = Space group of output data. Ignored if reflections are used as input. unit_cell = None .type = unit_cell .help = Unit cell of output data. Ignored if reflections are used as input. } modify_pdb { remove_waters = True .type = bool .help = Strip waters from input model. remove_alt_confs = False .type = bool .help = Strip alternate sidechains (and reset occupancy to 1). convert_to_isotropic = False .type = bool .help = Convert all atoms to isotropic before calculating F(model). set_mean_b_iso = None .type = float .help = Scale atomic B-factors to have this mean value. set_wilson_b = False .type = bool .help = Scale atomic B-factors to have a mean equal to the mean Wilson B-factor for this resolution (+/- 0.2A) } truncate .help = Options for data truncation at lower resolution. { add_b_iso = None .type = float .help = Isotropic B-factor to be added to data. add_b_aniso = 0 0 0 0 0 0 .type = floats(size=6) .help = Anisotropic B-factor to be added to data. Severely anisotropic data might have an anisotropic B of 80,80,200,0,0,0. apply_b_to_sigmas = True .type = bool .help = If True, the B-factor scaling will be done on experimental sigmas (if present) as well as amplitudes. add_random_error_percent = None .type = float .help = Adds random noise as a percentage of amplitude, evenly across all resolutions. This is probably inferior to the sigma-based noise generation. remove_cone_around_axis = None .type = float .help = Radius in degrees of cone of missing data around the axis of rotation (data collection pathology). If specified, axis_of_rotation must be defined. axis_of_rotation = None .type = ints(size=3) .help = Axis of rotation (as Miller indices, i.e. three integers) of crystal during data collection. Only used if remove_cone_around_axis is defined. elliptical_truncation = False .type = bool .help = Truncate data anisotropically, using the overall anisotropic B-factor plus ellipse_scale to set the ellipse dimensions. ellipse_scale = 1.0 .type = float ellipse_target_completeness = None .type = float .help = Target completeness of data (as percent, max=100) after elliptical truncation } generate_noise { add_noise = False .type = bool noise_profile_file = None .type = path profile_model_file = None .type = path profile_data_label = None .type = str scale_noise = 1.0 .type = float n_resolution_bins = 20 .type = int n_intensity_bins = 20 .type = int } fake_data_from_fmodel .help = Options for generating model-based reflections using phenix.fmodel. { include scope mmtbx.command_line.fmodel.fmodel_from_xray_structure_params } } tprocess_includesc C s=|dkrtj}ntdd||dIJt|dksPd|kr|dIJd|krytjddn tjdSd d lm}tj d d }d}d}d}g}x|D]}t j j |r|j |} | jd kr*| j}|j|jdd| jq| jdkre| j}|j|jdd| jq| jdkr|j| jqqy|jd|} Wntk r|d|IJqX|j| qWtjd|} tdd|| jdd| j} | j} td| d|d|d|dS(Nsmmtbx.simulate_low_res_datatoutsO For generation of realistic data (model-based, or using real high-resolution data) for methods development. *********************************** WARNING: *********************************** this is an experimental program - definitely NOT bug-free. Use at your own risk! Usage: mmtbx.simulate_low_res_data model.pdb [options...] (generate data from a PDB file) mmtbx.simulate_low_res_data highres.mtz [model.pdb] [options...] (truncate high-resolution data) mmtbx.simulate_low_res_data --help (print full parameters with additional info) is--helps# full parameters:tattributes_levelii(t file_readert home_scopet simulate_datatpdbtargs pdb_file=%sthkls hkl_file=%stphils&ignoring uninterpretable argument '%s'tsourcessWorking parameterstprefixs tparamsthkl_intpdb_in(tNonetsyststdoutRtlent master_philtshowtiotbxR t!command_line_argument_interpretertostpathtisfiletany_filet file_typet file_objecttappendtprocesst file_namet RuntimeErrortfetchtextractR t prepare_data(targsRR t interpreterRt pdb_hierarchyRt user_philR tftarg_philt working_philtparams_R((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pytrunsX         # #   R)cB sJeZddejdZdZdZdZdZ dZ RS(c C st|t||_||_d|_|jdkrX|jdkrXtdn|j j r|jdkr|j j dkrtdqn|dkr|jdk rt j |jdd}|jd|j|_n|jdkr8|jdk r8t j |jdd}|jd|j|_n|jdk rY|jj|_n|jdk rtddtj|j\}}n7|jdk rtddtj|j\}}n|jjdk r|j|\}}n||_td dtj|d |jIJd\|_|_|jdk r]t |j\|_|_n|j!||_"|j j rtd dtj|j"j#dkr|jdk rt$|j"|j|j\}} nt%d |j"d |j d|j&dt'} nt(| | j)|_"ntddtj|d|j"j*dIJ|d|j"jIJ|jdk rt$|j"|j|j\}} |dIJ|dIJt+|| d||dIJn|j,dS(NsNo PDB file specified.sJnoise_profile_file required when add_noise=True and hkl_file is undefined.t force_typeR RsExtracting experimental dataRs'Generating fake data with phenix.fmodels!Applying low-resolution filterings Target resolution: %.2f As Adding noise using sigma profiletf_obsRtwilson_btreturn_as_amplitudessDone processings' Completeness after processing: %.2f%%gY@s Final resolution: %.2f Ats% Scaling statistics for output data:(NN(-RtlocalsRRRR,tpdb_filethkl_fileRtgenerate_noiset add_noisetnoise_profile_fileR R tassert_file_typeR"RRthkl_Filet hierarchyRRRtfrom_hkltfrom_pdbt r_free_flagsR%timport_r_free_flagstr_freetd_mint n_residuestn_basest get_countst truncate_datatf_outtsigmastwilson_scalingt create_sigmastb_wilsontFalsetapply_sigma_noiset f_sq_as_ft completenesstshow_b_factor_infot write_output( tselfRRRRR.tf_rawREt iso_scalet aniso_scaleti_obs((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyt__init__sv               c C s|j}|j}|j}|jdk rH|jdddd}n|jdddd}|jdk r|jj|}|jd|ddddn|j }|j dkr'|j dk rt j jt j j|j d}n%t j jt j j|jd}|d |_ n|jd |j |d |j IJ|jdk r|jrt j j|j dd }t|d } | jddj|jjj| j|jj| j|d|IJndS(Ntcolumn_root_labeltFt column_typestFQt miller_arrayt FreeR_flagtIis _low_res.mtzR%s Wrote %ss.pdbtws%s s s Wrote modified model to %s(RKRRRLRtas_mtz_datasetREt common_settadd_miller_arrayt mtz_objectt output_fileR:RRtsplitexttbasenameR9twriteR,twrite_modified_pdbtopentjoinRtinputtcrystallographic_sectiont as_pdb_stringtclose( RVRKRRt mtz_datasetRERgt base_nametpdb_outR.((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRUs>          (%& c!C sx|j}|j}|j}d+\}}|jj}|dk r]|j}|j}n|}|}|jjdk r|jj}n ||j_|jjdk r|jj}n ||j_|dks|dkrt ddnddl m } m } ddl m} | jd|d|} |jjr|dIJx|jD]} x| jD]}xx|jD]j}x6|jD](}|jd,kr|jd |qqWt|jd kru|jd |ququWt|jd krb| jd|qbqbWqOWn|jjrR|dIJddlm}|jd|n|jjd| }|j j!j"}|j#}d|kpd|ks|dIJ|dIJ|dIJ|dIJn|jj$r|j$nd}|jj%dk rM|jj& st'|d|jj%IJ|jj%d ks>t'|jj%}n;|jj&r|dIJt(d|j)dd}|dIJn|dk r|j*}|j+|}| j,|}| j-|}||}|j.||jj/d|j0d|jnddl1}ddlm2}|j3j4j5j6}|j)|_7|j8}|j4|_4|j4j9d kr|d IJ|d!IJ|d"IJ|d#IJd$|j4_9|dIJn|j4j:d kr|d%IJ|d!IJ|d"IJ|d&IJd'|j4_:|dIJn|j;|_;|j<|_<|j=j>|_?t@|_Ad(|jB_C|jDd)|d*|}|jE}|j=} || fS(-Ns9Incomplete symmetry information - please specify a space s'group and unit cell for this structure.i(tcrystaltadptbx(tflext unit_celltspace_group_infos Removing solvent atoms...tHOHtWATt atom_groupit residue_grouptchainsC Removing all alternate conformations and resetting occupancies...(tpdbtoolsR@tcrystal_symmetrytHtDs6 WARNING: this model does not contain hydrogen atoms!s9 strongly recommend running phenix.ready_set ors9 equivalent to ensure realistic simulated data.R7s( Scaling B-factors to have mean of %.2fs> Scaling B-factors to match mean Wilson B for this resolutionRFt stat_typeR5t scatterers(tutilss8 b_sol is zero - will use mean value for d_min +/- 0.2As8 (this is not strongly correlated with resolution, buts< it is preferrable to use a real value instead of leavings it set to 0)gG@s8 k_sol is zero - will use mean value for d_min +/- 0.2As it set to 0)gffffff?trealtxray_structureR(NN(R{R|(FRRR,RRRRzRyt space_groupRtcctbxRvRwtscitbx.array_familyRxtsymmetryt modify_pdbt remove_waterstmodelstchainstresidue_groupst atom_groupstresnametremove_atom_groupRtremove_residue_grouptatomst remove_chaintremove_alt_confstmmtbxRtxray_structure_simpletscattering_type_registryttype_count_dicttkeyst hd_selectiontconvert_to_isotropictset_mean_b_isot set_wilson_btAssertionErrort!get_mean_statistic_for_resolutionRFtextract_u_iso_or_u_equivtselecttmeantu_as_bt scale_adpstset_adps_from_scatterersRtmmtbx.command_line.fmodelRt command_linetfmodelt(fmodel_from_xray_structure_master_paramsR(thigh_resolutiontfake_data_from_fmodeltb_soltk_soltstructure_factors_accuracytmaskRCtfractiontr_free_flags_fractionRPt add_sigmastoutputttypetfmodel_from_xray_structuretf_model(!RVRRR,tpdb_sgtpdb_uctpdb_symmtapply_sgtapply_ucRvRwRxt apply_symmtmodelRR~R}RRt sctr_keysRtset_btu_isotu_meantb_meantscaleRRt fmodel_paramst fake_datatfmodel_RRC((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRB7s                                                           cC s|j}|j}|jj}d}d}xo|D]g}|jdk rv|jj|jkr|j}qn|j r|j}|jj|_|d|jIJnG|j r|j j}|jj|_|d|jIJn|j j dk rZ|jj|j j kr|j}|d|jjIJqq4|jjdkr4|j}|d|jjIJq4q4W|jd|j}|dk r|j|}n||fS(Ns F-obs: %ss I-obs: %ss R-free: %sRatFREERF(s FreeR_flagR(RRRtas_miller_arraysRt data_labeltinfot label_stringt map_to_asutis_xray_amplitude_arraytis_xray_intensity_arrayRRRCtlabeltresolution_filterRFRe(RVRRt miller_arraysR4REtarray((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRAs8         ! c C s|jj}|j}ddlm}|j|j}|jd|j}|j ddd|j ddddd t \}}|j d |j|k} | jj|} |d |j|jjfIJ|jj| jjkr|jj| jj} | d ks.t|jd kr]|d| IJ|j| }q|d| IJ|j| } | jd| jjt| jjdddt} | jd| } n|jj| jjkst|| fS(Ni(R RR%Rttest_flag_valuetparameter_scopessimulate_data.r_free_flagstdisable_suitability_testtdatas Using R-free flags from %s:%sitdiscards1 discarding %d amplitudes without R-free flagss6 generating missing R-free flags for %d reflectionsRtmax_freetuse_lattice_symmetrytother(RRCRRR R R%R>t file_servertget_r_free_flagsRRRPtcustomized_copyRRReRRtsizeRt missing_flagstlone_settgenerate_r_free_flagstcounttTruet concatenate( RVR]RRR trfree_int hkl_servert r_free_rawt flag_valueREt n_missingt missing_setR((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRDsB       $" % *c C se|j}|j}ddlm}|jjdk ro|d|jjIJ|jd|jjd|jj}n|jj ddddddgkr|dIJ|jd|jj d|jj}n|jj dk r|d IJt d |d |jj }n|jj dk r|d IJ|d |jj IJ|jj dk sFtt d|d|jj d|jj \}}|d||fIJn|jjra|dIJ|jj}|jd}||kr|dIJ|dIJ|dIJqa|dIJtd|d|jd|j\} } td|d| jd|jjd|\}}|d||fIJn|S(Ni(Rxs) Applying isotropic B-factor of %.2f A^2tb_isotapply_to_sigmasis Adding anisotropy...tb_carts0 Adding random error as percent of amplitude...R4t error_percents2 Removing cone of data around axis of rotation...s radius = %.1f degreesRtaxist cone_radiuss& removed %d reflections (out of %d)s% Truncating the data elliptically...gY@s3 completeness is already less than target value:s %.2f versus %.2fs* elliptical truncation will be skipped.s+ using overall anisotropic B as a guide.R]RGRHt scale_factorttarget_completeness(RRRRxttruncatet add_b_isoRtapply_debye_waller_factorstapply_b_to_sigmast add_b_anisotadd_random_error_percenttadd_random_errortremove_cone_around_axistaxis_of_rotationRtelliptical_truncationtellipse_target_completenessRSRMRGRHRt ellipse_scale( RVR]RRRxtn_hklt delta_n_hklRtcompleteness_startRXRY((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRJsd     $                       N( t__name__t __module__RRRR[RURBRARDRJ(((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyR)s A # m  $cC sd}d}xx|jdjD]`}|jd}|jr^|t|j7}q#|jr#|t|j7}q#q#W||fS(Ni(RRt conformerst is_proteinRRtis_na(R@RGRHRt main_conf((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRIs  cC sVddlm}|jd|d|d|}|jd|d|d|}||fS(Ni(tabsolute_scalingR`RGRH(t mmtbx.scalingR tml_iso_absolute_scalingtml_aniso_absolute_scaling(R]RGRHR RXRY((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRM&s    cC so|j}|d|jIJ|d|d|d|dfIJ|d|d|dfIJ|d |d IJdS( Ns' overall isotropic B-factor: %6.2fs5 overall anisotropic B-factor: %6.2f, %6.2f, %6.2fiiis/ %14.2f, %6.2fiis( %22.2fi(RRO(RXRYRR((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRT2s cC sddlm}d|ko'dkns2t|j}tj|d}|j}xNt|jD]:}tj dd}|dkrd}n|j |qqW|||}|j d|S(Ni(RxidigY@iR( RRxRRR]tdoubletxrangeRtrandomtrandintR#R(R4RRxRtfrtrittrialtr((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyR;s"   g?cC s^ddlm}ddlm}|j}d}tj}x<t|dd!D]'\} } | |krO| }| }qOqOWd|kodknstdddg} x3|D]+} | || |kr| || |          c C sg|jdkstddlm}|j}|j|j|j}td|j d|j d|d|j d|j d|j }|j|jd }|jd |j x|jjD]l} |jj| } |j| } x?| jD]1} | j|| } |j| | || RRRRRRRLRRPRRR"R@RIRMRTRORR@RRRTRURVRWRtshell_intensity_binsR#(RVRLR9R5RRMRNRR RxR.RR4RZRR,RGRHRXRYtdelta_bti_meanR[Rati_bintselti_shelltsn_shellt noise_bins((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyR[sh       $  $      cC s|j|dS(Ni(Re(RVRj((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRXsN(RRRR[RX(((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pyRQs  RRPRNRQRQRgRoRtargv(((sV/public2/ftp/pub/ccp4/6.5/ccp4-6.5/lib/py2/mmtbx/command_line/simulate_low_res_data.pytsF        AJ  *  ! G