import phaser

import urllib
import urllib2
import StringIO
import gzip
import operator
import re
import time
import subprocess
import os
import tempfile
import glob
import itertools
import shutil

# Exceptions
class ProxyException(Exception):
    pass


# Downloader classes
class DownloaderBase(object):

    REQUEST_HEADER = "Accept-encoding"
    STREAM_HEADER = "Content-Encoding"


    def handles(self, data_stream):

        encoding = data_stream.headers.get( self.STREAM_HEADER, "" ).lower()

        return self.interpretable_encoding( encoding )


    def interpretable_encoding(self, encoding):

        return encoding == self.ENCODING


    def enable_encoding(self, request):

        settings = request.get_header( self.REQUEST_HEADER )

        if settings:
            if self.ENCODING not in settings:
                request.add_header(
                    self.REQUEST_HEADER,
                    "%s, %s" % ( settings, self.ENCODING )
                    )

        else:
            request.add_header( self.REQUEST_HEADER, self.ENCODING )


class SimpleDownloader(DownloaderBase):

    ENCODING = "identity"

    def fetch(self, data_stream):

        return data_stream.read()


    def interpretable_encoding(self, encoding):

        return ( not encoding or encoding == self.ENCODING )


class GzipDownloader(DownloaderBase):

    ENCODING = "gzip"

    def fetch(self, data_stream):

        compressed_data = data_stream.read()

        try:
            data = gzip.GzipFile(
                fileobj = StringIO.StringIO( compressed_data )
                ).read()

        except IOError, e:
            raise ProxyException, "Bad encoding: error '%s'" % e

        return data


# HTTP functionality
class HTTPInterface(object):
    """
    Mixin class to provide HTTP functionality
    """

    USER_AGENT = "HTTPInterface/0.0.1 +http://www.phenix-online.org/"

    DOWNLOADERS = [
        SimpleDownloader(),
        GzipDownloader(),
        ]

    OPENER = urllib2.build_opener()

    REQUEST_KEYS = set(
        [ "url", "data", "headers", "origin_req_host", "unverifiable" ]
        )


    def http_call(self, request):

        if not isinstance( request, dict ):
            raise ValueError, "Badly formed query"

        request_dict = request.copy()

        if ( not "url" in request_dict
            or not all( [ key in self.REQUEST_KEYS for key in request_dict ] ) ):
            raise ValueError, "Badly formed query"

        if "data" in request_dict:
            if not isinstance( request_dict[ "data" ], dict ):
                raise ValueError, "Badly formed query"

            request_dict[ "data" ] = urllib.urlencode( request_dict[ "data" ] )


        if "headers" in request_dict:
            if not isinstance( request_dict[ "headers" ], dict ):
                raise ValueError, "Badly formed query"

        else:
            request_dict[ "headers" ] = {}

        request_dict[ "headers" ][ "User-Agent" ] = self.USER_AGENT
        http_request = urllib2.Request( **request_dict )

        try:
            data_stream = self.OPENER.open( http_request )

        except urllib2.URLError, e:
            raise ProxyException, e

        data = None

        for downloader in self.DOWNLOADERS:
            if downloader.handles( data_stream ):
                data = downloader.fetch( data_stream )
                break

        else:
            raise ProxyException, "Cannot interpret encoding"

        return data


    def resource_online(self, url):

        try:
            data_stream = self.OPENER.open( url )

        except urllib2.URLError:
            return False

        return True


class OCAHTTP(HTTPInterface):
    """
    Interface to EBI OCA service using HTTP
    """

    URL = "http://www.ebi.ac.uk/msd-srv/oca/oca-bin/ocamain"
    REGEX = re.compile( r"<pre>(.*)</pre>", re.DOTALL )

    def __init__(self):

        if not self.resource_online( url = self.URL ):
            raise ProxyException, "Cannot contact service"


    def fetch(self, identifier, format = "seq"):

        request = {
            "url": "http://www.ebi.ac.uk/msd-srv/oca/oca-bin/send-%s?%s" % (
                format.lower(),
                identifier
                ),
            }

        raw = self.http_call( request = request )
        match = self.REGEX.search( raw )

        if not match:
            raise ProxyException, "Unexpected response from %s" % str( self )

        return match.group( 1 )


    def __str__(self):

        return "OCA service at EBI"


class DbfetchHTTP(HTTPInterface):
    """
    Interface to EBI Dbfetch service using HTTP
    """

    URL = "http://www.ebi.ac.uk/cgi-bin/dbfetch"

    def __init__(self):

        if not self.resource_online( url = self.URL ):
            raise ProxyException, "Cannot contact service"


    def fetch(self, identifier, database = "pdb", format = "pdb"):

        request = {
            "url": "http://www.ebi.ac.uk/Tools/webservices/rest/dbfetch/%s/%s/%s" % (
                database.lower(),
                identifier,
                format.lower()
                ),
            }

        return self.http_call( request = request )


    def __str__(self):

        return "Dbfetch service at EBI"


class RCSBHTTP(HTTPInterface):
    """
    Interface to RCSB service using HTTP
    """

    URL = "http://www.rcsb.org/pdb/home/home.do"

    def __init__(self):

        if not self.resource_online( url = self.URL ):
            raise ProxyException, "Cannot contact service"


    def fetch(self, identifier, format = "pdb", compression = False):

        if compression:
            request = {
                "url": "http://www.rcsb.org/pdb/files/%s.%s.gz" % (
                        identifier,
                        format.lower()
                        ),
                }

        else:
            request = {
                "url": "http://www.rcsb.org/pdb/download/downloadFile.do",
                "data": {
                    "fileFormat": format.lower(),
                    "compression": "NO",
                    "structureId": identifier,
                    }
                }

        return self.http_call( request = request )


    def __str__(self):

        return "PDB service at RCSB"



class NCBIBlastHTTP(HTTPInterface):
    """
    Interface to NCBI Blast using HTTP
    """

    URL = "http://blast.ncbi.nlm.nih.gov/Blast.cgi"

    RID_EXTRACT = re.compile(
        r"QBlastInfoBegin\s+RID\s*=(.*)\s+RTOE\s*=(.*)\s+QBlastInfoEnd"
        )
    STATUS_EXTRACT = re.compile(
        r"QBlastInfoBegin\s+Status=(.*)\s+QBlastInfoEnd"
        )
    SUBMISSION_PAGE_EXTRACT = re.compile(
        r"<strong>\s*Format\s+Request\s*</strong>"
        )

    def __init__(self):

        if not self.resource_online( url = self.URL ):
            raise ProxyException, "Cannot contact service"


    def get(
        self,
        rid,
        alignments = 500,
        alignment_view = "Pairwise",
        descriptions = 500,
        entrez_links_new_window = None,
        expect_low = 0,
        expect_high = None,
        format_entrez_query = None,
        format_object = "Alignment",
        format_type = "HTML",
        ncbi_gi = "no",
        results_file = "no",
        service = "plain",
        show_overview = "yes"
        ):

        data = [
            ( ( "ALIGNMENTS", alignments ), ) if alignments else (),
            ( ( "ALIGNMENT_VIEW", alignment_view ), ) if alignment_view else (),
            ( ( "DESCRIPTIONS", descriptions ), ) if descriptions else (),
            ( ( "ENTREZ_LINKS_NEW_WINDOW", entrez_links_new_window ), ) if entrez_links_new_window else (),
            ( ( "EXPECT_LOW", expect_low ), ) if expect_low else (),
            ( ( "EXPECT_HIGH", expect_high ), ) if expect_high else (),
            ( ( "FORMAT_ENTREZ_QUERY", format_entrez_query ), ) if format_entrez_query else (),
            ( ( "FORMAT_OBJECT", format_object ), ) if format_object else (),
            ( ( "FORMAT_TYPE", format_type ), ) if format_type else (),
            ( ( "NCBI_GI", ncbi_gi ), ) if ncbi_gi else (),
            ( ( "RESULTS_FILE", results_file ), ) if results_file else (),
            ( ( "RID", rid ), ) if rid else (),
            ( ( "SERVICE", service ), ) if service else (),
            ( ( "SHOW_OVERVIEW", show_overview ), ) if show_overview else (),
            ( ( "CMD", "Get" ), ),
            ]

        request = {
            "url": self.URL,
            "data": dict( reduce( operator.add, data ) ),
            }

        return self.http_call( request = request )


    def put(
        self,
        program,
        database,
        query,
        auto_format = "Off",
        composition_based_statistics = "no",
        db_genetic_code = 1,
        endpoints = "no",
        entrez_query = None,
        expect = 10.0,
        filter = None,
        gapcosts = None,
        genetic_code = 1,
        hitlist_size = 500,
        I_thresh = 0.001,
        layout = "OneWindow",
        lcase_mask = "no",
        matrix_name = "BLOSUM62",
        nucl_penalty = -3,
        nucl_reward = 1,
        other_advanced = None,
        perc_ident = 99,
        phi_pattern = None,
        query_file = None,
        query_believe_defline = "no",
        query_from = 0,
        query_to = 0,
        searchsp_eff = 0,
        service = "plain",
        threshold = None,
        ungapped_alignment = "no",
        word_size = None
        ):

        data = [
            ( ( "AUTO_FORMAT", auto_format ), ) if auto_format else (),
            ( ( "COMPOSITION_BASED_STATISTICS", composition_based_statistics ), ) if composition_based_statistics else (),
            ( ( "DATABASE", database ), ),
            ( ( "DB_GENETIC_CODE", db_genetic_code ), ) if db_genetic_code else (),
            ( ( "ENDPOINTS", endpoints ), ) if endpoints else (),
            ( ( "ENTREZ_QUERY", entrez_query ), ) if entrez_query else (),
            ( ( "EXPECT", expect ), ) if expect else (),
            ( ( "FILTER", filter ), ) if filter else (),
            ( ( "GAPCOSTS", gapcosts ), ) if gapcosts else (),
            ( ( "GENETIC_CODE", genetic_code ), ) if genetic_code else (),
            ( ( "HITLIST_SIZE", hitlist_size ), ) if hitlist_size else (),
            ( ( "I_THRESH", I_thresh ), ) if I_thresh else (),
            ( ( "LAYOUT", layout ), ) if layout else (),
            ( ( "LCASE_MASK", lcase_mask ), ) if lcase_mask else (),
            ( ( "MATRIX_NAME", matrix_name ), ) if matrix_name else (),
            ( ( "NUCL_PENALTY", nucl_penalty ), ) if nucl_penalty else (),
            ( ( "NUCL_REWARD", nucl_reward ), ) if nucl_reward else (),
            ( ( "OTHER_ADVANCED", other_advanced ), ) if other_advanced else (),
            ( ( "PERC_IDENT", perc_ident ), ) if perc_ident else (),
            ( ( "PHI_PATTERN", phi_pattern ), ) if phi_pattern else (),
            ( ( "PROGRAM", program ), ),
            ( ( "QUERY", query ), ),
            ( ( "QUERY_FILE", query_file ), ) if query_file else (),
            ( ( "QUERY_BELIEVE_DEFLINE", query_believe_defline ), ) if query_believe_defline else (),
            ( ( "QUERY_FROM", query_from ), ) if query_from else (),
            ( ( "QUERY_TO", query_to ), ) if query_to else (),
            ( ( "SEARCHSP_EFF", searchsp_eff ), ) if searchsp_eff else (),
            ( ( "SERVICE", service ), ) if service else (),
            ( ( "THRESHOLD", threshold ), ) if threshold else (),
            ( ( "UNGAPPED_ALIGNMENT", ungapped_alignment ), ) if ungapped_alignment else (),
            ( ( "WORD_SIZE", word_size ), ) if word_size else (),
            ( ( "CMD", "Put" ), ),
            ]

        request = {
            "url": self.URL,
            "data": dict( reduce( operator.add, data ) ),
            }

        data = self.http_call( request = request )
        result = self.SUBMISSION_PAGE_EXTRACT.search( data )

        if not result:
            raise ProxyException, "Error in parameters, job not started"

        result = self.RID_EXTRACT.search( data )

        if not result:
            raise ProxyException, "Unexpected response from server"

        return result.group( 1 )


    def status(self, rid):

        data = self.get( rid = rid )

        result = self.STATUS_EXTRACT.search( data )

        if not result:
            raise ProxyError, "Unexpected response from server"

        return result.group( 1 )


    def delete(self, rid):

        request = {
            "url": self.URL,
            "data": dict( [ ( "RID", rid ), ( "CMD", "Delete" ) ] )
            }
        self.http_call( request = request )


    def __str__(self):

        return "BLAST service at NCBI"


class NCBIBlastStandalone(object):
    """
    Interface to NCBI Blast running locally
    """

    def __init__(self, command_line = ( "blastall", ), osdepbool = True):
        self.osdepbool = osdepbool # should be True for win32, otherwise False

        if len( command_line ) == 0:
            raise ProxyException, "Empty command line"

        try:
            process = subprocess.Popen(
                command_line,
                shell = self.osdepbool,
                stdout = subprocess.PIPE,
                stderr = subprocess.PIPE
                )

        except OSError, e:
            raise ProxyException, "Cannot find '%s': %s" % ( command_line[0], e )

        ( out, err ) = process.communicate()

        if err and not out:
            raise ProxyException, (
                "Error from executable: '%s', '%s'" % ( out, err )
                )

        self.command_line = command_line
        self.job_counter = itertools.count()
        self.results_for = {}


    def results(self, jobid):

        return self.results_for.get( jobid, "" )


    def submit(
        self,
        program,
        database,
        query,
        maxresults,
        expectation = 10.0,
        matrix = "BLOSUM62",
        alignment_view = 7,
        pcmdstr = None
        ):

        jobname = self.job_counter.next()
        job_data = (
            "-p", program,
            "-d", database,
            "-e", str( expectation ),
            "-M", matrix,
            "-m", str( alignment_view ),
            "-v", str(maxresults),
            "-b", str(maxresults),
            )

        cmdline = self.command_line + job_data
        if pcmdstr != None: # pass on explicit cmdline to calling fucntion
            pcmdstr[0] = "Executing: " + subprocess.list2cmdline(cmdline) \
             + " " + query

        process = subprocess.Popen(
            cmdline,
            shell = self.osdepbool,
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT
            )

        ( out, err ) = process.communicate( input = query )

        self.results_for[ jobname ] = out

        return jobname


    def status(self, jobid):

        if jobid not in self.results_for:
            return "UNKNOWN"

        return "FINISHED"


    def cleanup(self, jobid):

        if self.status( jobid  = jobid ) == "UNKNOWN":
            return

        del self.results_for[ jobid ]


    def existing_jobs(self):

        return self.results_for.keys()


    def __str__(self):

        return "BLAST standalone executable"


class ClustalWLocal(object):
    """
    Interface to clustalw running locally
    """

    INP = "clustal.inp"
    OUT = "clustal.out"
    LOG = "clustal.log"
    ERR = "clustal.err"

    def __init__(self, job_data_dir, executable = "clustalw2"):

        if os.name in [ "nt" ]:
            command_line_arg_lead = "/"

        else:
            command_line_arg_lead = "-"

        try:
            process = subprocess.Popen(
                [ executable, command_line_arg_lead + "help" ],
                stdout = subprocess.PIPE,
                stderr = subprocess.PIPE
                )

        except OSError, e:
            raise ProxyException, "Cannot find '%s': %s" % ( executable, e )

        ( out, error ) = process.communicate()

        if not os.path.exists( job_data_dir ):
            raise ProxyException, "Job data folder unavailable"

        self.job_data_dir = job_data_dir
        self.executable = executable
        self.command_line_arg_lead = command_line_arg_lead
        self.process_data_for = {}


    def run(
        self,
        sequences,
        type = None,
        output = None,
        iteration = None,
        numiter= None,
        outorder = None,
        quicktree = None,
        ktuple = None,
        window = None,
        pairgap = None,
        topdiags = None,
        score = None,
        gapopen = None,
        gapext = None,
        endgaps = None,
        gapdist = None,
        matrix = None
        ):

        if not os.path.exists( self.job_data_dir ):
            raise ProxyException, "Job data folder unavailable"

        tmp_dir = tempfile.mkdtemp( dir = self.job_data_dir )

        infile_name = os.path.join( tmp_dir, self.INP )
        infile = file( infile_name, "w" )
        infile.write( sequences )
        infile.close()

        outfile_name = os.path.join( tmp_dir, self.OUT )

        logfile = file( os.path.join( tmp_dir, self.LOG ), "w" )
        errfile = file( os.path.join( tmp_dir, self.ERR ), "w" )

        data = [
            ( "infile=%s" % infile_name, ),
            ( "outfile=%s" % outfile_name, ),
            ( "type=%s" % type, ) if type else (),
            ( "output=%s" % output, ) if output else (),
            ( "iteration=%s" % iteration, ) if iteration else (),
            ( "numiter=%s" % numiter, ) if numiter else (),
            ( "outorder=%s" % outorder, ) if outorder else (),
            ( "quicktree", ) if quicktree is not None else (),
            ( "ktuple=%s" % ktuple, ) if ktuple else (),
            ( "window=%s" % window, ) if window else (),
            ( "pairgap=%s" % pairgap, ) if pairgap else (),
            ( "topdiags=%s" % topdiags, ) if topdiags else (),
            ( "score=%s" % score, ) if score else (),
            ( "gapopen=%s" % gapopen, ) if gapopen else (),
            ( "gapext=%s" % gapext, ) if gapext else (),
            ( "endgaps", ) if endgaps is not None else (),
            ( "gapdist=%s" % gapdist, ) if gapdist else (),
            ( "matrix=%s" % matrix, ) if matrix else (),
            ]

        process = subprocess.Popen(
            [ self.executable ] + [ "%s%s" % ( self.command_line_arg_lead, d )
                for d in reduce( operator.add, data ) ],
            stdout = logfile,
            stderr = errfile
            )

        self.process_data_for[ tmp_dir ] = process

        return tmp_dir


    def fetch_aln(self, jobid):

        return self.fetch_file( jobid = jobid, key = self.OUT )


    def fetch_log(self, jobid):

        return self.fetch_file( jobid = jobid, key = self.LOG )


    def fetch_err(self, jobid):

        return self.fetch_file( jobid = jobid, key = self.ERR )


    def fetch_file(self, jobid, key):

        if self.status( jobid = jobid ) != "FINISHED":
            return ""

        file_name = os.path.join( jobid, key )

        if os.path.exists( file_name ):
            return file( file_name ).read()

        return ""


    def status(self, jobid):

        if jobid not in self.process_data_for:
            return "UNKNOWN"

        process = self.process_data_for[ jobid ]

        if process:
            if process.poll() is None:
                return "RUNNING"

            else:
                self.process_data_for[ jobid ] = None

        return "FINISHED"


    def cleanup(self, jobid):

        if self.status( jobid  = jobid ) == "UNKNOWN":
            return

        process = self.process_data_for[ jobid ]

        if process:
            process.kill()

        if os.path.exists( jobid ):
            for file_name in os.listdir( jobid ):
                os.remove( os.path.join( jobid, file_name ) )

            os.rmdir( jobid )

        del self.process_data_for[ jobid ]


    def existing_jobs(self):

        return self.process_data_for.keys()


class ClustalWSynchronous(object):
    """
    Interface to clustalw running locally
    """

    INP = "clustal.inp"
    OUT = "clustal.out"

    def __init__(
        self,
        executable = "clustalw2",
        type = None,
        output = None,
        iteration = None,
        numiter= None,
        outorder = None,
        quicktree = None,
        ktuple = None,
        window = None,
        pairgap = None,
        topdiags = None,
        score = None,
        gapopen = None,
        gapext = None,
        endgaps = None,
        gapdist = None,
        matrix = None
        ):

        if os.name in [ "nt" ]:
            command_line_arg_lead = "/"

        else:
            command_line_arg_lead = "-"

        try:
            process = subprocess.Popen(
                [ executable, command_line_arg_lead + "help" ],
                stdout = subprocess.PIPE,
                stderr = subprocess.PIPE
                )

        except OSError, e:
            raise ProxyException, "Cannot find '%s': %s" % ( executable, e )

        ( out, error ) = process.communicate()

        self.executable = executable
        self.command_line_arg_lead = command_line_arg_lead
        self.options = [
            ( "type=%s" % type, ) if type else (),
            ( "output=%s" % output, ) if output else (),
            ( "iteration=%s" % iteration, ) if iteration else (),
            ( "numiter=%s" % numiter, ) if numiter else (),
            ( "outorder=%s" % outorder, ) if outorder else (),
            ( "quicktree", ) if quicktree is not None else (),
            ( "ktuple=%s" % ktuple, ) if ktuple else (),
            ( "window=%s" % window, ) if window else (),
            ( "pairgap=%s" % pairgap, ) if pairgap else (),
            ( "topdiags=%s" % topdiags, ) if topdiags else (),
            ( "score=%s" % score, ) if score else (),
            ( "gapopen=%s" % gapopen, ) if gapopen else (),
            ( "gapext=%s" % gapext, ) if gapext else (),
            ( "endgaps", ) if endgaps is not None else (),
            ( "gapdist=%s" % gapdist, ) if gapdist else (),
            ( "matrix=%s" % matrix, ) if matrix else (),
            ]


    def run(self, sequences):

        tmp_dir = tempfile.mkdtemp()

        infile_name = os.path.join( tmp_dir, self.INP )
        infile = file( infile_name, "w" )
        infile.write( sequences )
        infile.close()

        outfile_name = os.path.join( tmp_dir, self.OUT )

        files = [
            ( "infile=%s" % infile_name, ),
            ( "outfile=%s" % outfile_name, ),
            ]

        process = subprocess.Popen(
            [ self.executable ] + [ "%s%s" % ( self.command_line_arg_lead, d )
                for d in reduce( operator.add, files + self.options ) ],
            stdout = logfile,
            stderr = errfile
            )
        ( log, err ) = process.communicate()

        aln_file_name = os.path.join( tmp_dir, self.OUT )

        if os.file.exists( aln_file_name ):
            aln = file( aln_file_name ).read()

        else:
            aln = ""

        shutils.rmtree( tmp_dir )

        return ( aln, log, err )


class PhaserOpenMP(object):
    """
    Runs phaser
    """

    STATUS_FINISHED = "FINISHED"
    STATUS_UNKNOWN = "UNKNOWN"
    STATUS_RUNNING = "RUNNING"

    def __init__(self):

        self.job_counter = itertools.count()
        self.tempdir_for = {}
        self.results_for = {}
        self.debug = False


    def results(self, job_name):

        if self.status( job_name = job_name ) != self.STATUS_FINISHED:
            return None

        else:
            return self.results_for[ job_name ]


    def submit_mr_auto_job(
        self,
        hkl,
        fp,
        sigfp,
        space_group,
        unit_cell,
        asu_weight,
        models_for,
        search_ensembles,
        search_number = 1,
        partial_structure = None,
        high_resolution = None,
        low_resolution = None,
        skip_anisotropy_correction = False,
        jobs = 1,
        ):

        # Temporary folder
        job_name = self.get_next_job_name()
        self.tempdir_for[ job_name ] = tempfile.mkdtemp()

        # Run object
        input = phaser.InputMR_AUTO()
        input.setROOT( os.path.join( self.tempdir_for[ job_name ], "PHASER" ) )
        input.setJOBS( jobs )
        input.setMUTE( not self.debug )

        # Crystal data
        input.setREFL( hkl, fp, sigfp )
        input.setSPAC_NAME( space_group )
        input.setCELL6( unit_cell )

        # Composition
        input.addCOMP_PROT_MW_NUM( asu_weight, 1 )

        # Model data
        for id in models_for:
            for ( index, ( file_name, identity ) ) in enumerate( models_for[ id ] ):
                input.addENSE_PDB_ID( id, file_name, identity )

        # Search ensemble
        for ( ensemble_id, number ) in search_ensembles:
            input.addSEAR_ENSE_NUM( ensemble_id, number )

        # Known molecules
        if partial_structure:
            input.setSOLU( partial_structure )

        # Resolution limits
        if high_resolution:
            if low_resolution:
                input.setRESO( high_resolution, low_resolution )

            else:
                input.setHIRES( high_resolution )

        # Anisotropy correction
        if skip_anisotropy_correction:
            input.setMACA_OFF()

        self.start_mr_auto_job( job_name = job_name, input = input )

        return job_name


    def submit_mr_dat_job(self, hkl_file, fp_label, sigfp_label):

        job_name = self.get_next_job_name()

        input = phaser.InputMR_DAT()
        input.setHKLI( hkl_file )
        input.setLABI( fp_label, sigfp_label )
        input.setMUTE( not self.debug )

        self.start_mr_dat_job( job_name = job_name, input = input )

        return job_name


    def cleanup(self, job_name):

        # Temporary files
        if job_name in self.tempdir_for:
            for file_name in os.listdir( self.tempdir_for[ job_name ] ):
                os.remove(
                    os.path.join( self.tempdir_for[ job_name ], file_name )
                    )

            os.rmdir( self.tempdir_for[ job_name ] )
            del self.tempdir_for[ job_name ]

        # Result object
        if job_name in self.results_for:
            del self.results_for[ job_name ]


    def status(self, job_name):

        if job_name in self.results_for:
            return self.STATUS_FINISHED

        else:
            return self.STATUS_UNKNOWN


    def finished(self, job_name):

        return self.status( job_name ) == self.STATUS_FINISHED


    def existing_jobs(self):

        return self.tempdir_for.keys()


    # Internal methods
    def start_mr_auto_job(self, job_name, input):

        result = phaser.runMR_AUTO( input )
        self.results_for[ job_name ] = ( result.Success(), result.getDotSol() )


    def start_mr_dat_job(self, job_name, input):

        result = phaser.runMR_DAT( input )
        self.results_for[ job_name ] = (
            result.Success(),
            result.getMiller(),
            result.getF(),
            result.getSIGF(),
            result.getSpaceGroupName(),
            result.getUnitCell(),
            )


    def get_next_job_name(self):

        return str( self.job_counter.next() )


class PhaserMultiprocessing(PhaserOpenMP):
    """
    Runs phaser
    """

    def __init__(self):

        import phaser.pickle_support
        phaser.pickle_support.enable()

        import multiprocessing
        self.Process = multiprocessing.Process
        self.manager = multiprocessing.Manager()

        import omptbx
        omptbx.omp_set_num_threads( 1 )

        super( PhaserMultiprocessing, self ).__init__()

        # Need to overwrite previous definition
        self.results_for = self.manager.dict()
        self.process_for = {}


    def status(self, job_name):

        self.poll_jobs()

        if job_name in self.process_for:
            return self.STATUS_RUNNING

        else:
            return super( PhaserMultiprocessing, self ).status( job_name )


    def cleanup(self, job_name):

        # Process object
        if job_name in self.process_for:
            self.process_for[ job_name ].terminate()
            self.process_for[ job_name ].join()
            del self.process_for[ job_name ]

        super( PhaserMultiprocessing, self ).cleanup( job_name )


    # Internal method
    def start_mr_auto_job(self, job_name, input):

        # Set JOBS to 1
        input.setJOBS( 1 )

        # Run job
        pr = self.Process(
            target = super( PhaserMultiprocessing, self ).start_mr_auto_job,
            args = ( job_name, input )
            )

        pr.start()
        self.process_for[ job_name ] = pr


    def start_mr_dat_job(self, job_name, input):

        input.setJOBS( 1 )
        super( PhaserMultiprocessing, self ).start_mr_dat_job(
            job_name = job_name,
            input = input
            )


    def poll_jobs(self):

        for job_name in self.process_for.keys():
            if not self.process_for[ job_name ].is_alive():
                self.process_for[ job_name ].join()
                del self.process_for[ job_name ]


class SuperposePdbs(object):
    """
    Runs phenix.superpose_pdbs
    """

    def __init__(self):

        import phenix.command_line.superpose_pdbs
        self._module = phenix.command_line.superpose_pdbs


    def run(self, fixed, moving):

        import libtbx.phil

        working_phil =[
            libtbx.phil.parse( "input.pdb_file_name_fixed=%s" % fixed ),
            libtbx.phil.parse( "input.pdb_file_name_moving=%s" % moving ),
            ]
        merged_phil = self._module.master_params.fetch( sources = working_phil )
        params = merged_phil.extract()
        log = StringIO.StringIO()
        manager = self._module.manager(
            params = params,
            log = log,
            write_output = False,
            save_lsq_fit_obj = True,
            )
        assert manager.lsq_fit_obj
        return ( manager.lsq_fit_obj, manager.rmsd )


class LsqmanAlign(object):
    """
    Runs Lsqman to create a structural alignment
    """

    LSQMAN_MACRO = """
!
! align.lsqmac - Gerard Kleywegt @ 2001-11-22/2002-03-06,09-24,25/
!                                  2006-08-01/2007-07-06
!
! LSQMAN macro with Gerard's favourite recipe for superimposing
! just about any two protein structures from scratch
!
! note: only works with LSQMAN version 9.3 or newer !
!
! ==> PDB file of first structure ?
& pdb1 %s
! ==> chain to use in first structure (use _underscore_ for blank) ?
& chn1 %s
!
! ==> PDB file of second structure ?
& pdb2 %s
! ==> chain to use in second structure (use _underscore_ for blank) ?
& chn2 %s
!
! => No modification needed from here...
! initial cut-off (A)
& cutforce 5.0
!
! final cut-off (A)
& cutdist 3.5
!
! nr of DP cycles
& numdp 10
!
! fast_force frag_length
& fffl 30
!
! fast_force frag_step
& fffs 15
!
! fast_force min_match
& ffmm 0.9
!
! fast_force slide_step
& fsli 2
!
echo on
hetatm strip
hydrogen strip
set reset
nmr_mode first
chain_mode non-blank
!
! change the following line for nucleic acids !
atom_type ca
!
delete gmod1
read gmod1 $pdb1
!
delete gmod2
read gmod2 $pdb2
!
set coarse
set optim nmatch
set dist $cutforce
fast_force gmod1 $chn1 gmod2 $chn2 $fffl $fffs $ffmm $fsli
!
!set intermediate
!improve gmod1 $chn1 gmod2 $chn2
!
!set reset
!improve gmod1 $chn1 gmod2 $chn2
!
! optimise operator by dynamic programming; optimise SAS(1) score
set optim s1
dp gmod1 $chn1 gmod2 $chn2 sq $cutdist $numdp
!
!
global gmod1 $chn1 gmod2 $chn2 $cutdist
!
! ALL DONE !!!!
!
quit
"""

    BIOLSQMAN_MACRO = """
!
! align_long.lsqmac - Gerard Kleywegt @ 2005-04-27/2006-08-01
!
! LSQMAN macro for aligning larger numbers of residues than with
!        the align.lsqmac macro (the latter would be favoured by
!        most crystallographers; the present by bioinformaticians)
!
! note: only works with LSQMAN version 9.3 or newer !
!
! see: http://xray.bmc.uu.se/usf/lsqman_man.html
!
! this macro derived from Gerard's "crystallographer's" macro
! at: ftp://xray.bmc.uu.se/pub/gerard/omac/align.lsqmac
!
! --------------------------------------------------------------
!
! ------------- define which structures and chains to use ------
!
& pdb1 %s
! ==> PDB file of first structure ?
& chn1 %s
! ==> chain to use in first structure (use _underscore_ for blank) ?
& pdb2 %s
! ==> PDB file of second structure ?
& chn2 %s
! ==> chain to use in second structure (use _underscore_ for blank) ?
!
! ------------------------ end of "user input" -----------------
!
! --------------------------------------------------------------
!
! atom type CA for proteins (change for RNA, DNA, sugars, etc.)
& mytype CA
!
! distance cut-off (A) (very critical)
& cutdist 8.0
!
! max nr of DP cycles (rarely critical)
& numdp 10
!
! fast_force frag_length (influences brute-force step)
& fffl 30
!
! fast_force frag_step (influences brute-force step)
& fffs 10
!
! fast_force min_match ("0.8" means: stop when you have found
! an alignment involving at least 80%% of the residues in the
! shortest protein - obviously the higher this value, the more
! likely that the longest possible alignment will be found,
! but also the more CPU time will be required)
& ffmm 0.8
!
! fast_force slide_step (set to 1 or 2 for "normal" structures, or to
! 5 or 10 when comparing biiiig molecules)
& fsli 2
!
echo off
hetatm strip
hydrogen strip
set reset
nmr_mode first
chain_mode non-blank
!
atom_type $mytype
!
delete gmod1
read gmod1 $pdb1
!
delete gmod2
read gmod2 $pdb2
!
! get initial operator by brute force; optimise nr of aligned residues
set coarse
set optim nmatch
set dist $cutdist
fast_force gmod1 $chn1 gmod2 $chn2 $fffl $fffs $ffmm
!
! optimise operator by dynamic programming; optimise SAS(1) score
set optim s1
dp gmod1 $chn1 gmod2 $chn2 sq $cutdist $numdp
!
! derive structure-based sequence alignment
global gmod1 $chn1 gmod2 $chn2 $cutdist
!
! ALL DONE !!!!
!
quit
"""

    EXTRACT = re.compile(
        r"^ \s* Sequence \s+ (\d+) \s+ ([\w\-\?]+) \s*$",
        re.VERBOSE | re.MULTILINE
        )
    UNKOWN = re.compile( r"\?" )

    def __init__(self, executable = "lsqman", macro = LSQMAN_MACRO):

        self.executable = executable

        try:
            self.execute( input = "quit" )

        except OSError, e:
            raise ProxyException, "Cannot find '%s': %s" % ( self.executable, e )

        self.macro = macro


    def run(self, pdb1, chain1, pdb2, chain2):

        ( out, err ) = self.execute(
            input = self.macro % ( pdb1, chain1, pdb2, chain2 )
            )

        matches = self.EXTRACT.findall( out )
        ali1 = self.reconstruct( matches = matches, identifier = "1" )
        ali2 = self.reconstruct( matches = matches, identifier = "2" )

        if not matches or len( ali1 ) != len( ali2 ):
            raise ProxyException, "Error in LSQMAN:\n%s\n%s" % ( out, err )

        return ( ali1, ali2 )


    def reconstruct(self, matches, identifier):

        lines = [ m for m in matches if m[0] == identifier ]
        return self.UNKOWN.sub( "X", reduce( operator.add, [ m[1] for m in lines ] ) )


    def execute(self, input):

        process = subprocess.Popen(
            [ self.executable ],
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE
            )
        ( out, error ) = process.communicate( input = input )

        return ( out, error )


class FFASAlign(object):
    """
    Alignment from FFAS
    """


class DSSPInterface(object):
    """
    Runs DSSP
    """

    HELIX = "helix"
    SHEET = "sheet"

    def __init__(self):

        import iotbx.pdb.hierarchy


        self.root = iotbx.pdb.hierarchy.root()
        self.model = iotbx.pdb.hierarchy.model()
        self.root.append_model( self.model )
        self.chain = iotbx.pdb.hierarchy.chain()
        self.model.append_chain( self.chain )

        import iotbx.pdb.secondary_structure
        import mmtbx.secondary_structure
        self._run = mmtbx.secondary_structure.run_ksdssp_direct
        self._process = iotbx.pdb.secondary_structure.process_records


    def run(self, residue_groups):

        try:
            for ( index, rg ) in enumerate( residue_groups ):
                copy = rg.detached_copy()
                copy.resseq = index
                copy.icode = ""
                self.chain.append_residue_group( copy )

            data = self.root.as_pdb_string()

        finally:
            for rg in self.chain.residue_groups():
                self.chain.remove_residue_group( rg )

        ( records, errors ) = self._run( data )

        if errors and not records:
            raise ValueError, errors

        ann = self._process( records = records )

        assignment = [ None ] * len( residue_groups )

        for helix in ann.helices:
            ( start, end ) = self.limits( sec_str = helix )
            assignment[ start : end ] = [ self.HELIX ] * ( end - start )

        for sheet in ann.sheets:
            for strand in sheet.strands:
                ( start, end ) = self.limits( sec_str = strand )
                assignment[ start : end ] = [ self.SHEET ] * ( end - start )

        return assignment

    @staticmethod
    def limits(sec_str):

        assert sec_str.start_icode == "" and sec_str.end_icode == ""
        start = sec_str.start_resseq
        end = sec_str.end_resseq
        assert start <= end
        return ( start, end )


class CATH(HTTPInterface):
    """
    Interface to CATH using HTTP
    """

    URL = "http://http://www.cathdb.info"

    def __init__(self):

        if not self.resource_online( url = self.URL ):
            raise ProxyException, "Cannot contact service"


    def fetch(self, identifier):

        request = {
            "url": "http://www.cathdb.info/pdb/%s?view=xml" % identifier,
            }

        raw = self.http_call( request = request )

        return raw


    def __str__(self):

        return "CATH database"