## plugintools.py
## This exports methods available for use by plugins for sos

## Copyright (C) 2006 Steve Conklin <sconklin@redhat.com>

### This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.

## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# pylint: disable-msg = R0902
# pylint: disable-msg = R0904
# pylint: disable-msg = W0702
# pylint: disable-msg = W0703
# pylint: disable-msg = R0201
# pylint: disable-msg = W0611
# pylint: disable-msg = W0613

"""
This is the base class for sosreport plugins
"""
from sos.helpers import *
from threading import Thread, activeCount
import os, os.path, sys, string, itertools, glob, re, traceback
import logging
from stat import *
from time import time
import fnmatch

# python < 2.4 (RHEL3 and RHEL4) doesn't have format_exc, activate work-around
if sys.version_info[0] <= 2 and sys.version_info[1] < 4:
   def format_exc():
      import StringIO

      output = StringIO.StringIO()
      traceback.print_exc(file = output)
      toret = output.getvalue()
      output.close()
      return toret

   traceback.format_exc = format_exc
   
class PluginBase:
    """
    Base class for plugins
    """
    def __init__(self, pluginname, commons):
        # pylint: disable-msg = E0203
        try:
            len(self.optionList)
        except:
            self.optionList = []
        # pylint: enable-msg = E0203
        self.copiedFiles = []
        self.copiedDirs = []
        self.executedCommands = []
        self.diagnose_msgs = []
        self.alerts = []
        self.customText = ""
        self.optNames = []
        self.optParms = [] 
        self.piName = pluginname
        self.cInfo = commons
        self.forbiddenPaths = []
        self.copyPaths = []
        self.collectProgs = []
        self.thread = None
        self.pid = None
        self.eta_weight = 1
        self.time_start = None
        self.time_stop  = None

        self.soslog = logging.getLogger('sos')

        # get the option list into a dictionary
        for opt in self.optionList:
            self.optNames.append(opt[0])
            self.optParms.append({'desc':opt[1], 'speed':opt[2], 'enabled':opt[3]})

    # Method for applying regexp substitutions
    def doRegexSub(self, srcpath, regexp, subst):
        '''Apply a regexp substitution to a file archived by sosreport.
        '''
        if len(self.copiedFiles):
            for afile in self.copiedFiles:
                if fnmatch.fnmatch(afile['srcpath'],srcpath):
                    abspath = os.path.join(self.cInfo['dstroot'],
                                            afile['srcpath'].lstrip(os.path.sep))
                    try:
                        fp = open(abspath, 'r')
                        tmpout, occurs = re.subn( regexp, subst, fp.read() )
                        fp.close()
                        if occurs > 0:
                           fp = open(abspath,'w')
                           fp.write(tmpout)
                           fp.close()
                           return occurs
                    except SystemExit:
                      raise SystemExit
                    except KeyboardInterrupt:
                      raise KeyboardInterrupt
                    except Exception, e:
                        self.soslog.log(logging.VERBOSE, "Problem at path %s (%s)" % (abspath,e))
                        break
        return False

    def doRegexFindAll(self,regex,fname):
        ''' Return a list of all non overlapping matches in the string(s)
        '''
        out=[]
        f=open(fname,'r')
        content=f.read()
        f.close()
        reg=re.compile(regex,re.MULTILINE)
        for i in reg.findall(content):
            out.append(i)
        return out
        
    # Methods for copying files and shelling out
    def doCopyFileOrDir(self, srcpath):
        # pylint: disable-msg = R0912
        # pylint: disable-msg = R0915
        ''' Copy file or directory to the destination tree. If a directory, then everything
        below it is recursively copied. A list of copied files are saved for use later
        in preparing a report
        '''
        copyProhibited = 0
        for path in self.forbiddenPaths:
            if ( srcpath.count(path) > 0 ):
                copyProhibited = 1

        if copyProhibited:
            return ''

        if not os.path.exists(srcpath):
            self.soslog.debug("file or directory %s does not exist" % srcpath)
            return

        if os.path.islink(srcpath):
            # This is a symlink - We need to also copy the file that it points to

            # FIXME: ignore directories for now
            if os.path.isdir(srcpath):
                return

            link = os.readlink(srcpath)

            # What's the name of the symlink on the dest tree?
            dstslname = os.path.join(self.cInfo['dstroot'], srcpath.lstrip(os.path.sep))

            if os.path.isabs(link):
                # the link was an absolute path, and will not point to the new
                # tree. We must adjust it.
                rpth = sosRelPath(os.path.dirname(dstslname), os.path.join(self.cInfo['dstroot'], link.lstrip(os.path.sep)))
            else:
                # no adjustment, symlink is the relative path
                rpth = link

            # make sure the link doesn't already exists
            if os.path.exists(dstslname):
                self.soslog.log(logging.DEBUG, "skipping symlink creation: already exists (%s)" % dstslname)
                return

            # make sure the dst dir exists
            if not (os.path.exists(os.path.dirname(dstslname)) and os.path.isdir(os.path.dirname(dstslname))):
                os.makedirs(os.path.dirname(dstslname))

            self.soslog.log(logging.VERBOSE3, "creating symlink %s -> %s" % (dstslname, rpth))
            os.symlink(rpth, dstslname)
            if os.path.isabs(link):
                self.doCopyFileOrDir(link)
            else:
                self.doCopyFileOrDir(os.path.join(os.path.dirname(srcpath), link))
            self.copiedFiles.append({'srcpath':srcpath, 'dstpath':rpth, 'symlink':"yes", 'pointsto':link})
            return

        else: # not a symlink
            if os.path.isdir(srcpath):
                for afile in os.listdir(srcpath):
                    if afile == '.' or afile == '..':
                        pass
                    else:
                        self.doCopyFileOrDir(srcpath+'/'+afile)
                return

        # if we get here, it's definitely a regular file (not a symlink or dir)

        self.soslog.log(logging.VERBOSE3, "copying file %s" % srcpath)
        try:
            tdstpath, abspath = self.__copyFile(srcpath)
        except "AlreadyExists":
            self.soslog.log(logging.DEBUG, "error copying file %s (already exists)" % (srcpath))
            return
        except IOError, e:
            self.soslog.warning("error copying file %s (IOError)" % (srcpath))
            raise e
        except Exception, e:
            self.soslog.log(logging.VERBOSE2, "error copying file %s (%s)" % (srcpath, e))
            return 

        self.copiedFiles.append({'srcpath':srcpath, 'dstpath':tdstpath, 'symlink':"no"}) # save in our list

        return abspath

    def __copyFile(self, src):
        """ call cp to copy a file, collect return status and output. Returns the
        destination file name.
        """
        try:
            # pylint: disable-msg = W0612
            status, shout, runtime = sosGetCommandOutput("/bin/cp --parents -P --preserve=mode,ownership,timestamps,links " + src +" " + self.cInfo['dstroot'])
            if status:
                self.soslog.debug(shout)
            abspath = os.path.join(self.cInfo['dstroot'], src.lstrip(os.path.sep))
            relpath = sosRelPath(self.cInfo['rptdir'], abspath)
            return relpath, abspath
        except SystemExit:
          raise SystemExit
        except KeyboardInterrupt:
          raise KeyboardInterrupt
        except Exception,e:
            self.soslog.warning("Problem copying file %s (%s)" % (src, e))

    def addForbiddenPath(self, forbiddenPath):
        """Specify a path to not copy, even if it's part of a copyPaths[] entry.
        """
        # Glob case handling is such that a valid non-glob is a reduced glob
        for filespec in glob.glob(forbiddenPath):
            self.forbiddenPaths.append(filespec)
    
    def getAllOptions(self):
        """
        return a list of all options selected
        """
        return (self.optNames, self.optParms)
  
    def setOption(self, optionname, enable):
        ''' enable or disable the named option.
        '''
        for name, parms in zip(self.optNames, self.optParms):
            if name == optionname:
                parms['enabled'] = enable

    def isOptionEnabled(self, optionname):
        ''' see whether the named option is enabled.
        '''
        for name, parms in zip(self.optNames, self.optParms):
            if name == optionname:
                return parms['enabled']
        # nonexistent options aren't enabled.
        return 0

    def addCopySpecLimit(self,copyspec,sizelimit = 0):
        """Add a file specification (with limits)
        """
        if (sizelimit == 0):
            return self.addCopySpec(copyspec)
        files = glob.glob(copyspec)
        if (not len(files)):
            self.soslog.debug("glob %s is empty" % copyspec)
            return
        files.sort()
        cursize = 0
        limit_reached = False
        sizelimit *= 1024 * 1024 # in MB
        for flog in files:
            cursize += os.stat(flog)[ST_SIZE]
            if sizelimit and cursize > sizelimit:
                limit_reached = True
                break
            self.addCopySpec(flog)
        # Truncate the first file (others would likely be compressed),
        # ensuring we get at least some logs
        if flog == files[0] and limit_reached:
            self.collectExtOutput("tail -c%d %s" % (sizelimit, flog),
                "tail_" + os.path.basename(flog), flog[1:])

    def addCopySpec(self, copyspec):
        """ Add a file specification (can be file, dir,or shell glob) to be
        copied into the sosreport by this module
        """
        # Glob case handling is such that a valid non-glob is a reduced glob
        for filespec in glob.glob(copyspec):
            self.copyPaths.append(filespec)

    def copyFileGlob(self, srcglob):
        """ Deprecated - please modify modules to use addCopySpec()
        """
        sys.stderr.write("Warning: thecopyFileGlob() function has been deprecated.  Please")
        sys.stderr.write("use addCopySpec() instead.  Calling addCopySpec() now.")
        self.addCopySpec(srcglob)
        
    def copyFileOrDir(self, srcpath):
        """ Deprecated - please modify modules to use addCopySpec()
        """
        sys.stderr.write("Warning: the copyFileOrDir() function has been deprecated.  Please\n")
        sys.stderr.write("use addCopySpec() instead.  Calling addCopySpec() now.\n")
        raise ValueError
        #self.addCopySpec(srcpath)
        
    def runExeInd(self, exe):
        """ Deprecated - use callExtProg()
        """
        sys.stderr.write("Warning: the runExeInd() function has been deprecated.  Please use\n")
        sys.stderr.write("the callExtProg() function.  This should only be called\n")
        sys.stderr.write("if collect() is overridden.")
        pass
        
    def callExtProg(self, prog):
        """ Execute a command independantly of the output gathering part of
        sosreport
        """                        
        # Log if binary is not runnable or does not exist
        if not os.access(prog.split()[0], os.X_OK):
            self.soslog.log(logging.VERBOSE, "binary '%s' does not exist or is not runnable" % prog.split()[0])

        # pylint: disable-msg = W0612
        status, shout, runtime = sosGetCommandOutput(prog)                                                            
        return status
                                                                        
    def callExtProgWithOutput(self, prog):
        """ Execute a command independantly of the output gathering part of
        sosreport
        """                        
        # Log if binary is not runnable or does not exist
        if not os.access(prog.split()[0], os.X_OK):
            self.soslog.log(logging.VERBOSE, "binary '%s' does not exist or is not runnable" % prog.split()[0])

        # pylint: disable-msg = W0612
        return sosGetCommandOutput(prog)                                                            

    def runExe(self, exe):
        """ Deprecated - use collectExtOutput()
        """
        sys.stderr.write("Warning: the runExe() function has been deprecated.  Please use\n")
        sys.stderr.write("the collectExtOutput() function.\n")
        pass

    def collectExtOutput(self, exe, suggest_filename = None, symlink = None):
        """
        Run a program and collect the output
        """
        self.collectProgs.append( (exe,suggest_filename,symlink) )

    def fileGrep(self, regexp, fname):
        results = []

        fp = open(fname, "r")
        for line in fp.readlines():
            if re.match(regexp, line):
                results.append(line)
        fp.close()
        return results

    def mangleCommand(self, exe):
        # FIXME: this can be improved
        mangledname = re.sub(r"^/(usr/|)(bin|sbin)/", "", exe)
        mangledname = re.sub(r"[^\w\-\.\/]+", "_", mangledname)
        mangledname = re.sub(r"/", ".", mangledname).strip(" ._-")[0:64]
        return mangledname

    def makeCommandFilename(self, exe):
        """ The internal function to build up a filename based on a command """

        outfn = self.cInfo['cmddir'] + "/" + self.piName + "/" + self.mangleCommand(exe)

        # check for collisions
        if os.path.exists(outfn):
            inc = 2
            while True:
               newfn = "%s_%d" % (outfn, inc)
               if not os.path.exists(newfn):
                  outfn = newfn
                  break
               inc +=1

        return outfn

    def collectOutputNow(self, exe, suggest_filename = None, symlink = False):
        """ Execute a command and save the output to a file for inclusion in
        the report
        """
        # First check to make sure the binary exists and is runnable.
        if not os.access(exe.split()[0], os.X_OK):
            self.soslog.log(logging.VERBOSE, "binary '%s' does not exist or is not runnable, trying anyways" % exe.split()[0])

        # FIXME: we should have a timeout or we may end waiting forever

        # pylint: disable-msg = W0612
        status, shout, runtime = sosGetCommandOutput(exe)

        if suggest_filename:
            outfn = self.makeCommandFilename(suggest_filename)
        else:
            outfn = self.makeCommandFilename(exe)

        if not os.path.isdir(os.path.dirname(outfn)):
            os.mkdir(os.path.dirname(outfn))

        if not (status == 127 or status == 32512):
            try:
                outfd = open(outfn, "w")
                if len(shout):
                    outfd.write(shout+"\n")
                outfd.close()
            except IOError, e:
                raise e

            if symlink:
                dst_from_root = outfn[len(self.cInfo['dstroot'])+1:]
                target = ("../" * string.count(symlink, "/")) + dst_from_root
                curdir = os.getcwd()
                os.chdir(self.cInfo['dstroot'])
                try:
                    os.symlink(target, symlink.strip("/."))
                except:
                    pass
                os.chdir(curdir)

            outfn_strip = outfn[len(self.cInfo['cmddir'])+1:]

        else:
            self.soslog.log(logging.VERBOSE, "could not run command: %s" % exe)
            outfn = None
            outfn_strip = None

        # sosStatus(status)
        # save info for later
        self.executedCommands.append({'exe': exe, 'file':outfn_strip}) # save in our list
        self.cInfo['xmlreport'].add_command(cmdline=exe,exitcode=status,f_stdout=outfn_strip,runtime=runtime)
        return outfn

    def writeTextToCommand(self, exe, text):
        """ A function that allows you to write a random text string to the
        command output location referenced by exe; this is useful if you want
        to conditionally collect information, but still want the output file
        to exist so as not to confuse readers """

        outfn = self.makeCommandFilename(exe)

        if not os.path.isdir(os.path.dirname(outfn)):
            os.mkdir(os.path.dirname(outfn))

        outfd = open(outfn, "w")
        outfd.write(text)
        outfd.close()

        self.executedCommands.append({'exe': exe, 'file': outfn}) # save in our list
        return outfn

    # For adding warning messages regarding configuration sanity
    def addDiagnose(self, alertstring):
        """ Add a configuration sanity warning for this plugin. These
        will be displayed on-screen before collection and in the report as well.
        """
        self.diagnose_msgs.append(alertstring)
        return
        
    # For adding output
    def addAlert(self, alertstring):
        """ Add an alert to the collection of alerts for this plugin. These
        will be displayed in the report
        """
        self.alerts.append(alertstring)
        return


    def addCustomText(self, text):
        """ Append text to the custom text that is included in the report. This
        is freeform and can include html.
        """
        self.customText = self.customText + text
        return

    def doCollect(self):
        """ This function has been replaced with copyStuff(threaded = True).  Please change your
        module calls.  Calling setup() now.
        """
        return self.copyStuff(threaded = True)

    def isRunning(self):
        """
        if threaded, is thread running ?
        """
        if self.thread: return self.thread.isAlive()
        return None
        
    def wait(self,timeout=None):
        """
        wait for a thread to complete - only called for threaded execution
        """
        self.thread.join(timeout)
        return self.thread.isAlive()

    def copyStuff(self, threaded = False, semaphore = None):
        """
        Collect the data for a plugin
        """
        if threaded and self.thread == None:
            self.thread = Thread(target=self.copyStuff, name=self.piName+'-thread', args = [True, semaphore] )
            self.thread.start()
            return self.thread

        if semaphore: semaphore.acquire()

        self.soslog.log(logging.VERBOSE, "starting threaded plugin %s" % self.piName)

        self.time_start = time()
        self.time_stop  = None

        for path in self.copyPaths:
            self.soslog.debug("copying pathspec %s" % path)
            try:
                self.doCopyFileOrDir(path)
            # propagate IO exceptions to caller
            except IOError, e:
                raise e    
            except SystemExit:
              if threaded:
                 return SystemExit
              else:
                 raise SystemExit
            except KeyboardInterrupt:
              if threaded:
                 return KeyboardInterrupt
              else:
                 raise KeyboardInterrupt
            except Exception, e:
                self.soslog.log(logging.VERBOSE2, "error copying from pathspec %s (%s), traceback follows:" % (path,e))
                self.soslog.log(logging.VERBOSE2, traceback.format_exc())
        for (prog,suggest_filename,symlink) in self.collectProgs:
            self.soslog.debug("collecting output of '%s'" % prog)
            try:
                self.collectOutputNow(prog, suggest_filename, symlink)
            # propagate IO exceptions to caller
            except IOError, e:
                raise e
            except SystemExit:
              if threaded:
                 return SystemExit
              else:
                 raise SystemExit
            except KeyboardInterrupt:
              if threaded:
                 return KeyboardInterrupt
              else:
                 raise KeyboardInterrupt
            except:
                self.soslog.log(logging.VERBOSE2, "error collection output of '%s', traceback follows:" % prog)
                self.soslog.log(logging.VERBOSE2, traceback.format_exc())

        self.time_stop = time()

        if semaphore: semaphore.release()
        self.soslog.log(logging.VERBOSE, "plugin %s returning" % self.piName)
        sys.stdout.write("\r plugin %s finished ...                   " % (self.piName,))
        sys.stdout.flush()

    def get_description(self):
        """ This function will return the description for the plugin"""
        try:
            return self.__doc__.strip()
        except:
            return "<no description available>"

    def checkenabled(self):
        """ This function can be overidden to let the plugin decide whether
        it should run or not.
        """
        return True 

    def defaultenabled(self):
        """This devices whether a plugin should be automatically loaded or
        only if manually specified in the command line."""
        return True
    
    def collect(self):
        """ This function has been replaced with setup().  Please change your
        module calls.  Calling setup() now.
        """
        self.setup()

    def diagnose(self):
        """This class must be overridden to check the sanity of the system's
        configuration before the collection begins.
        """
        pass

    def setup(self):
        """This class must be overridden to add the copyPaths, forbiddenPaths,
        and external programs to be collected at a minimum.
        """
        pass

    def analyze(self):
        """
        perform any analysis. To be replaced by a plugin if desired
        """
        pass

    def postproc(self):
        """
        perform any postprocessing. To be replaced by a plugin if desired
        """
        pass
    
    def report(self):
        """ Present all information that was gathered in an html file that allows browsing
        the results.
        """
        # make this prettier
        html = '<hr/><a name="%s"></a>\n' % self.piName

        # Intro
        html = html + "<h2> Plugin <em>" + self.piName + "</em></h2>\n"

        # Files
        if len(self.copiedFiles):
            html = html + "<p>Files copied:<br><ul>\n"
            for afile in self.copiedFiles:
                html = html + '<li><a href="%s">%s</a>' % (afile['dstpath'], afile['srcpath'])
                if (afile['symlink'] == "yes"):
                    html = html + " (symlink to %s)" % afile['pointsto']
                html = html + '</li>\n'
            html = html + "</ul></p>\n"

        # Dirs
        if len(self.copiedDirs):
            html = html + "<p>Directories Copied:<br><ul>\n"
            for adir in self.copiedDirs:
                html = html + '<li><a href="%s">%s</a>\n' % (adir['dstpath'], adir['srcpath'])
                if (adir['symlink'] == "yes"):
                    html = html + " (symlink to %s)" % adir['pointsto']
                html = html + '</li>\n'
            html = html + "</ul></p>\n"

        # Command Output
        if len(self.executedCommands):
            html = html + "<p>Commands Executed:<br><ul>\n"
            # convert file name to relative path from our root
            for cmd in self.executedCommands:
              if cmd["file"] and len(cmd["file"]):
                cmdOutRelPath = sosRelPath(self.cInfo['rptdir'], self.cInfo['cmddir'] + "/" + cmd['file'])
                html = html + '<li><a href="%s">%s</a></li>\n' % (cmdOutRelPath, cmd['exe'])
              else:
                html = html + '<li>%s</li>\n' % (cmd['exe'])
            html = html + "</ul></p>\n"

        # Alerts
        if len(self.alerts):
            html = html + "<p>Alerts:<br><ul>\n"
            for alert in self.alerts:
                html = html + '<li>%s</li>\n' % alert
            html = html + "</ul></p>\n"

        # Custom Text
        if (self.customText != ""):
            html = html + "<p>Additional Information:<br>\n"
            html = html + self.customText + "</p>\n"

        return html