#!/usr/bin/python
# vim: set fdm=marker:
__author__ = 'iseletsk'

#Copyright (c) 2014 Cloud Linux Zug GbhT
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENCE.TXT

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

import urllib
import urllib2
import urlparse
from base64 import b64encode

try:
    import gpgme, gpgme.editutil #todo: not available on ubuntu
except ImportError:
    gpgme= None

from datetime import datetime, timedelta
import subprocess
import logging.handlers
import shutil
import platform
import base64
import ConfigParser
import socket
import threading
import uuid
import logging
import os
import re
import time
import errno
import fcntl
import random

kcarelog = logging.getLogger("kcare")
kcarelog.setLevel(logging.DEBUG)
if os.path.exists('/dev/log'):
    try:
        handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_USER)
        formatter = logging.Formatter('kcare: %(message)s')
        handler.setFormatter(formatter)
        kcarelog.addHandler(handler)
    except:
        pass
else:
    handler = logging.StreamHandler()
    formatter = logging.Formatter('kcare: %(message)s')
    handler.setFormatter(formatter)
    kcarelog.addHandler(handler)

try:
    import OpenSSL
    import distutils.version
    if (distutils.version.StrictVersion(OpenSSL.__version__) <
            distutils.version.StrictVersion('0.13')):
        OpenSSL = None
    else:
        from OpenSSL.SSL import TLSv1_METHOD, Context, Connection

except ImportError:
    OpenSSL = None

VERSION="2.14-11"
KERNEL_VERSION_FILE="/proc/version"
ROLLOUT_SERVER_PROTOCOL="https://"
ROLLOUT_SERVER="rollout.kernelcare.com/v1"
ROLLOUT_SERVER_URL=ROLLOUT_SERVER_PROTOCOL + ROLLOUT_SERVER
ROLLOUT_FEED="0h"
ROLLOUT_TIME_MULTIPLIER=float(os.environ.get("ROLLOUT_TIME_MULTIPLIER", "1"))
MAX_ROLLOUT_WAIT_TIME=12 * 60 * 60 * ROLLOUT_TIME_MULTIPLIER  # 12h
ROLLOUT_SWITCH = 'n'  # 'n' always by default. Only cron job specifies this as 'auto'.
ROLLOUT_PREFIX = "0h"
SMART_UPDATE = False
PATCH_SERVER_PROTOCOL="https://"
PATCH_SERVER="patches.kernelcare.com"
PATCH_METHOD=''
PATCH_SERVER_URL=PATCH_SERVER_PROTOCOL+PATCH_SERVER
REGISTRATION_API_URL='https://cln.cloudlinux.com/api/kcare'
TEST_PREFIX=''
PATCH_BIN="kpatch.bin"
PATCH_INFO="kpatch.info"
BLACKLIST_FILE="kpatch.blacklist"
FIXUPS_FILE="kpatch.fixups"
KMOD_BIN="kcare.ko"
PATCH_DONE=".done"
PATCH_CACHE="/var/cache/kcare"
PATCH_LATEST=('latest.v2', 'latest.v1')
EFFECTIVE_LATEST = ('latest.v', 2)
CONFIG='/etc/sysconfig/kcare/kcare.conf'
SYSCTL_CONFIG='/etc/sysconfig/kcare/sysctl.conf'
FREEZER_BLACKLIST='/etc/sysconfig/kcare/freezer.modules.blacklist'
SYSTEMID='/etc/sysconfig/kcare/systemid'
KCARE_DEV='/dev/kcare'
GPG_KEY_DIR='/etc/pki/kcare-gpg/'
GPG_KEY_ID='KernelCare <info@kernelcare.com>'
CACHE_ENTRIES=3
KPATCH_CTL='/usr/libexec/kcare/kpatch_ctl'
VIRTWHAT='/usr/libexec/kcare/virt-what'
# A level to "stick on". If 0 then use latest level
LEVEL=None

if 'KCDEV' in os.environ:
    """ To simplify development, lets add environment variable.
    This would allow placing config file and system id in current dir.
    """
    CONFIG='./kcare.dev.conf' # todo fix
    SYSTEMID='./systemid'


BLACKLIST_RE = re.compile("==BLACKLIST==\n(.*)==END BLACKLIST==\n", re.DOTALL)
CONFLICTING_MODULES_RE = re.compile('(kpatch.*|ksplice.*|kpatch_livepatch.*)')
KCARE_UNAME_FILE='/proc/kcare/effective_version'

# allowed values: REMOTE, LOCAL, LOCAL_FIRST
UPDATE_POLICY="REMOTE"
AUTO_UPDATE=True
UPDATE_FROM_LOCAL=False
USE_SIGNATURE=True

IGNORE_UNKNOWN_KERNEL=False
LOAD_KCARE_SYSCTL=True
KPATCH_DEBUG=False
CHECK_SSL_CERTS=True
PATCH_TYPE=''

AWS = {
    "region": "",
    "bucket": "",
    "prefix": "",
    "access_key": "",
    "secret_key": ""
}
AWS_USE = False

PRINT_INFO=1
PRINT_WARN=2
PRINT_ERROR=3

print_level=PRINT_INFO

def set_print_level(level):
    global print_level
    print_level = level

GPG_BIN='/usr/bin/gpg'


ANCHOR_TIMEOUT = 5 # minutes


# define emulate functools.wraps{{{

def partial(func, *args, **kwds):
    "Emulate Python2.6's functools.partial"
    return lambda *fargs, **fkwds: func(*(args+fargs), **dict(kwds, **fkwds))
WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
WRAPPER_UPDATES = ('__dict__',)

def get_freezer_blacklist():
    result = set()
    try:
        f = open(FREEZER_BLACKLIST, 'r')
        for line in f:
            result.add(line.rstrip())
        f.close()
    except:
        pass
    return result

def update_wrapper(wrapper,
                   wrapped,
                   assigned = WRAPPER_ASSIGNMENTS,
                   updated = WRAPPER_UPDATES):
    """Update a wrapper function to look like the wrapped function

       wrapper is the function to be updated
       wrapped is the original function
       assigned is a tuple naming the attributes assigned directly
       from the wrapped function to the wrapper function (defaults to
       functools.WRAPPER_ASSIGNMENTS)
       updated is a tuple naming the attributes off the wrapper that
       are updated with the corresponding attribute from the wrapped
       function (defaults to functools.WRAPPER_UPDATES)
    """
    for attr in assigned:
        setattr(wrapper, attr, getattr(wrapped, attr))
    for attr in updated:
        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
    # Return the wrapper so this can be used as a decorator via partial()
    return wrapper

def wraps(wrapped,
          assigned = WRAPPER_ASSIGNMENTS,
          updated = WRAPPER_UPDATES):
    """Decorator factory to apply update_wrapper() to a wrapper function

       Returns a decorator that invokes update_wrapper() with the decorated
       function as the wrapper argument and the arguments to wraps() as the
       remaining arguments. Default arguments are as for update_wrapper().
       This is a convenience function to simplify applying partial() to
       update_wrapper().
    """
    return partial(update_wrapper, wrapped=wrapped,
                   assigned=assigned, updated=updated)

# end of wraps}}}


def _apply_ptype(ptype, filename):
    name_parts = filename.split('.')
    if ptype:
        filename = '.'.join([name_parts[0], ptype, name_parts[-1]])
    else:
        filename = '.'.join([name_parts[0], name_parts[-1]])
    return filename


def apply_ptype(ptype):
    global PATCH_BIN, PATCH_INFO, BLACKLIST_FILE, FIXUPS_FILE, PATCH_DONE
    PATCH_BIN = _apply_ptype(ptype, PATCH_BIN)
    PATCH_INFO = _apply_ptype(ptype, PATCH_INFO)
    BLACKLIST_FILE = _apply_ptype(ptype, BLACKLIST_FILE)
    FIXUPS_FILE = _apply_ptype(ptype, FIXUPS_FILE)
    PATCH_DONE = _apply_ptype(ptype, PATCH_DONE)


def printinfo(message, level):
    global print_level
    if print_level <= level:
        print(message)

def loginfo(message):
    printinfo(message, PRINT_INFO)
    kcarelog.info(message)

def logerror(message):
    printinfo(message, PRINT_ERROR)
    kcarelog.error(message)

# KCARE-509


class PreviousPatchFailedException(Exception):

    def __init__(self, timestamp, anchor, *args, **kwargs):
        super(PreviousPatchFailedException, self).__init__(*args, **kwargs)
        self.timestamp = timestamp
        self.anchor = anchor

    def __str__(self):
        return "It's seems to be a crash during latest patch applying at {0}" \
                " and further attempts will be suspended." \
                " To force patch applying remove `{1}` file".format(
                        self.timestamp, self.anchor)


def touch_anchor():
    """ Check the fact that there was a failed patching attempt.
    If anchor file not exists we shoulld create an anchor with
    timestamp and schedule its deletion at $timeout.

    If anchor exists and its timestamp more than $timeout from now
    we should raise an error.
    """
    timeout = timedelta(minutes=ANCHOR_TIMEOUT)
    anchor_filepath = os.path.join(PATCH_CACHE, '.kcareprev.lock')
    try:
        with open(anchor_filepath, 'a+', 0) as afile:
            try:
                timestamp = datetime.fromtimestamp(int(afile.read()))
                # achnor was created quite recentry
                # that means that somethong went wrong
                if timestamp + timeout > datetime.now():
                    raise PreviousPatchFailedException(timestamp, anchor_filepath)
            except ValueError:
                pass
            afile.seek(0)
            timestamp = str(int(time.mktime(datetime.now().timetuple())))
            # write a new timestamp
            afile.write(timestamp)
            afile.truncate()
            os.fsync(afile)

        # Schedule file deletion
        command = "( sleep {0} ; rm -f {1} ) &".format(
            timeout.seconds, anchor_filepath)
        subprocess.Popen(command, shell=True)

    except IOError:
        # File not found or corrupted
        pass


# source http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
    """Retry calling the decorated function using an exponential backoff.

    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry

    :param ExceptionToCheck: the exception to check. may be a tuple of
        exceptions to check
    :type ExceptionToCheck: Exception or tuple
    :param tries: number of times to try (not retry) before giving up
    :type tries: int
    :param delay: initial delay between retries in seconds
    :type delay: int
    :param backoff: backoff multiplier e.g. value of 2 will double the delay
        each retry
    :type backoff: int
    :param logger: logger to use. If None, print
    :type logger: logging.Logger instance
    """
    def deco_retry(f):

        @wraps(f)
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                except ExceptionToCheck, e:
                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
                    if logger:
                        logger.warning(msg)
                    else:
                        print msg
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            return f(*args, **kwargs)

        return f_retry  # true decorator

    return deco_retry


def save_to_file(response, dst):
    parent_dir = os.path.dirname(dst)
    if not os.path.exists(parent_dir):
        os.makedirs(parent_dir)
    with open(dst, "wb") as f:
        shutil.copyfileobj(response, f)


def clean_directory(directory, exclude_path):
    if not os.path.exists(directory):
        return
    data = []
    for item in os.listdir(directory):
        full_path = os.path.join(directory, item)
        if full_path != exclude_path:
            data.append((os.stat(full_path).st_mtime, full_path))
        data.sort(reverse=True)
    for entry in data[CACHE_ENTRIES:]:
        shutil.rmtree(entry[1])


def clear_cache(khash, plevel):
    clean_directory(os.path.join(PATCH_CACHE, 'modules'), get_kmod_cache_path(khash, ''))
    clean_directory(os.path.join(PATCH_CACHE, 'patches'), get_cache_path(khash, plevel, ''))


def get_cache_path(khash, plevel, fname):
    prefix = (TEST_PREFIX or 'none').strip('/')
    ptype = PATCH_TYPE or 'default'
    patch_dir = '-'.join([prefix, khash, plevel, ptype])
    if not fname:
        return os.path.join(PATCH_CACHE, 'patches', patch_dir)
    return os.path.join(PATCH_CACHE, 'patches', patch_dir, fname)


def get_kmod_cache_path(khash, fname):
    prefix = (TEST_PREFIX or 'none').strip('/')
    module_dir = '-'.join([prefix, khash])
    if not fname:
        return os.path.join(PATCH_CACHE, 'modules', module_dir)
    return os.path.join(PATCH_CACHE, 'modules', module_dir, fname)


def save_cache_latest(khash, patch_level):
    with open(get_kmod_cache_path(khash, 'latest'), 'w') as out:
        out.write(patch_level)


def get_cache_latest(khash):
    path_with_latest = get_kmod_cache_path(khash, 'latest')
    if os.path.isfile(path_with_latest):
        return open(path_with_latest, 'r').read()


class NotAuthorizedException(Exception):
    def __init__(self):
        Exception.__init__(self, "Invalid License")


class UnknownKernelException(Exception):
    def __init__(self):
        Exception.__init__(self, "Unknown Kernel (%s %s)" % (get_distro()[0], platform.release()))


class UnableToGetLicenseException(Exception):
    def __init__(self, code):
        Exception.__init__(self, "Unknown Issue when getting trial license, error code: "+str(code))


class ApplyPatchError(Exception):
    def __init__(self, code, freezer_style, level, patch_file):
        Exception.__init__(self, "Unable to apply patch (%s %s %d %s %s, %s, %s, %s, %s)" %
                                  ((patch_file, level, code, get_distro()[0], platform.release()) + freezer_style))


class NoTrialException(Exception):
    def __init__(self):
        Exception.__init__(self, "Not Authorized")


class AlreadyTrialedException(Exception):
    def __init__(self, ip, created):
        self.created = created[0:created.index('T')]
        self.ip = ip
        Exception.__init__(self, "The IP %s was already used for trialing on %s " % (ip, self.created))


class BadSignatureException(Exception):
    pass


def http_request(url, auth_string):
    request = urllib2.Request(url)
    if not UPDATE_FROM_LOCAL and auth_string:
        request.add_header("Authorization", "Basic %s" % auth_string)
    return request


def print_cln_http_error(ex, url=None):
    url = url or '<route cannot be logged>'
    if 'code' in ex:
        logerror("Unable to fetch %s, please, try again later "
                 "(code: %d, error: %s)" % (url, ex.code, str(ex)))
    else:
        logerror("Unable to fetch %s, please, try again later "
                 "(error: %s)" % (url, str(ex)))


def parse_response_date(str):
    str = str[0:str.index('T')]
    if hasattr(datetime, 'strptime'):
        pdate = datetime.strptime(str, "%Y-%m-%d")
    else:
        pdate = datetime(*(time.strptime(str, "%Y-%m-%d")[:6]))
    return pdate


def register_key_for_ip_license(key):
    url = REGISTRATION_API_URL + '/nagios/register_key.plain?key=%s' % key
    try:
        response = urlopen(url)
        res = parse_plain(response.read())
        code = int(res['code'])
        if code == 0:
            print "Key succesfully registered"
        elif code == 1:
            print "Wrong key size or format"
        elif code == 2:
            print "No KernelCare license for that IP"
        else:
            print "Unknown error %d" % code
        return code
    except urllib2.HTTPError, e:
        print_cln_http_error(e, url)
    return -1


def update_config(property, value):
    cf = open(CONFIG)
    lines = cf.readlines()
    cf.close()
    updated = False
    for i in xrange(0, len(lines)):
        if lines[i].startswith(property):
            lines[i] = property + " = " + str(value) + "\n"
            updated = True
            break
    if not updated:
        lines.append(property + " = " + str(value) + "\n")
    cf = open(CONFIG, "w")
    cf.writelines(lines)
    cf.close()

def plugin_info():
    """
    The output will consist of:
    Ignore output up to the line with "--START--"
    Line 1: show if update is needed:
        0 - updated to latest,
        1 - update available,
        2 - unknown kernel
        3 - kernel doesn't need patches
        4 - no license, cannot determine
    Line 2: licensing message (can be skipped, can be more then one line)
    Line 3: LICENSE: CODE: 1: license present, 2: trial license present, 0: no license
    Line 4: Update mode (True - auto-update, False, no auto update)
    Line 5: Effective kernel version
    Line 6: Real kernel version
    Line 7: Patchset Installed # --> If None, no patchset installed
    Line 8: Uptime (in seconds)

    Any other output means error retrieving info
    :return:
    """

    try:
        pli = _patch_level_info()
        update_code = pli.code
        remote_pl = pli.remote_lvl
        loaded_pl = pli.applied_lvl
        license_info_result = -1
    except AlreadyTrialedException, ae:
        loaded_pl = loaded_patch_level()
        license_info_result = 0
        license_error_message = "Your trial license for the IP %s expired on %s" % (ae.ip, ae.created)
        update_code = 4

    print("--START--")
    print(str(update_code))
    if license_info_result == 0:
        print license_error_message
    else:
        license_info_result = license_info()
    print("LICENSE: "+str(license_info_result))
    print(AUTO_UPDATE)
    print(kcare_uname())
    print(platform.release())
    print(loaded_pl)
    print(get_uptime())


def license_info():
    server_id = serverid_store.get_serverid()
    if server_id:
        url = REGISTRATION_API_URL + '/check.plain?server_id=%s' % server_id
        try:
            response = urlopen(url)
            res = parse_plain(response.read())
            code = int(res['code'])
            if code == 0:
                print("Key based valid license found")
                return 1
            else:
                license_type = _get_license_info_by_ip(key_checked=1)
                if license_type == 0:
                    print "No valid key based license found"
                return license_type
        except urllib2.HTTPError, e:
            print_cln_http_error(e, url)
            return 0
    else:
        return _get_license_info_by_ip()


def _get_license_info_by_ip(key_checked=0):
    url = REGISTRATION_API_URL + '/check.plain'
    try:
        response = urlopen(url)
        res = parse_plain(response.read())
        if res['success'].lower() == 'true':
            code = int(res['code'])
            if code == 0:
                print("Valid license found for ip %s" % (res['ip']))
                return 1 # valid license
            if code == 1:
                ip = res['ip']
                expires_str = parse_response_date(res['expire_date']).strftime("%Y-%m-%d")
                print("You have a trial license for the IP %s that will expire on %s" % (ip, expires_str))
                return 2 # trial license
            if code == 2 and key_checked == 0:
                ip = res['ip']
                expires_str = parse_response_date(res['expire_date']).strftime("%Y-%m-%d")
                print("Your trial license for the IP %s expired on %s" % (ip, expires_str))
            if code == 3 and key_checked == 0:
                if 'ip' in res:
                    print("The IP %s hasn't been licensed" % (res['ip']))
                else:
                    print("This server hasn't been licensed")
        else:
            message = res.get('message', '')
            print("Error retrieving license info %s" % (message))
    except urllib2.HTTPError, e:
        print_cln_http_error(e, url)
    except KeyError, ke:
        print("Unknown Key: %s" % ke)
    return 0 # no valid license


def register_trial():
    try:
        response = urlopen(REGISTRATION_API_URL+'/trial.plain')
        res = parse_plain(response.read())
        try:
            if res['success'].lower() == "true":
                if res['expired'] == 'true':
                    raise AlreadyTrialedException(res['ip'], res['created'])
                print "Requesting trial license for %s, please wait..." % res['ip']
                return None
            elif res['success'] == "na":
                raise NotAuthorizedException()
            else:
                raise UnableToGetLicenseException(-1) # Invalid response?
        except KeyError, ke:
            raise UnableToGetLicenseException(ke)
    except urllib2.HTTPError, e:
        raise UnableToGetLicenseException(e.code)

def get_uptime():
    f=None
    try:
        f = open('/proc/uptime', 'r')
        line = f.readline()
        result = str(int(float(line.split()[0])))
        f.close()
        return result
    except:
        if f is not None:
            f.close()
        return "-1"

def get_distro():
    if hasattr(platform, 'linux_distribution'):
        return platform.linux_distribution()
    else:
        return platform.dist()


def server_info():
    hash={}
    hash['machine']=platform.machine()
    hash['node']=platform.node()
    hash['processor']=platform.processor()
    hash['release']=platform.release()
    hash['system']=platform.system()
    hash['version']=platform.version()
    distro = get_distro()
    hash['distro']=distro[0]
    hash['distro_version']=distro[1]
    hash['euname']=kcare_uname()
    hash['patch_level']=loaded_patch_level()
    hash['virt']=check_output([VIRTWHAT])
    hash['kcare_version']=VERSION
    hash['uptime']=get_uptime()
    server_id = serverid_store.get_serverid()
    if server_id:
        hash['server_id'] = serverid_store.get_serverid()
    return base64.b16encode(str(hash))


def get_httpauth_string():
    server_id = serverid_store.get_serverid()
    if server_id:
        return base64.encodestring('%s:%s' % (server_id, 'kernelcare')).replace('\n', '')
    return None


def parse_url(url):
    # python 2.4-2.7 compatible version of urlparse
    scheme, netloc, url, query, fragment = urlparse.urlsplit(url)
    return UrlParseResult(scheme, netloc, url, query, fragment)


class UrlParseResult(object):
    def __init__(self, scheme, netloc, url, query, fragment):
        self.scheme = scheme
        self.netloc = netloc
        self.url = url
        self.query = query
        self.fragment = fragment

    @property
    def username(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.rsplit("@", 1)[0]
            if ":" in userinfo:
                userinfo = userinfo.split(":", 1)[0]
            return userinfo
        return None

    @property
    def password(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.rsplit("@", 1)[0]
            if ":" in userinfo:
                return userinfo.split(":", 1)[1]
        return None

    @property
    def hostname(self):
        netloc = self.netloc.split('@')[-1]
        if '[' in netloc and ']' in netloc:
            return netloc.split(']')[0][1:].lower()
        elif ':' in netloc:
            return netloc.split(':')[0].lower()
        elif netloc == '':
            return None
        else:
            return netloc.lower()

    @property
    def port(self):
        netloc = self.netloc.split('@')[-1].split(']')[-1]
        if ':' in netloc:
            port = netloc.split(':')[1]
            if port:
                port = int(port, 10)
                # verify legal port
                if (0 <= port <= 65535):
                    return port
        if self.scheme == 'http':
            return 80
        elif self.scheme == 'https':
            return 443
        return None


def get_proxy_server(scheme):
    proxy = None
    if scheme == 'http':
        proxy = os.getenv('http_proxy', None)
    elif scheme == 'https':
        proxy = os.getenv('https_proxy', None)
    if proxy:
        scheme, netloc, url, query, fragment = urlparse.urlsplit(proxy)
        return UrlParseResult(scheme, netloc, url, query, fragment)


def get_server_cert(server):
    sock = get_ssl_socket(server)
    conn = Connection(Context(TLSv1_METHOD), sock)
    conn.set_connect_state()
    conn.set_tlsext_host_name(server.hostname)
    conn.do_handshake()
    cert = conn.get_peer_certificate()
    conn.close()
    return cert


def get_ssl_socket(address):
    proxy = get_proxy_server(address.scheme)
    sock = socket.socket()

    if proxy is None:
        sock.connect((address.hostname, address.port))
        return sock

    headers = {
        'host': address.hostname
    }

    if proxy.password and proxy.username:
        headers['proxy-authorization'] = 'Basic ' + b64encode('%s:%s' % (proxy.username, proxy.password))

    sock = socket.socket()
    sock.connect((proxy.hostname, proxy.port))
    fp = sock.makefile('r+')

    fp.write('CONNECT %s:%d HTTP/1.0\r\n' % (address.hostname, address.port))
    fp.write('\r\n'.join('%s: %s' % (k, v) for (k, v) in headers.items()) + '\r\n\r\n')
    fp.flush()

    statusline = fp.readline().rstrip('\r\n')

    try:
        version, status, statusmsg = statusline.split(' ', 2)
        if not version in ('HTTP/1.0', 'HTTP/1.1'):
            fp.close()
            sock.close()
            raise IOError('Unsupported HTTP version')
    except ValueError:
        raise IOError('Bad response from proxy server: %s' % statusline)

    try:
        status = int(status)
    except ValueError:
        fp.close()
        sock.close()
        raise IOError('Bad response from proxy server: %s' % statusline)

    fp.close()
    if status != 200:
        raise Exception("Proxy server responded with error code: %s" % status)

    return sock


def verify_hostname(url):
    url_parse_result = parse_url(url)
    if url_parse_result.scheme == 'http':
        return True

    if OpenSSL is None:
        return True
    try:
        cert = get_server_cert(url_parse_result)
        match_hostname(cert, url_parse_result.hostname)
    except CertificateError:
        return False
    return True


def urlopen(url, *args, **kwargs):
    """ urllib2 method wrapper with optional
    hostname validation
    """

    try:
        request_url = url.get_full_url()
    except AttributeError:
        request_url = url

    if not CHECK_SSL_CERTS or verify_hostname(request_url):
        try:
            return urllib2.urlopen(url, *args, **kwargs)
        except urllib2.URLError as err:
            try:
                err.reason = "Request for `{0}` failed: {1}".format(
                    request_url, err)
            except AttributeError:
                pass
            raise err
    else:
        raise urllib2.URLError("Host ssl certificate check failed")


def cacheable(fn):
    """Cache func result and do not call it next times."""
    def wrapper(*args, **kwargs):
        if not hasattr(fn, "cached_res"):
            fn.cached_res = fn(*args, **kwargs)
        return fn.cached_res
    return wrapper


class LockExists(Exception):
    pass


def cross_process_lock(process_name):
    """Take a cross process lock basing on Linux sockets.

    Lock gets released automatically once the process is exited.
    """

    # Without holding a reference to our socket somewhere it gets garbage
    # collected when the function exits
    cross_process_lock._lock_socket = socket.socket(
        socket.AF_UNIX, socket.SOCK_DGRAM)
    try:
        cross_process_lock._lock_socket.bind('\0' + process_name)
        # got the lock
    except socket.error:
        raise LockExists()


def get_patch_file_url(patch_server, level, name):
    return patch_server + level + '/' + name

class PatchFetcher:
    def __init__(self, _hash, patch_level=None):
        self.hash = _hash
        self.patch_server = self.get_patch_server_url()
        self.auth_string = get_httpauth_string()
        self._patch_level = patch_level
        self._release_id = -1

    @property
    def patch_level(self):
        if self._patch_level is None:
            # Note that this may be blocking dependent of GradualRollout settings.
            # It may contain a wait loop of the new level from GradualRollout server.
            self._patch_level, self._release_id = self.get_latest_patch_level()
        return self._patch_level

    @property
    def release_id(self):
        return self._release_id

    def get_rollout_server_url(self):
        return urlparse.urljoin(ROLLOUT_SERVER_URL+'/', "rollout/" + ROLLOUT_PREFIX + "/" + self.hash)

    def get_patch_server_url(self):
        return urlparse.urljoin(PATCH_SERVER_URL, TEST_PREFIX + "/" + self.hash + "/")

    def _request(self, url):
        return http_request(url, self.auth_string)

    def get_latest_patch_level(self):
        if LEVEL is not None:
            release_id = -1
            return LEVEL, release_id

        patch_level_fetcher = PatchLevelFetcher(
            rollout_server=self.get_rollout_server_url(),
            patch_server=self.patch_server,
            auth_request=self._request,
        )
        return patch_level_fetcher.get_latest()

    def patch_file_exist(self, *path):
        url = self.patch_server + '/'.join(path)
        return self.url_exist(url)

    def fetch_patch_file_level(self, level, name, check_signature=False):
        url = get_patch_file_url(self.patch_server, level, name)
        dst = get_cache_path(self.hash, level, name)
        return self.fetch_url(url, dst, check_signature)

    def fetch_patch_file(self, name, check_signature=False):
        return self.fetch_patch_file_level(self.patch_level, name, check_signature)

    def fetch_kmod(self):
        url = self.patch_server + KMOD_BIN
        dst = get_kmod_cache_path(self.hash, KMOD_BIN)
        return self.fetch_url(url, dst, USE_SIGNATURE)

    @retry(urllib2.URLError, tries=4, delay=3, backoff=2)
    @retry(BadSignatureException, tries=2, delay=3, backoff=2)
    def fetch_url(self, url, dst, check_signature=False):
        try:
            response = urlopen(self._request(url))
            if dst is not None:
                save_to_file(response, dst)
            else:
                print(response.read())
        except urllib2.HTTPError, e:
            if e.code == 403:
                register_trial()
            raise e

        if check_signature:
            sig_url = url + '.sig'
            try:
                return self.check_signature(dst, sig_url)
            except BadSignatureException, err:
                if os.path.exists(dst):
                    os.unlink(dst)
                raise err
        else:
            return True

    def check_signature(self, file_path, signature_url):
        """
        check_signature checks if the signature of the file is correct.
        If signature is wrong BadSignatureException will be raised.

        :param file_path:
        :param signature_url:
        :raises: BadSignatureException
        """
        os.environ['GNUPGHOME'] = GPG_KEY_DIR
        signature = urlopen(self._request(signature_url))

        if gpgme is None:
            return self._check_gpg_signature(file_path, signature)
        else:
            return self._check_gpgme_signature(file_path, signature)

    def _check_gpg_signature(self, file_path, signature):
        """
        Check a file signature using the gpg tool.
        If signature is wrong BadSignatureException will be raised

        :param file_path: Path to file which signature will be checked
        :param signature: a file-like object with the signature
        :return: None
        :raises: BadSignatureException
        """
        check_gpg_bin()
        sig_dst = file_path + '.sig'

        save_to_file(signature, sig_dst)
        if subprocess.call([GPG_BIN, '--verify', sig_dst, file_path], stderr=fnull()) != 0:
            raise BadSignatureException("Bad Signature: %s" % file_path)  # Todo: verify validity
        else:
            return True

    def _check_gpgme_signature(self, file_path, signature):
        """
        Check a file signature using the gpgme library.
        If signature is wrong BadSignatureException will be raised

        :param file_path: a path to file which signature will be checked
        :param signature: a file-like object with the signature
        :return: None
        :raises: BadSignatureException
        """
        try:
            ctx = gpgme.Context()
        except gpgme.GpgmeError:
            # workaround bug in old pygpgme with new gpgme
            # https://bugzilla.redhat.com/show_bug.cgi?id=647059
            return self._check_gpg_signature(file_path, signature)

        valid_signatures = (gpgme.VALIDITY_FULL,
                            gpgme.VALIDITY_MARGINAL,
                            gpgme.VALIDITY_ULTIMATE)
        try:
            signed_file = open(file_path, "r")
            signatures = ctx.verify(signature, signed_file, None)
            if signatures and signatures[0] and signatures[0].validity in valid_signatures:
                return True
        except gpgme.GpgmeError:
            raise BadSignatureException("Bad Signature: %s" % file_path)
        raise BadSignatureException("Bad Signature: %s" % file_path)

    def url_exist(self, url):
        try:
            urlopen(self._request(url))
            return True
        except urllib2.URLError:
            return False

    def is_patch_fetched(self):
        return os.path.exists(get_cache_path(self.hash, self.patch_level, PATCH_DONE))

    def probe_patch(self):
        url = self.patch_server + self.patch_level + '/' + PATCH_BIN
        kcarelog.debug("Probing patch url: %s" % url)
        try:
            urlopen(self._request(url))
        except urllib2.URLError, e:
            if e.code == 404:
                return False
        return True

    def fetch_patch(self):
        if self.patch_level == "0":
            return self.patch_level

        if self.is_patch_fetched():
            loginfo("Updates already downloaded")
            return self.patch_level

        loginfo("Downloading updates")
        self.fetch_patch_file(PATCH_BIN, check_signature=USE_SIGNATURE)
        self.fetch_patch_file(PATCH_INFO, check_signature=USE_SIGNATURE)
        self.fetch_kmod()

        self.extract_blacklist()
        open(get_cache_path(self.hash, self.patch_level, PATCH_DONE), "wb").close()
        return self.patch_level

    def extract_blacklist(self):
        buf = open(get_cache_path(self.hash, self.patch_level, PATCH_INFO), 'r').read()
        if buf:
            mo = BLACKLIST_RE.search(buf)
            if mo:
                open(get_cache_path(self.hash, self.patch_level, BLACKLIST_FILE), 'w').write(mo.group(1))

    def fetch_fixups(self):
        level = loaded_patch_level()
        if level is None:
            return
        fixups = get_cache_path(self.hash, level, FIXUPS_FILE)

        if not self.patch_file_exist(level, FIXUPS_FILE):
            return

        if not os.path.exists(fixups):
            self.fetch_patch_file_level(level, FIXUPS_FILE, check_signature=USE_SIGNATURE)

        # we create this file in previous line of code if it doesn't exist earlier
        assert os.path.exists(fixups), "%s not exist" % fixups

        f = open(fixups, 'rt')
        fixups = set([fixup.strip() for fixup in f.readlines()])
        f.close()

        for fixup in fixups:
            if not os.path.exists(get_cache_path(self.hash, level, fixup)):
                self.fetch_patch_file_level(level, fixup, check_signature=USE_SIGNATURE)


class S3PatchFetcher(PatchFetcher):
    def get_patch_server_url(self):
        url_template = "%s%s/%s/%s/%s/"
        return url_template % ("https://",
                               "s3.amazonaws.com",
                               AWS["bucket"],
                               AWS["prefix"],
                               self.hash)

    def _request(self, url):
        return http_request(self.get_amazon_s3_auth(url), False)

    def get_amazon_s3_auth(self, amazon_uri):
        try:
            from hashlib import sha1
        except ImportError:
            from sha import sha as sha1
        try:
            from urllib import quote
        except ImportError:
            from urllib.parse import quote
        try:
            from urlparse import urlparse, urljoin
        except ImportError:
            from urllib.parse import urlparse, urljoin
        from hmac import new
        from datetime import timedelta
        timestamp = str(int(time.mktime((datetime.utcnow()
                                         + timedelta(120)).timetuple())))
        h = new(AWS["secret_key"],
                "GET\n\n\n%s\n%s" % (timestamp,
                                     urlparse(amazon_uri).path),
                sha1)
        sgn = quote(h.digest().encode("base64").strip())
        url_template = "%s?AWSAccessKeyId=%s&Expires=%s&Signature=%s"
        return url_template % (urljoin(amazon_uri, urlparse(amazon_uri).path),
                               AWS["access_key"],
                               timestamp,
                               sgn)


def get_kernel_hash():
    try:
        # noinspection PyCompatibility
        from hashlib import sha1
    except ImportError:
        from sha import sha as sha1
    f = open(KERNEL_VERSION_FILE, 'rb')
    try:
        return sha1(f.read()).hexdigest()
    finally:
        f.close()


@cacheable
def get_patch_fetcher(update_policy='REMOTE', level=None):
    if AWS_USE:
        cls = S3PatchFetcher
    else:
        cls = PatchFetcher

    khash = get_kernel_hash()

    if update_policy != "REMOTE":
        level = get_cache_latest(khash)
        if update_policy == "LOCAL" and level is None:
            level = "0"
    return cls(khash, level)


def kcare_check():
    pli = _patch_level_info()
    print(pli.msg)
    if pli.code == PLI.PATCH_NEED_UPDATE:
        sys.exit(0)
    else:
        sys.exit(1)


def kcare_latest_patch_info():
    """
    retrieve and output to STDOUT latest patch info, so it is easy to get
    list of CVEs in use. More info at
    https://cloudlinux.atlassian.net/browse/KCARE-952
    :return: None
    """
    url = None
    try:
        pf = get_patch_fetcher(update_policy='REMOTE')
        latest = pf.patch_level
        if not latest or latest == '0':
            print("No patches available")
            return 0
        url = get_patch_file_url(pf.patch_server, pf.patch_level, PATCH_INFO)
        pf.fetch_url(url, None)
        return 0
    except urllib2.HTTPError, e:
        print_cln_http_error(e, url)
        return 1
    except UnknownKernelException:
        print("No patches available")
        return 0


def patch_info():
    pli = patch_level_info()
    if pli.applied_lvl is None:
        return
    khash = get_kernel_hash()
    info = open(get_cache_path(khash, pli.applied_lvl, PATCH_INFO), 'r').read()
    if info:
        info = BLACKLIST_RE.sub('', info)
        print info


UNAME_LABEL = 'uname: '


def is_uname_char(c):
    return str.isalnum(c) or c in '.-_+'


def parse_uname(patch_level):
    khash = get_kernel_hash()
    f = open(get_cache_path(khash, patch_level, PATCH_INFO), 'r')
    try:
        for line in f.readlines():
            if line.startswith(UNAME_LABEL):
                return filter(is_uname_char, line[len(UNAME_LABEL):].strip())
    finally:
        f.close()
    return ""


def kcare_uname_su():
    patch_level = loaded_patch_level()
    if patch_level == None or patch_level=="0":
        return platform.release()
    return parse_uname(patch_level)

BUILD_TIME_LABEL='kpatch-build-time: '

def is_same_patch(patch_file):
    args = [KPATCH_CTL, 'file-info', patch_file]
    new_patch_info = check_output(args)
    current_patch_info = _patch_info()
    return get_patch_value(new_patch_info, BUILD_TIME_LABEL) == get_patch_value(current_patch_info, BUILD_TIME_LABEL)

def kcare_update_effective_version(new_version):
    if os.path.exists(KCARE_UNAME_FILE):
        try:
            f = open(KCARE_UNAME_FILE, "w")
            f.write(new_version)
            f.close()
            return True
        except:
            pass
    return False

def kcare_uname():
    if os.path.exists(KCARE_UNAME_FILE):
        return open(KCARE_UNAME_FILE, 'r').read().strip()
    else:
        return kcare_uname_su()


def kcare_need_update(applied_level, fetcher):
    if fetcher.patch_level == '0':
        loginfo("No updates are needed for this kernel")
        return False

    if not fetcher.probe_patch():
        if PATCH_TYPE == '':
            ptype = 'default'
        else:
            ptype = PATCH_TYPE
        raise Exception("The patch for '%s' type is not found. Please select existing patch type." % ptype)

    try:
        cur_int = int(applied_level)
        new_int = int(fetcher.patch_level)
        if new_int < cur_int:
            # Ignore down-patching
            loginfo("No updates are needed for this kernel.")
            return False
    except (TypeError, ValueError):
        pass

    if not fetcher.is_patch_fetched():
        return True
    if applied_level != fetcher.patch_level:
        return True
    patch_file = get_cache_path(fetcher.hash, applied_level, PATCH_BIN)
    if not is_same_patch(patch_file):
        return True
    return False


class FakeSecHead(object):
    def __init__(self, fp):
        self.fp = fp
        self.sechead = '[asection]\n'
    def readline(self):
        if self.sechead:
            try: return self.sechead
            finally: self.sechead = None
        else: return self.fp.readline()


def init_config_settings():
    cp = ConfigParser.ConfigParser()
    try:
        cp.readfp(FakeSecHead(open(CONFIG)))
    except:
        return

    try:
        tmp=cp.get('asection', 'UPDATE_POLICY', '')
        if tmp:
            global UPDATE_POLICY
            UPDATE_POLICY=tmp.upper()
    except:
        pass
    try:
        tmp=cp.get('asection', 'PATCH_METHOD', '')
        if tmp:
            global PATCH_METHOD
            PATCH_METHOD=tmp.upper()
    except:
        pass

    try:
        tmp=cp.get('asection', 'PATCH_SERVER', '')
        if tmp:
            global PATCH_SERVER_URL
            PATCH_SERVER_URL=tmp
    except:
        pass

    try:
        tmp=cp.get('asection', 'ROLLOUT_SERVER', '')
        if tmp:
            global ROLLOUT_SERVER_URL
            ROLLOUT_SERVER_URL=tmp
    except:
        pass

    try:
        tmp=cp.get('asection', 'AUTO_UPDATE', '1').upper()
        global AUTO_UPDATE
        AUTO_UPDATE = (tmp in ("1", "TRUE", "YES", "Y"))
    except:
        pass

    try:
        tmp=cp.get('asection', 'REGISTRATION_URL', '')
        if tmp:
            global REGISTRATION_API_URL
            REGISTRATION_API_URL = tmp
            if REGISTRATION_API_URL.endswith('/'):
                REGISTRATION_API_URL = REGISTRATION_API_URL[:-1]
    except:
        pass

    try:
        tmp=cp.get('asection', 'PREFIX', '')
        if tmp:
            global TEST_PREFIX
            TEST_PREFIX = '/'+tmp
    except:
        pass

    try:
        tmp=cp.get('asection', 'IGNORE_UNKNOWN_KERNEL', '1').upper()
        global IGNORE_UNKNOWN_KERNEL
        IGNORE_UNKNOWN_KERNEL = (tmp in ("1", "TRUE", "YES", "Y"))
    except:
        pass

    try:
        tmp=cp.get('asection', 'UPDATE_SYSCTL_CONFIG', '1').upper()
        global LOAD_KCARE_SYSCTL
        LOAD_KCARE_SYSCTL = (tmp in ("1", "TRUE", "YES", "Y"))
    except:
        pass

    try:
        tmp=cp.get('asection', 'CHECK_SSL_CERTS', '1').upper()
        global CHECK_SSL_CERTS
        CHECK_SSL_CERTS = (tmp in ("1", "TRUE", "YES", "Y"))
    except:
        pass

    try:
        global PATCH_TYPE
        PATCH_TYPE = cp.get('asection', 'PATCH_TYPE', '').lower()
    except:
        pass

    try:
        global LEVEL
        LEVEL = cp.get('asection', 'PATCH_LEVEL', None) or None
    except:
        pass

    try:
        global STICKY
        STICKY = cp.get('asection', 'STICKY_PATCH').upper()
    except:
        pass


def update_kcare_sysctl_conf():
    if LOAD_KCARE_SYSCTL:
        # Check kcare sysctl path and read access
        if not (os.path.isfile(SYSCTL_CONFIG)
                and os.access(SYSCTL_CONFIG, os.R_OK)):
            kcarelog.warning('File %s doesnt exist or has no read access' % SYSCTL_CONFIG)
            return
        # Try to load kcare sysctl
        code = subprocess.call(['/sbin/sysctl', '-q', '-p', SYSCTL_CONFIG], stdout=fnull())
        if code != 0:
            kcarelog.warning('Unable to load kcare sysctl.conf: %s' % code)


def skip_if_unk_ker(f):
    @wraps(f)
    def f_deco(*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except UnknownKernelException, e:
            if not IGNORE_UNKNOWN_KERNEL:
                raise e
            msg = str(e)
            kcarelog.warning(msg)
    return f_deco


def print_error(param):
    print param


# we need python 2.6 compatibility
def check_output(args):
    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]


def get_loaded_modules_uncached():
    return [l.split()[0] for l in open('/proc/modules')]


def get_loaded_modules():
    modules = getattr(get_loaded_modules, 'modules', None)
    if modules:
        return modules
    get_loaded_modules.modules = get_loaded_modules_uncached()
    return get_loaded_modules.modules


def detect_conflicting_modules(modules):
    for module in modules:
        if CONFLICTING_MODULES_RE.match(module):
            raise Exception("Detected '%s' kernel module loaded. Please, unload that module first." % module)


def is_kmod_version_changed(hash):
    KMOD_VERSION_FILE='/sys/module/kcare/version'

    if not os.path.exists(KMOD_VERSION_FILE):
        return True

    old_version = open(KMOD_VERSION_FILE,'r').read().strip()
    new_version = check_output(['/sbin/modinfo', '-F', 'version', get_kmod_cache_path(hash, KMOD_BIN)]).strip()
    return old_version != new_version


def get_kcare_kmod_link():
    return '/lib/modules/%s/extra/kcare.ko' % platform.uname()[2]


def load_kmod(kmod, **kwargs):
    cmd = ['/sbin/insmod', kmod]
    for key, value in kwargs.iteritems():
        cmd.append('%s=%s' % (key, value))
    code = subprocess.call(cmd, stdout=fnull())
    if code != 0:
        if inside_vz_container():
            raise Exception("You are running inside VZ container. KernelCare has to execute on host node instead.")
        raise Exception("Unable to load kmod (%s %d)" % (kmod, code))


def load_kcare_kmod(kernid):
    # To make `kdump` service work. We need to copy
    # `kcare.ko` into `/lib/modules/$(uname -r)/extra/kcare.ko`
    # and call `/sbin/depmod`
    kcare_link = get_kcare_kmod_link()
    kcare_file = get_kmod_cache_path(kernid, KMOD_BIN)
    try:
        shutil.copy(kcare_file, kcare_link)
    except Exception:
        kcare_link = kcare_file

    kmod_args = {}
    if KPATCH_DEBUG:
        kmod_args['kpatch_debug'] = 1
    load_kmod(kcare_link, **kmod_args)
    subprocess.call(['/sbin/depmod'], stdout=fnull(), stderr=fnull())


def unload_kmod(modname):
    code = subprocess.call(['/sbin/rmmod', modname], stdout=fnull())
    if code != 0:
        raise Exception("Unable to unload %s kmod %d" % (modname, code))


def apply_fixups(kernid, current_level, modules):
    loaded = []
    for mod in ['vmlinux'] + modules:
        modpath = get_cache_path(kernid, current_level, 'fixup_%s.ko' % mod)
        if os.path.exists(modpath):
            load_kmod(modpath)
            loaded.append('fixup_%s' % mod)
    return loaded


def remove_fixups(kernid, fixups):
    for mod in fixups:
        try:
            unload_kmod(mod)
        except Exception:
            pass


def fnull():
    return open(os.devnull, "w")


def get_freezer_style(freezer, modules):
    if freezer:
        method = freezer
    elif PATCH_METHOD:
        method = PATCH_METHOD
    elif get_freezer_blacklist().intersection(modules):
        # blacklist module found, use smart freezer
        # xxx: this branch could be safely removed when smart would work by default
        return ('freeze_conflict', freezer, PATCH_METHOD, True)
    else:
        # user doesn't provide patch method and no conflicting modules loaded
        return ('default', freezer, PATCH_METHOD, False)

    # non default patch method, translate it into form accepted by kpatch_ctl
    patch_method_map = {
        'NONE':     'freeze_none',
        'NOFREEZE': 'freeze_none',
        'FULL':     'freeze_all',
        'FREEZE':   'freeze_all',
        'SMART':    'freeze_conflict',
    }

    method = method.upper()

    if method in patch_method_map:
        method = patch_method_map[method]
    else:
        raise Exception("Unable to detect freezer style (%s, %s, %s, %s)" %
                        (method, freezer, PATCH_METHOD, False))
    return (method, freezer, PATCH_METHOD, False)


def kcare_load(khash, level, freezer=''):
    current_level = loaded_patch_level()
    need_fixup = current_level is not None
    # We don't want do anything if conflicting module detected.
    # So try to detect it at first.
    modules = get_loaded_modules()
    detect_conflicting_modules(modules)

    # We can be misconfigured, then get_freezer_style() will raise
    # exception. So call it here to prevent any futher job in this case.
    freezer_style = get_freezer_style(freezer, modules)

    patch_file = get_cache_path(khash, level, PATCH_BIN)
    blacklist_file = get_cache_path(khash, level, BLACKLIST_FILE)
    save_cache_latest(khash, level)

    new_version = '%s-%s;%s' % (level, PATCH_TYPE, parse_uname(level))

    if 'kcare' in modules:
        if is_same_patch(patch_file) and kcare_update_effective_version(new_version):
            # if patch isn't changed do not do anything with module
            GradualRollout.escape_ongoing()
            return
        if is_kmod_version_changed(khash):
            kcare_unload(freezer)
            load_kcare_kmod(khash)
            need_fixup = False
    else:
        load_kcare_kmod(khash)
        need_fixup = False

    if need_fixup:
        fixups = apply_fixups(khash, current_level, modules)

    args = [KPATCH_CTL]
    if os.path.exists(blacklist_file):
        args.extend(['-b', blacklist_file])

    # KCARE-509
    if SMART_UPDATE:
        touch_anchor()

    args.extend(['patch', '-d', new_version])
    args.extend(['-m', freezer_style[0]])
    args.append(patch_file)
    code = subprocess.call(args, stdout=fnull())
    if need_fixup:
        remove_fixups(khash, fixups)
    if code != 0:
        GradualRollout.escape_ongoing()
        raise ApplyPatchError(code, freezer_style, level, patch_file)
    update_kcare_sysctl_conf()
    loginfo("Patch Level %s applied, effective kernel version %s" % (level, kcare_uname()))


def kcare_unload(freezer=''):
    current_level = loaded_patch_level()

    # get latest fixups
    pf = get_patch_fetcher()
    pf.fetch_fixups()

    modules = get_loaded_modules()
    if 'kcare' in modules:
        current_level = loaded_patch_level()
        need_unpatch = current_level is not None
        if need_unpatch:
            freezer_style = get_freezer_style(freezer, modules)

            kernid = get_kernel_hash()

            fixups = apply_fixups(kernid, current_level, modules)
            code = subprocess.call([KPATCH_CTL, 'unpatch', '-m', freezer_style[0]], stdout=fnull())
            remove_fixups(kernid, fixups)
            if code != 0:
                GradualRollout.escape_ongoing()
                raise Exception("Error unpatching [%d] %s" % (code, str(freezer_style)))
        unload_kmod('kcare')
    try:
        os.unlink(get_kcare_kmod_link())
    except:
        pass

def kcare_info():
    pli = patch_level_info()
    if pli.applied_lvl == None:
        return ""
    return _patch_info()

def _patch_info():
    return check_output([KPATCH_CTL, 'info'])

DESC_LABEL="kpatch-description: "

def get_patch_value(info, label):
    lines = info.split("\n")
    for line in lines:
        line.strip()
        if line.startswith(label):
            return line[len(label):].strip()

def loaded_patch_description():
    if 'kcare' not in get_loaded_modules():
        return None
    description = get_patch_value(_patch_info(), DESC_LABEL)
    if description is not None:
        return description

def loaded_patch_level():
    desc = loaded_patch_description()
    if desc:
        return desc.split(';', 1)[0].split('-', 1)[0]

LEVEL_PATCH_AVAILABLE="PATCH_AVAILABLE"
LEVEL_UNKNOWN_KERNEL="UNKNOWN_KERNEL"
LEVEL_NO_PATCHES="NO_PATCHES"

class PLI:
    PATCH_LATEST      = 0
    PATCH_NEED_UPDATE = 1
    PATCH_UNAVALIABLE = 2
    PATCH_NOT_NEEDED  = 3

    def __init__(self, code, msg, remote_lvl, applied_lvl):
        self.code = code
        self.msg = msg
        self.remote_lvl = remote_lvl
        self.applied_lvl = applied_lvl

def _patch_level_info():
    patch_level = loaded_patch_level()
    try:
        pf = get_patch_fetcher()

        if patch_level:
            if kcare_need_update(patch_level, pf):
                code, msg = PLI.PATCH_NEED_UPDATE, "Update available, run 'kcarectl --update'."
            else:
                code, msg = PLI.PATCH_LATEST, "The latest patch is applied."
        else:
            # no patch applied
            if pf.patch_level == "0":
                code, msg = PLI.PATCH_NOT_NEEDED, "This kernel doesn't require any patches."
            else:
                code, msg = PLI.PATCH_NEED_UPDATE, "No patches applied, but some are available, run 'kcarectl --update'."
        info = PLI(code, msg, pf.patch_level, patch_level)
    except UnknownKernelException:
        code = PLI.PATCH_UNAVALIABLE
        if STICKY:
            msg = "Invalid sticky patch tag %s for kernel (%s %s). " \
                  "Please check /etc/sysconfig/kcare/kcare.conf " \
                  "STICKY_PATCH setting" % (STICKY, get_distro()[0],
                                            platform.release())
        else:
            msg  = "Unknown kernel (%s %s), no patches available" % (get_distro()[0], platform.release())
        info = PLI(code, msg, None, None)
    return info

def patch_level_info():
    pli = _patch_level_info()
    if pli.code != 0:
        print pli.msg
    return pli

def check_gpg_bin():
    if not os.path.isfile(GPG_BIN):
        raise Exception("No %s present, please install gnupg" % GPG_BIN)

GPG_OWNER_TRUST='034327E8206469CB296AC14ECCE80D2B8B53D14B:6:\n'
def import_gpg_key(import_key):
    if not os.path.exists(GPG_KEY_DIR):
        os.makedirs(GPG_KEY_DIR)
    os.environ['GNUPGHOME'] = GPG_KEY_DIR
    if gpgme is None:
        check_gpg_bin()
        subprocess.call([GPG_BIN, '--import', import_key], stderr=fnull())
        p = subprocess.Popen([GPG_BIN, '--import-ownertrust'], stdin=subprocess.PIPE, stderr=fnull()) \
            .communicate(input=GPG_OWNER_TRUST)
        return
    ctx = gpgme.Context()
    fp = open(os.path.join(GPG_KEY_DIR, 'gpg.conf'), 'wb')
    fp.write('')
    fp.close()
    f = open(import_key, 'r')
    try:
        ctx.import_(f)
    finally:
        f.close()
    k = ctx.get_key(GPG_KEY_ID)
    gpgme.editutil.edit_trust(ctx, k, gpgme.VALIDITY_ULTIMATE)


def rm_serverid():
    os.unlink(SYSTEMID)

def set_serverid(server_id):
    f = open(SYSTEMID, 'wb')
    try:
        f.write("server_id=%s\n" % server_id)
    finally:
        f.close()

def parse_plain(response):
    result = {}
    for line in response.split('\n'):
        if line and ':' in line:
            sp = line.split(':', 1)
            result[sp[0]] = sp[1]
    return result


def unregister(silent=False):
    url = None
    try:
        server_id = serverid_store.get_serverid()
        if server_id is None:
            if not silent:
                print "Error unregistering server: cannot find server id"
            return
        url = REGISTRATION_API_URL + '/unregister_server.plain?server_id=%s' % server_id
        response = urlopen(url)
        res = parse_plain(response.read())
        if res['success'] == 'true':
            rm_serverid()
            if not silent:
                print "Server was unregistered"
        elif not silent:
            print res
            print "Error unregistering server: "+res['message']
    except urllib2.HTTPError, e:
        if not silent:
            print_cln_http_error(e, url)


def register_retry(url):
    print("Register auto-retry has been enabled, system might get registered later")
    pid = os.fork()
    if pid>0:
        return
    os.setsid()
    pid = os.fork()
    import sys
    if pid > 0:
        sys.exit(0)
    sys.stdout.flush()
    si = file('/dev/null', 'r')
    so = file('/dev/null', 'a+')
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(so.fileno(), sys.stdout.fileno())
    os.dup2(so.fileno(), sys.stderr.fileno())
    while True:
        time.sleep(60*60*2)
        code, result = try_register(url)
        if result:
            sys.exit(0)

def tag_server(tag):
    """
    Request to tag server from ePortal. See KCARE-947 for more info

    :param tag: String used to tag the server
    :return: 0 on success, -1 on wrong server id, other values otherwise
    """
    url = None
    try:
        server_id = serverid_store.get_serverid()
        tag_encoded = urllib.quote(tag)
        url = REGISTRATION_API_URL + '/tag_server.plain?server_id=%s&tag=%s' % (server_id, tag_encoded)
        response = urlopen(url)
        res = parse_plain(response.read())
        return int(res['code'])
    except urllib2.HTTPError, e:
        print_cln_http_error(e, url)
        return -3
    except urllib2.URLError, ue:
        print_cln_http_error(ue, url)
        return -4
    except Exception, ee:
        print("Internal Error %s" % ee)
        return -5


def try_register(url):
    try:
        response = urlopen(url)
        res = parse_plain(response.read())
        code = res['code']
        if code == "0":
            set_serverid(res['server_id'])
            print "Server Registered"
            return code, True
        else:
            return code, False
    except urllib2.HTTPError, e:
        print_cln_http_error(e, url)
        return None, False
    except urllib2.URLError, ue:
        print_cln_http_error(ue, url)
        return None, False
    except Exception, ee:
        return None, False


def register(key, retry=False):
    try:
        unregister(True)
    except:
        pass

    query = urllib.urlencode({'key': key,
                              'hostname': platform.node()})
    url = "{0}/register_server.plain?{1}".format(REGISTRATION_API_URL, query)

    code, result = try_register(url)
    if result:
        return 0
    elif code == "1":
        print "Account Locked"
    elif code == "2":
        print "Invalid Key"
    elif code == "3":
        print "Key limit reached"
    elif code == "4":
        print "IP not allowed, please change allowed IP ranges for the key in KernelCare Key tab in CLN"
    elif code == "5":
        print "This IP was already used for trial, you cannot use it for trial again"
    elif code == "6":
        print "This IP was banned, contact support for more information"
    else:
        print "Unknown Error", code
    if retry:
        register_retry(url)
        return 0
    return code and int(code) or -1

def inside_vz_container():
    return os.path.exists('/proc/vz/veinfo') and not os.path.exists('/proc/vz/version')


def kcdoctor():
    try:
        request = urllib2.Request('https://www.cloudlinux.com/clinfo/kcdoctor.sh')
        response = urlopen(request)
        script = response.read()
        p = subprocess.Popen(['bash'], stdin=subprocess.PIPE)
        p.communicate(script)
    except Exception, e:
        logerror("Error submitting report "+str(e))

# ############## Gradual-rollout{{{

class ServerIdStore:
    def __init__(self):
        self._server_id = None
        self._temp_server_id = None

    def get_serverid(self):
        """Get the real serverid from a SYSTEMID, or None if not present."""
        if not self._server_id:
            cp = ConfigParser.ConfigParser()
            try:
                cp.readfp(FakeSecHead(open(SYSTEMID)))
                self._server_id = cp.get('asection', 'server_id', '1')
            except IOError:
                # no config, nothing to do
                self._server_id = None
        return self._server_id

    def get_temp_serverid(self):
        """Get a temp server id generated basing on a hostname and uuid.

        This is used for the clients with IP-based license, to still be able
        treat them uniquely on a rollout server.
        """
        if not self._temp_server_id:
            self._temp_server_id = "%s_%s" % (socket.gethostname(),
                                              uuid.uuid4())
        return self._temp_server_id

    def get_real_or_temp(self):
        """Get serverid if present, or temp serverid if not."""
        return self.get_serverid() or self.get_temp_serverid()

serverid_store = ServerIdStore()


class GradualRollout:
    AUTO = 'auto'
    FORCE_ENABLE = 'y'
    FORCE_DISABLE = 'n'

    @classmethod
    def is_applicable(cls):
        if ROLLOUT_SWITCH.lower() == cls.FORCE_ENABLE:
            return True
        if ROLLOUT_SWITCH.lower() == cls.FORCE_DISABLE:
            return False
        # AUTO logic below:
        in_eportal = "patches.kernelcare.com" not in PATCH_SERVER_URL
        if in_eportal:
            return False
        if UPDATE_FROM_LOCAL:
            return False
        return True

    @classmethod
    def escape(cls, release_id):
        if cls.is_applicable():
            server_id = serverid_store.get_real_or_temp()
            HeartbeatSeries(server_id, release_id).escape()

    @classmethod
    def escape_ongoing(cls):
        if cls.is_applicable():
            HeartbeatSeries.escape_ongoing()


def async(fn, *args, **kwargs):
    """Execute function in a sub-thread."""
    t = threading.Thread(target=fn, args=args, kwargs=kwargs)
    t.start()
    return t


def timeit(fn):
    def wrapped(*args, **kwags):
        start = time.time()
        try:
            return fn(*args, **kwags)
        finally:
            elapsed = time.time() - start
            kcarelog.debug("'%s' execution took %s s" % (fn.__name__, elapsed))
    return wrapped


class PatchLevelFetcher(object):
    """Responsible for fetching patch level latest VERSION"""
    def __init__(self, rollout_server, patch_server, auth_request):
        """
        Init PatchLevelFetcher.
        :param rollout_server: rollout server url
        :param patch_server: patch server url
        :param auth_request: authenticated request getter callable
        """
        self._rollout_server = rollout_server
        self._patch_server = patch_server
        self._auth_request = auth_request

    def get_latest(self):
        """
        Gets latest patch level from a predefined set of servers:
        1. Tries to reach rollout server, and get latest from there.
        Basing on a rollout logic it may wait up to 12h until active rollout
        is going.
        2. If rollout server is not accessible fetch patch level from a
        static server in usual way.

        :return: str - patch level
        """
        if not STICKY and GradualRollout.is_applicable():
            try:
                # 1. Try fetch patch_level from gradual rollout server first
                latest, release_id = self._get_latest_from_rollout_server()
                return latest, release_id
            except Exception, e:
                kcarelog.debug("Could not get latest from gradual "
                               "rollout: %s" % repr(e))
                pass
        else:
            kcarelog.debug("Gradual rollout is not applicable.")
        # 2. Try fetch patch_level from static patches server as usual.
        # Static server does not track releases, so always -1.
        latest = self._get_latest_from_patch_server()
        release_id = -1
        return latest, release_id

    def _get_latest_from_rollout_server(self):
        query = urllib.urlencode({'client_version': VERSION,
                                  'client_id': serverid_store.get_real_or_temp()})
        url = "{0}?{1}".format(self._rollout_server, query)

        wait_end = time.time() + MAX_ROLLOUT_WAIT_TIME
        while time.time() < wait_end:
            response = urlopen(http_request(url, None), timeout=3).read()
            kcarelog.debug("Rollout server response: %s" % response)

            latest, check_period, release_id = response.split(";")
            try:
                latest = int(latest)
                check_period = float(check_period)
                release_id = int(release_id)
            except (ValueError, TypeError):
                kcarelog.debug("Malformed response from "
                               "rollout server: '%s'" % response)
                raise

            current_level = loaded_patch_level()
            try:
                current_level = int(current_level or 0)
            except (ValueError, TypeError):
                kcarelog.debug("Cannot parse current patch "
                               "level: '%s'" % current_level)
                raise

            if latest > current_level:
                kcarelog.debug("Got the new patch-level '%s', with release_id "
                               "'%s' proceeding..." % (latest, release_id))
                return str(latest), release_id

            if release_id > 0:
                kcarelog.debug("Got the same or lower patch-level '%s', with "
                               "release_id '%s'. Will not apply. Escaping "
                               "heartbeat series." % (latest, release_id))
                # This is needed because if server issued release_id it is
                # expecting heartbeats from this client. So send escape signal.
                GradualRollout.escape(release_id)
            else:
                kcarelog.debug("No updates available on rollout server this "
                               "time.")

            if (check_period / 60 / 60) >= 1:
                kcarelog.debug(">1h check periods are handled by cron. Done.")
                return str(latest), -1
            else:
                kcarelog.debug("Received short check period from rollout "
                               "server. Waiting until get into rollout cap "
                               "and new patch level is recieved.")
                time.sleep(check_period)

    @retry(urllib2.URLError, tries=4, delay=3, backoff=2, logger=kcarelog)
    def _get_latest_from_patch_server(self):
        self._check_new_kc_version()
        for latest in PATCH_LATEST:
            if UPDATE_FROM_LOCAL:
                url = self._patch_server + latest
            else:
                url = self._patch_server + stickyfy(latest) + "?" + server_info()

            try:
                response = urlopen(self._auth_request(url))
                return response.read().rstrip('\n')
            except urllib2.HTTPError, e:
                if e.code == 404:
                    continue
                elif e.code == 403:
                    register_trial()
                    raise e
                else:
                    raise e
        raise UnknownKernelException()

    def _check_new_kc_version(self):
        file_base, version = EFFECTIVE_LATEST
        new_version_latest = file_base + str(version + 1)
        message = ("New version of the kernelcare package is available."
                   " To continue to get kernel updates, please, install "
                   "the new version")
        if self._patch_file_exist(new_version_latest):
            print(message)

    def _patch_file_exist(self, *path):
        url = self._patch_server + '/'.join(path)
        return self._url_exist(url)

    def _url_exist(self, url):
        try:
            urlopen(self._auth_request(url))
            return True
        except urllib2.URLError:
            return False


class RolloutPatcher(object):
    """Initiates patching."""

    def kcare_update(self, freezer='', update_policy='REMOTE'):
        """
        Patch kernel to the latest version.

        :param freezer: a freezer to use
        :param update_policy: an update policy to use (REMOTE/LOCAL)
        :param heartbeat: whether to start sending heartbeats along with
            the patching.
        :return: None
        """
        applied_level = loaded_patch_level()
        pf = get_patch_fetcher(update_policy)

        # patch_level/release_id fetched inside
        if not kcare_need_update(applied_level, pf):
            return

        if GradualRollout.is_applicable() and pf.release_id > 0:
            async(self._patch, pf, freezer)
            self._heartbeat_series(pf.release_id)
        else:
            self._patch(pf, freezer)

    @skip_if_unk_ker
    def kcare_auto_update(self, freezer=''):
        """
        Auto-update if AUTO_UPDATE setting enabled.
        Simply fetch cache the latest patch otherwise.

        :param freezer: a freezer to use
        :return:
        """
        set_print_level(PRINT_ERROR)
        if AUTO_UPDATE:
            # auto-update is the only scenario where we send heartbeats
            self.kcare_update(freezer)
        else:
            pf = get_patch_fetcher()
            pf.fetch_fixups()
            pf.fetch_patch()

    def _try_patch(self, pf, freezer, applied_level):
        try:
            self._patch(pf, freezer)
        except ApplyPatchError, err:
            if applied_level is None:
                raise err
            kcarelog.error(str(err))
            self._patch(pf, freezer, applied_level)

    def _patch(self, pf, freezer, level=None):
        pf.fetch_fixups()
        if level is None:
            level = pf.fetch_patch()
        kcare_load(pf.hash, str(level), freezer)
        clear_cache(pf.hash, str(level))
        # clear cache should keep latest entries
        pf.fetch_fixups()
        pf.fetch_patch()

    def _heartbeat_series(self, release_id):
        server_identifier = serverid_store.get_real_or_temp()
        HeartbeatSeries(server_identifier, release_id).start()


class StubRolloutPatcher(RolloutPatcher):
    def __init__(self, fail_prob, escape_prob):
        super(StubRolloutPatcher, self).__init__()
        self.i_fail = (random.uniform(0.0001, 100.0) <= fail_prob)
        kcarelog.info("I WILL FAIL: %s" % self.i_fail)
        self.i_escape = (random.uniform(0.0001, 100.0) <= escape_prob)
        kcarelog.info("I WILL ESCAPE: %s" % self.i_escape)

    def _patch(self, pf, freezer, level=None):
        rnd_delay = random.randint(1, HeartbeatSeries.get_total_time())
        # Fail/Escape simulation
        if self.i_fail:
            Sleep.for_(rnd_delay)
            kcarelog.info("FAILED CLIENT SIMULATION.")
            os._exit(1)
        elif self.i_escape:
            Sleep.for_(rnd_delay)
            kcarelog.info("ESCAPED CLIENT SIMULATION.")
            GradualRollout.escape_ongoing()

        # Patching simulation
        pf.fetch_fixups()
        level = pf.fetch_patch()
        kcarelog.info("Patching to %s" % level)

        clear_cache(pf.hash, level)
        # clear cache should keep latest entries
        pf.fetch_fixups()
        pf.fetch_patch()


rollout_patcher = RolloutPatcher()


class Sleep(object):
    """
    Sleep utils allowing precise sleeps for a long-term periods.

    On a busy server a long sleep (>1m) will make kernel scheduler to lower
    the priority of the sleeping process, which will lead to a wake-up delay,
    up to 10 seconds. This is inappropriate to us - heartbeat will be treat
    as lost on a rollout server.

    These utils split sleep period to a shorter pieces making scheduling
    more precise.
    """
    SLEEP_STEP = 30 * ROLLOUT_TIME_MULTIPLIER

    @staticmethod
    def for_(sec):
        end = time.time() + sec
        Sleep.until_(end)

    @staticmethod
    def until_(end):
        remain = end - time.time()
        while remain > 0:
            time.sleep(min(Sleep.SLEEP_STEP, remain))
            remain = end - time.time()


class HeartbeatSeries(object):
    """Represents a series of heartbeats awaited by the rollout server."""
    _escape_ongoing_flag = False
    _instance = None

    # Total series time - 12h
    series = (
        # (num_heartbeats, interval, meta)
        (5, 2 * 60 * ROLLOUT_TIME_MULTIPLIER, "Series A (total 5 / each 2min)"),
        (5, 10 * 60 * ROLLOUT_TIME_MULTIPLIER, "Series B (total 5 / each 10min)"),
        (11, 60 * 60 * ROLLOUT_TIME_MULTIPLIER, "Series C (total 11 / each 1hr)"),
    )

    def __init__(self, server_identifier, release_id):
        self.server_identifier = server_identifier
        self.release_id = release_id
        if self.__class__._instance is None:
            self.__class__._instance = self
        else:
            raise Exception("Only 1 HeartbeatSeries object can exist within "
                            "1 python kcarectl process. If you see this, "
                            "there's a bug somewhere - heartbeat series "
                            "are spawned multiple times.")

    def start(self):
        """
        Send a series of heartbeats in a way that individual heartbeat duration
        does not affect the whole schedule.
        ASSUMPTION: Max heartbeat duration =< interval
        """
        hb = Heartbeat(self.server_identifier, self.release_id)
        time_points = self._get_hb_timepoints()
        # Trigger heartbeats at time points
        for hb_time_point, meta in time_points:
            Sleep.until_(hb_time_point)
            kcarelog.info("Initiating %s" % meta)
            hb.send()
            if not hb.is_alive:
                kcarelog.info("Heartbeat debt is too big, exiting series.")
                break
            if self._escape_ongoing_flag:
                kcarelog.info("Heartbeat series escaped, exiting series.")
                self.escape(hb)
                break

    def escape(self, hb=None):
        if hb is None:
            hb = Heartbeat(self.server_identifier, self.release_id)
        # Signal server that this client is escaping the series.
        hb.send(-1)

    @classmethod
    def escape_ongoing(cls):
        """
        Signals heartbeat series loop to break on the next iteration. Escape
        means that for whatever reasons client decided to not apply the patch
        and doesn't want to be a part of server statistics further.

        We cannot check this before starting the series because often this
        requires downloading the patch, which is long process. Heartbeats are
        started asynchronously before the patch is downloaded.

        See https://cloudlinux.atlassian.net/browse/KCARE-429
        :return: None
        """
        cls._escape_ongoing_flag = True

    @classmethod
    def get_total_time(cls):
        """
        Total duration of all heartbeat series.
        :return: float (seconds)
        """
        total = 0
        for num_heartbeats, interval, meta in cls.series:
            total += num_heartbeats * interval
        return total

    def _get_hb_timepoints(self):
        # Determine all heartbeat time points ahead of time
        time_points = []
        start = time.time()
        shift = 0
        for num_heartbeats, interval, meta in self.series:
            for i in range(num_heartbeats):
                i_meta = "Heartbeat %s of %s" % (i + 1, meta)
                shift += interval
                time_points.append([start + shift, i_meta])
        return time_points


class Heartbeat(object):
    """Represents a heartbeat with a debt state."""

    # Max heartbeat duration: 3*10 + 2*5 = 40 sec
    # This affects LATENCY_ASSUMPTION on a server.
    RETRIES = 3  # but only 2 sleeps in-between
    CONNECT_TIMEOUT = 10
    RETRY_INTERVAL = 5 * ROLLOUT_TIME_MULTIPLIER

    HeartbeatPrechecks = [
        ("LockupDetector", lambda: not LockupDetector().detect()),
        ("DummyPrecheck", lambda: True),
    ]

    def __init__(self, server_identifier, release_id):
        self.server_identifier = server_identifier
        self.release_id = release_id
        self.heartbeat_debt = 1

    def send(self, override_hb_debt=None):
        """Send one heartbeat with retries, taking into account the current
        heartbeat debt."""
        if override_hb_debt is not None:
            self.heartbeat_debt = override_hb_debt
        self.not_passed_prechecks = self._hb_precheck()
        self._retry_send()

    @property
    def is_alive(self):
        """Shows whether it makes sense to continue sending heartbeats to
        the server, or this client is already treated as dead."""
        return self.heartbeat_debt < 3

    def _retry_send(self):
        retries_left = self.RETRIES
        while retries_left > 0:
            retries_left -= 1
            try:
                self._send_one(self.heartbeat_debt, self.not_passed_prechecks)
                self.heartbeat_debt = 1  # success!
                return
            except Exception, e:
                kcarelog.info("Cannot send heartbeat: %s" % repr(e))
                kcarelog.info("Retries   left: %s" % retries_left)
                kcarelog.info("Heartbeat debt: %s" % self.heartbeat_debt)
                if retries_left > 0:
                    time.sleep(self.RETRY_INTERVAL)

        # No luck this time, accumulating debt.
        self.heartbeat_debt += 1

    def _send_one(self, debt, not_passed_prechecks):
        url = "%s/%s/%s/%s" % (ROLLOUT_SERVER_URL.rstrip('/'),
                               "heartbeat",
                               self.release_id,
                               self.server_identifier)

        urlopen(http_request(url, auth_string=None),
                data=';'.join([
                    str(debt),
                    ','.join(not_passed_prechecks)
                ]),
                timeout=self.CONNECT_TIMEOUT)

    @timeit
    def _hb_precheck(self):
        return [name for name, precheck in self.HeartbeatPrechecks if not precheck()]


class LockupDetector(object):

    PROC_STAT_RE = re.compile("(?P<pid>\d+) \((?P<name>.+)\) (?P<state>\w+)")

    def __init__(self):
        self.sched_debug_state = None
        self.dev_kmsg_last_msg_no = -1
        try:
            self.dev_kmsg = open("/dev/kmsg")
            if fcntl.fcntl(self.dev_kmsg, fcntl.F_SETFL, os.O_NONBLOCK):
                raise IOError()
        except IOError:
            self.dev_kmsg = None

    @staticmethod
    def _parse_state_from_sched_debug(fh):
        result = {}

        lines = iter(fh)
        try:
            while True:
                for line in lines:
                    if line.startswith('runnable tasks'):
                        break
                next(lines) # skip headers
                next(lines)
                for line in lines:
                    if not line:
                        break
                    tokens = line.split()
                    tokens = iter(tokens)
                    for token in tokens:
                        try:
                            pid = int(token) # skip to pid, fix for 'foo bar'
                                             # names
                            break
                        except ValueError:
                            continue
                    next(tokens)
                    switches = int(next(tokens))
                    result[pid] = switches
        except StopIteration:
            pass

        return result

    def parse_state_from_sched_debug(self):
        with open("/proc/sched_debug") as fh:
            return self._parse_state_from_sched_debug(fh)

    @classmethod
    def diff_sched_debug_states(cls, new_state, old_state):
        result = []
        for pid in new_state:
            if new_state.get(pid) != old_state.get(pid):
                # process switched at least once
                continue
            with open("/proc/%s/stat" % pid) as fh:
                line = fh.readline()
                m = cls.PROC_STAT_RE.match(line)
                if not m:
                    raise ValueError("not matched %s" % line)
                if m.group('state') != 'S':
                    result.append(pid)
        return result

    def detect_sched_debug(self):
        diff = []
        new_sched_debug_state = self.parse_state_from_sched_debug()
        if self.sched_debug_state is not None:
            diff = self.diff_sched_debug_states(
                    new_sched_debug_state,
                    self.sched_debug_state)

        self.sched_debug_state = new_sched_debug_state

        if diff:
            return self.FOUND
        return self.NOT_FOUND

    @staticmethod
    def _readlines_nonblock(fh):
        lines = []

        try:
            while True:
                line = fh.readline()
                if not line:
                    break
                if line[0] == ' ':
                    lines[-1] = lines[-1] + line
                else:
                    lines.append(line)
        except IOError, e:
            # EAGAIN if no more messages are available
            #
            # EPIPE if circular bufffer was overwritten. Caller will detect
            # this from skipping message number
            if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
                raise

        return lines

    MSG_SEPARATOR = ';'
    FIELDS_SEPARATOR = ','
    LOCKUP_MSG_RE = re.compile('lockup', re.IGNORECASE)

    NOT_FOUND = 0
    FOUND = 1
    MISSING = 2

    def _analyse_kmsg_lines(self, lines):
        state = self.NOT_FOUND

        for line in lines:
            header, msg = line.split(self.MSG_SEPARATOR, 1)
            header = header.split(self.FIELDS_SEPARATOR)
            msgno = int(header[1])
            if (msgno != self.dev_kmsg_last_msg_no + 1
                and state is self.NOT_FOUND):
                print('missing', msgno, self.dev_kmsg_last_msg_no)
                state = self.MISSING
            self.dev_kmsg_last_msg_no = msgno

            if self.LOCKUP_MSG_RE.search(msg):
                state = self.FOUND
        return state

    def detect_dev_kmsg(self):
        if self.dev_kmsg is None:
            return self.MISSING
        lines = self._readlines_nonblock(self.dev_kmsg)
        return self._analyse_kmsg_lines(lines)

    def detect(self):
        try:
            state = self.detect_dev_kmsg()
            if state is self.MISSING:
                state = self.detect_sched_debug()
            return state == self.FOUND
        except Exception:
            return False


""" 
This is needed to support sticky keys as per 
https://cloudlinux.atlassian.net/browse/KCARE-953 
"""
STICKY = False

def _stickfy(prefix, file):
    #  todo VERIFY prefix is valid, exit with message otherwise
    return prefix + '.' + file


def stickyfy(file):
    """
    Used to add sticky prefix to satisfy KCARE-953
    :param file: name of the file to stickify
    :return: stickified file.
    """
    if STICKY:
        if STICKY != 'KEY':
            return _stickfy(STICKY, file)

        url = None
        try:
            server_id = serverid_store.get_serverid()
            if server_id:
                response = urlopen(REGISTRATION_API_URL + '/sticky_patch.plain?server_id=%s' % server_id)
                res = parse_plain(response.read())
                code = int(res['code'])
                if code == 0:
                    return _stickfy(res['prefix'], file)
                if code == 1:
                    return file
                if code == 2:
                    kcarelog.info("Server ID is not recognized, "
                                  "is server registered?")
                    sys.exit(-1)
                kcarelog.info("Error: " + res['message'])
                sys.exit(-3)
            else:
                kcarelog.info("Patch set to STICKY=KEY, but server is "
                              "not registered with the key")
                sys.exit(-4)
        except urllib2.HTTPError, e:
            print_cln_http_error(e, url)
            sys.exit(-5)
    return file




# ############## }}}END Gradual-rollout

def main():
    parser = ArgumentParser(description='Manage KernelCare patches for your kernel')
    parser.add_argument("-i", "--info",
                        help="Display information about KernelCare",
                        action="store_true")
    parser.add_argument("-u", "--update",
                        help="Download latest patches, and apply them to current kernel",
                        action="store_true")
    parser.add_argument("--unload",
                        help="Unload patches",
                        action="store_true")
    parser.add_argument("--smart-update",
                        help="Try to patch kernel based on UPDATE POLICY settings",
                        action="store_true")
    parser.add_argument("--auto-update",
                        help="Check if update is needed, and update",
                        action="store_true")
    parser.add_argument("--local",
                        help="Update from local directory on a server, accepts path where patches are located",
                        metavar='PATH')
    parser.add_argument("--patch-info",
                        help="Lists applied patched",
                        action="store_true")
    parser.add_argument("--freezer",
                        help="Freezer type, available: full (default), smart, none",
                        metavar='freezer')
    parser.add_argument("--nofreeze",
                        help="[deprecated]don't freeze tasks before patch",
                        action="store_true")
    parser.add_argument("--force",
                        help="[deprecated] when used with update, forces applying the patch even if unable to freeze some threads",
                        action="store_true")
    parser.add_argument("--uname",
                        help="Prints safe kernel version",
                        action="store_true")
    parser.add_argument("--license-info",
                        help="Output current license info",
                        action="store_true")
    parser.add_argument("--import-key",
                        help="Import gpg key", metavar='PATH')
    parser.add_argument("--register",
                        help="Register using KernelCare Key",
                        metavar='KEY')
    parser.add_argument('--register-autoretry',
                        help='Retry registering indefinatly if failed on first attempt',
                        action="store_true")
    parser.add_argument("--unregister",
                        help="Unregister from KernelCare for Key based servers",
                        action="store_true")
    parser.add_argument("--check",
                        help="Check if new update available",
                        action="store_true")
    parser.add_argument("--latest-patch-info",
                        help="Outputs patch info for latest available patch",
                        action="store_true")
    parser.add_argument("--test",
                        help="Try test builds instead of production builds (deprecated, use --prefix=test instead)",
                        action="store_true")
    parser.add_argument('--tag',
                        help='Tag server with custom metadata, ePortal only',
                        metavar='TAG')
    parser.add_argument('--prefix',
                        help="Patch source prefix, used to test different builds by downloading builds from different "+
                             "locations based on prefix",
                        metavar='PREFIX')
    # Use StubPatcher instead of the real one - do not do any real patching, only log.
    # Used as --stub-patcher=<fail_probability_percent>,<escape_probability_percent> e.g.
    # Used as --stub-patcher=2,10 or --stub-patcher=0,0
    parser.add_argument("--stub-patcher",
                        help=SUPPRESS)
    # Use given hash & patchlevel instead of the real server ones. Typically used with stub_patcher.
    # Used as --stub-hash=<hash>,<patchlevel>.
    parser.add_argument("--stub-hash",
                        help=SUPPRESS)
    parser.add_argument("--gradual-rollout",
                        help="Manually control gradual rollout behavior:\n"
                             "--gradual-rollout=auto - on/off automatically\n"
                             "--gradual-rollout=y - force-enable\n"
                             "--gradual-rollout=n - force-disable (default)")
    parser.add_argument("--test-s3",
                        help="Try test builds from Amazon S3 instead of patch server",
                        action="store_true")
    parser.add_argument("--aws-access-key",
                        help="AWS Access key",
                        metavar='KEY')
    parser.add_argument("--aws-secret-key",
                        help="AWS Secret key",
                        metavar='KEY')
    parser.add_argument("--aws-region",
                        help="AWS Region",
                        metavar='REGION')
    parser.add_argument("--aws-bucket",
                        help="Amazon S3 bucket name",
                        metavar='BUCKET')
    parser.add_argument("--aws-prefix",
                        help="Prefix for navigation in S3 bucket",
                        metavar='PREFIX')
    parser.add_argument("--nosignature",
                        help="Do not check signature",
                        action="store_true")
    parser.add_argument('--set-monitoring-key',
                        help="Set monitoring key for IP based licenses. 16 to 32 characters, alphanumeric only",
                        metavar='KEY')
    parser.add_argument('--doctor',
                        help="Submits a vitals report to CloudLinux for analysis by CloudLinux staff",
                        action="store_true")
    parser.add_argument('--enable-auto-update',
                        help="Enable auto updates",
                        action="store_true")
    parser.add_argument('--disable-auto-update',
                        help="Disable auto updates",
                        action="store_true")
    parser.add_argument('--plugin-info',
                        help="Provides range of information shown in control panel plugins for KernelCare",
                        action="store_true")
    parser.add_argument("--version",
                        help="Print KernelCare version",
                        action="store_true")
    parser.add_argument("--kpatch-debug",
                        help="Enables debug mode",
                        action="store_true")
    parser.add_argument("--no-check-cert",
                        help="Disable the patch server ssl certificates checking",
                        action="store_true")
    parser.add_argument("--set-patch-type", help="Set patch type feed. To select default feed use 'default' option",
                        action="store")
    parser.add_argument("--set-patch-level", help="Set patch level to be applied. To select latest patch level set -1",
                        action="store", type=int, default=None, required=False)

    parser.add_argument('--set-sticky-patch',
                        help="Set patch to stick to date in format DDMMYY, or retrieve it from KEY if set to KEY. Empty to unstick",
                        action="store", default=None, required=False)

    args = parser.parse_args()

    init_config_settings()

    if args.set_patch_level:
        global LEVEL
        if args.set_patch_level >= 0:
            LEVEL = str(args.set_patch_level)
            update_config("PATCH_LEVEL", LEVEL)
        else:
            LEVEL = None
            update_config("PATCH_LEVEL", '')

    if args.set_sticky_patch is not None:
        update_config("STICKY_PATCH", args.set_sticky_patch)

    if args.nosignature:
        global USE_SIGNATURE
        USE_SIGNATURE=False

    if args.no_check_cert:
        global CHECK_SSL_CERTS
        CHECK_SSL_CERTS = False

    if args.kpatch_debug:
        global KPATCH_DEBUG
        KPATCH_DEBUG=True

    global TEST_PREFIX
    if args.prefix:
        assert args.prefix in ('12h', '24h', '48h')
        TEST_PREFIX = '/' + args.prefix
        global ROLLOUT_PREFIX
        ROLLOUT_PREFIX = args.prefix
    if args.test:
        TEST_PREFIX = '/test'

    if args.local:
        global UPDATE_FROM_LOCAL
        UPDATE_FROM_LOCAL = True
        global PATCH_SERVER_URL
        PATCH_SERVER_URL = 'file:'+args.local

    if args.test_s3:
        global AWS_USE
        global AWS
        AWS_USE = True
        if args.aws_access_key:
            AWS["access_key"] = args.aws_access_key
        if args.aws_secret_key:
            AWS["secret_key"] = args.aws_secret_key
        if args.aws_region:
            AWS["region"] = args.aws_region
        if args.aws_bucket:
            AWS["bucket"] = args.aws_bucket
        if args.aws_prefix:
            AWS["prefix"] = args.aws_prefix

    global PATCH_TYPE
    if args.set_patch_type:
        if args.set_patch_type != 'default':
            PATCH_TYPE = args.set_patch_type
        else:
            PATCH_TYPE = ''

        apply_ptype(PATCH_TYPE)

        if get_patch_fetcher().probe_patch():
            update_config('PATCH_TYPE', PATCH_TYPE)
            print "'%s' patch type selected" % args.set_patch_type
        else:
            raise Exception("'%s' patch type is unavailable for current kernel" % args.set_patch_type)
    else:
        apply_ptype(PATCH_TYPE)

    if args.doctor:
        kcdoctor()
        return

    if args.plugin_info:
        plugin_info()
        return

    if args.enable_auto_update:
        update_config("AUTO_UPDATE", "YES")
        return

    if args.disable_auto_update:
        update_config("AUTO_UPDATE", "NO")
        return

    if args.set_monitoring_key:
        return register_key_for_ip_license(args.set_monitoring_key)
    if args.unregister:
        unregister()
    if args.register:
        if PATCH_TYPE == 'free':
            update_config('PATCH_TYPE', 'extra')
        return register(args.register, args.register_autoretry)
    if args.license_info:
        # license_info returns zero if no valid license found and non-zero otherwise
        if license_info() != 0:
            return 0
        else:
            return 1

    if args.tag is not None:
        return tag_server(args.tag)

    if args.stub_patcher:
        fail_prob, escape_prob = args.stub_patcher.split(",")
        fail_prob, escape_prob = float(fail_prob), float(escape_prob)
        global rollout_patcher
        rollout_patcher = StubRolloutPatcher(fail_prob, escape_prob)
    if args.stub_hash:
        global get_kernel_hash
        global loaded_patch_level
        stub_hash, stub_patch_level = args.stub_hash.split(",")
        get_kernel_hash = lambda: stub_hash
        loaded_patch_level = lambda: stub_patch_level
    global ROLLOUT_SWITCH
    global SMART_UPDATE
    if args.gradual_rollout:
        ROLLOUT_SWITCH = args.gradual_rollout
    if args.version:
        print VERSION
    if args.info:
        print kcare_info()
    freezer = ''
    if (args.force or args.nofreeze):
        freezer = 'none'
    if args.freezer:
        freezer = args.freezer
    if args.smart_update:
        ROLLOUT_SWITCH = GradualRollout.FORCE_DISABLE
        SMART_UPDATE = True
        rollout_patcher.kcare_update(freezer, UPDATE_POLICY)
    if args.update:
        ROLLOUT_SWITCH = GradualRollout.FORCE_DISABLE
        rollout_patcher.kcare_update(freezer)
        print "Kernel is safe"
    if args.uname:
        print kcare_uname()
    if args.unload:
        kcare_unload(freezer)
        print "KernelCare protection disabled, kernel might not be safe"
    if args.auto_update:
        rollout_patcher.kcare_auto_update(freezer)
    if args.patch_info:
        patch_info()
    if args.latest_patch_info:
        kcare_latest_patch_info()
    if args.import_key:
        import_gpg_key(args.import_key)
    if args.check:
        kcare_check()





########Argparse{{{###############

############################ Author: Steven J. Bethard <steven.bethard@gmail.com>.
# Author: Steven J. Bethard <steven.bethard@gmail.com>.

"""Command-line parsing library

This module is an optparse-inspired command-line parsing library that:

    - handles both optional and positional arguments
    - produces highly informative usage messages
    - supports parsers that dispatch to sub-parsers

The following is a simple usage example that sums integers from the
command-line and writes the result to a file::

    parser = argparse.ArgumentParser(
        description='sum the integers at the command line')
    parser.add_argument(
        'integers', metavar='int', nargs='+', type=int,
        help='an integer to be summed')
    parser.add_argument(
        '--log', default=sys.stdout, type=argparse.FileType('w'),
        help='the file where the sum should be written')
    args = parser.parse_args()
    args.log.write('%s' % sum(args.integers))
    args.log.close()

The module contains the following public classes:

    - ArgumentParser -- The main entry point for command-line parsing. As the
        example above shows, the add_argument() method is used to populate
        the parser with actions for optional and positional arguments. Then
        the parse_args() method is invoked to convert the args at the
        command-line into an object with attributes.

    - ArgumentError -- The exception raised by ArgumentParser objects when
        there are errors with the parser's actions. Errors raised while
        parsing the command-line are caught by ArgumentParser and emitted
        as command-line messages.

    - FileType -- A factory for defining types of files to be created. As the
        example above shows, instances of FileType are typically passed as
        the type= argument of add_argument() calls.

    - Action -- The base class for parser actions. Typically actions are
        selected by passing strings like 'store_true' or 'append_const' to
        the action= argument of add_argument(). However, for greater
        customization of ArgumentParser actions, subclasses of Action may
        be defined and passed as the action= argument.

    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
        ArgumentDefaultsHelpFormatter -- Formatter classes which
        may be passed as the formatter_class= argument to the
        ArgumentParser constructor. HelpFormatter is the default,
        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
        not to change the formatting for help text, and
        ArgumentDefaultsHelpFormatter adds information about argument defaults
        to the help.

All other classes in this module are considered implementation details.
(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
considered public as object names -- the API of the formatter objects is
still considered an implementation detail.)
"""

__version__ = '1.2.1'
__all__ = [
    'ArgumentParser',
    'ArgumentError',
    'ArgumentTypeError',
    'FileType',
    'HelpFormatter',
    'ArgumentDefaultsHelpFormatter',
    'RawDescriptionHelpFormatter',
    'RawTextHelpFormatter',
    'Namespace',
    'Action',
    'ONE_OR_MORE',
    'OPTIONAL',
    'PARSER',
    'REMAINDER',
    'SUPPRESS',
    'ZERO_OR_MORE',
]


import copy as _copy
import os as _os
import re as _re
import sys as _sys
import textwrap as _textwrap

from gettext import gettext as _

try:
    set
except NameError:
    # for python < 2.4 compatibility (sets module is there since 2.3):
    from sets import Set as set

try:
    basestring
except NameError:
    basestring = str

try:
    sorted
except NameError:
    # for python < 2.4 compatibility:
    def sorted(iterable, reverse=False):
        result = list(iterable)
        result.sort()
        if reverse:
            result.reverse()
        return result


def _callable(obj):
    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')


SUPPRESS = '==SUPPRESS=='

OPTIONAL = '?'
ZERO_OR_MORE = '*'
ONE_OR_MORE = '+'
PARSER = 'A...'
REMAINDER = '...'
_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'

# =============================
# Utility functions and classes
# =============================

class _AttributeHolder(object):
    """Abstract base class that provides __repr__.

    The __repr__ method returns a string in the format::
        ClassName(attr=name, attr=name, ...)
    The attributes are determined either by a class-level attribute,
    '_kwarg_names', or by inspecting the instance __dict__.
    """

    def __repr__(self):
        type_name = type(self).__name__
        arg_strings = []
        for arg in self._get_args():
            arg_strings.append(repr(arg))
        for name, value in self._get_kwargs():
            arg_strings.append('%s=%r' % (name, value))
        return '%s(%s)' % (type_name, ', '.join(arg_strings))

    def _get_kwargs(self):
        return sorted(self.__dict__.items())

    def _get_args(self):
        return []


def _ensure_value(namespace, name, value):
    if getattr(namespace, name, None) is None:
        setattr(namespace, name, value)
    return getattr(namespace, name)


# ===============
# Formatting Help
# ===============

class HelpFormatter(object):
    """Formatter for generating usage messages and argument help strings.

    Only the name of this class is considered a public API. All the methods
    provided by the class are considered an implementation detail.
    """

    def __init__(self,
                 prog,
                 indent_increment=2,
                 max_help_position=24,
                 width=None):

        # default setting for width
        if width is None:
            try:
                width = int(_os.environ['COLUMNS'])
            except (KeyError, ValueError):
                width = 80
            width -= 2

        self._prog = prog
        self._indent_increment = indent_increment
        self._max_help_position = max_help_position
        self._width = width

        self._current_indent = 0
        self._level = 0
        self._action_max_length = 0

        self._root_section = self._Section(self, None)
        self._current_section = self._root_section

        self._whitespace_matcher = _re.compile(r'\s+')
        self._long_break_matcher = _re.compile(r'\n\n\n+')

    # ===============================
    # Section and indentation methods
    # ===============================
    def _indent(self):
        self._current_indent += self._indent_increment
        self._level += 1

    def _dedent(self):
        self._current_indent -= self._indent_increment
        assert self._current_indent >= 0, 'Indent decreased below 0.'
        self._level -= 1

    class _Section(object):

        def __init__(self, formatter, parent, heading=None):
            self.formatter = formatter
            self.parent = parent
            self.heading = heading
            self.items = []

        def format_help(self):
            # format the indented section
            if self.parent is not None:
                self.formatter._indent()
            join = self.formatter._join_parts
            for func, args in self.items:
                func(*args)
            item_help = join([func(*args) for func, args in self.items])
            if self.parent is not None:
                self.formatter._dedent()

            # return nothing if the section was empty
            if not item_help:
                return ''

            # add the heading if the section was non-empty
            if self.heading is not SUPPRESS and self.heading is not None:
                current_indent = self.formatter._current_indent
                heading = '%*s%s:\n' % (current_indent, '', self.heading)
            else:
                heading = ''

            # join the section-initial newline, the heading and the help
            return join(['\n', heading, item_help, '\n'])

    def _add_item(self, func, args):
        self._current_section.items.append((func, args))

    # ========================
    # Message building methods
    # ========================
    def start_section(self, heading):
        self._indent()
        section = self._Section(self, self._current_section, heading)
        self._add_item(section.format_help, [])
        self._current_section = section

    def end_section(self):
        self._current_section = self._current_section.parent
        self._dedent()

    def add_text(self, text):
        if text is not SUPPRESS and text is not None:
            self._add_item(self._format_text, [text])

    def add_usage(self, usage, actions, groups, prefix=None):
        if usage is not SUPPRESS:
            args = usage, actions, groups, prefix
            self._add_item(self._format_usage, args)

    def add_argument(self, action):
        if action.help is not SUPPRESS:

            # find all invocations
            get_invocation = self._format_action_invocation
            invocations = [get_invocation(action)]
            for subaction in self._iter_indented_subactions(action):
                invocations.append(get_invocation(subaction))

            # update the maximum item length
            invocation_length = max([len(s) for s in invocations])
            action_length = invocation_length + self._current_indent
            self._action_max_length = max(self._action_max_length,
                                          action_length)

            # add the item to the list
            self._add_item(self._format_action, [action])

    def add_arguments(self, actions):
        for action in actions:
            self.add_argument(action)

    # =======================
    # Help-formatting methods
    # =======================
    def format_help(self):
        help = self._root_section.format_help()
        if help:
            help = self._long_break_matcher.sub('\n\n', help)
            help = help.strip('\n') + '\n'
        return help

    def _join_parts(self, part_strings):
        return ''.join([part
                        for part in part_strings
                        if part and part is not SUPPRESS])

    def _format_usage(self, usage, actions, groups, prefix):
        if prefix is None:
            prefix = _('usage: ')

        # if usage is specified, use that
        if usage is not None:
            usage = usage % dict(prog=self._prog)

        # if no optionals or positionals are available, usage is just prog
        elif usage is None and not actions:
            usage = '%(prog)s' % dict(prog=self._prog)

        # if optionals and positionals are available, calculate usage
        elif usage is None:
            prog = '%(prog)s' % dict(prog=self._prog)

            # split optionals from positionals
            optionals = []
            positionals = []
            for action in actions:
                if action.option_strings:
                    optionals.append(action)
                else:
                    positionals.append(action)

            # build full usage string
            format = self._format_actions_usage
            action_usage = format(optionals + positionals, groups)
            usage = ' '.join([s for s in [prog, action_usage] if s])

            # wrap the usage parts if it's too long
            text_width = self._width - self._current_indent
            if len(prefix) + len(usage) > text_width:

                # break usage into wrappable parts
                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
                opt_usage = format(optionals, groups)
                pos_usage = format(positionals, groups)
                opt_parts = _re.findall(part_regexp, opt_usage)
                pos_parts = _re.findall(part_regexp, pos_usage)
                assert ' '.join(opt_parts) == opt_usage
                assert ' '.join(pos_parts) == pos_usage

                # helper for wrapping lines
                def get_lines(parts, indent, prefix=None):
                    lines = []
                    line = []
                    if prefix is not None:
                        line_len = len(prefix) - 1
                    else:
                        line_len = len(indent) - 1
                    for part in parts:
                        if line_len + 1 + len(part) > text_width:
                            lines.append(indent + ' '.join(line))
                            line = []
                            line_len = len(indent) - 1
                        line.append(part)
                        line_len += len(part) + 1
                    if line:
                        lines.append(indent + ' '.join(line))
                    if prefix is not None:
                        lines[0] = lines[0][len(indent):]
                    return lines

                # if prog is short, follow it with optionals or positionals
                if len(prefix) + len(prog) <= 0.75 * text_width:
                    indent = ' ' * (len(prefix) + len(prog) + 1)
                    if opt_parts:
                        lines = get_lines([prog] + opt_parts, indent, prefix)
                        lines.extend(get_lines(pos_parts, indent))
                    elif pos_parts:
                        lines = get_lines([prog] + pos_parts, indent, prefix)
                    else:
                        lines = [prog]

                # if prog is long, put it on its own line
                else:
                    indent = ' ' * len(prefix)
                    parts = opt_parts + pos_parts
                    lines = get_lines(parts, indent)
                    if len(lines) > 1:
                        lines = []
                        lines.extend(get_lines(opt_parts, indent))
                        lines.extend(get_lines(pos_parts, indent))
                    lines = [prog] + lines

                # join lines into usage
                usage = '\n'.join(lines)

        # prefix with 'usage:'
        return '%s%s\n\n' % (prefix, usage)

    def _format_actions_usage(self, actions, groups):
        # find group indices and identify actions in groups
        group_actions = set()
        inserts = {}
        for group in groups:
            try:
                start = actions.index(group._group_actions[0])
            except ValueError:
                continue
            else:
                end = start + len(group._group_actions)
                if actions[start:end] == group._group_actions:
                    for action in group._group_actions:
                        group_actions.add(action)
                    if not group.required:
                        if start in inserts:
                            inserts[start] += ' ['
                        else:
                            inserts[start] = '['
                        inserts[end] = ']'
                    else:
                        if start in inserts:
                            inserts[start] += ' ('
                        else:
                            inserts[start] = '('
                        inserts[end] = ')'
                    for i in range(start + 1, end):
                        inserts[i] = '|'

        # collect all actions format strings
        parts = []
        for i, action in enumerate(actions):

            # suppressed arguments are marked with None
            # remove | separators for suppressed arguments
            if action.help is SUPPRESS:
                parts.append(None)
                if inserts.get(i) == '|':
                    inserts.pop(i)
                elif inserts.get(i + 1) == '|':
                    inserts.pop(i + 1)

            # produce all arg strings
            elif not action.option_strings:
                part = self._format_args(action, action.dest)

                # if it's in a group, strip the outer []
                if action in group_actions:
                    if part[0] == '[' and part[-1] == ']':
                        part = part[1:-1]

                # add the action string to the list
                parts.append(part)

            # produce the first way to invoke the option in brackets
            else:
                option_string = action.option_strings[0]

                # if the Optional doesn't take a value, format is:
                #    -s or --long
                if action.nargs == 0:
                    part = '%s' % option_string

                # if the Optional takes a value, format is:
                #    -s ARGS or --long ARGS
                else:
                    default = action.dest.upper()
                    args_string = self._format_args(action, default)
                    part = '%s %s' % (option_string, args_string)

                # make it look optional if it's not required or in a group
                if not action.required and action not in group_actions:
                    part = '[%s]' % part

                # add the action string to the list
                parts.append(part)

        # insert things at the necessary indices
        for i in sorted(inserts, reverse=True):
            parts[i:i] = [inserts[i]]

        # join all the action items with spaces
        text = ' '.join([item for item in parts if item is not None])

        # clean up separators for mutually exclusive groups
        open = r'[\[(]'
        close = r'[\])]'
        text = _re.sub(r'(%s) ' % open, r'\1', text)
        text = _re.sub(r' (%s)' % close, r'\1', text)
        text = _re.sub(r'%s *%s' % (open, close), r'', text)
        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
        text = text.strip()

        # return the text
        return text

    def _format_text(self, text):
        if '%(prog)' in text:
            text = text % dict(prog=self._prog)
        text_width = self._width - self._current_indent
        indent = ' ' * self._current_indent
        return self._fill_text(text, text_width, indent) + '\n\n'

    def _format_action(self, action):
        # determine the required width and the entry label
        help_position = min(self._action_max_length + 2,
                            self._max_help_position)
        help_width = self._width - help_position
        action_width = help_position - self._current_indent - 2
        action_header = self._format_action_invocation(action)

        # ho nelp; start on same line and add a final newline
        if not action.help:
            tup = self._current_indent, '', action_header
            action_header = '%*s%s\n' % tup

        # short action name; start on the same line and pad two spaces
        elif len(action_header) <= action_width:
            tup = self._current_indent, '', action_width, action_header
            action_header = '%*s%-*s  ' % tup
            indent_first = 0

        # long action name; start on the next line
        else:
            tup = self._current_indent, '', action_header
            action_header = '%*s%s\n' % tup
            indent_first = help_position

        # collect the pieces of the action help
        parts = [action_header]

        # if there was help for the action, add lines of help text
        if action.help:
            help_text = self._expand_help(action)
            help_lines = self._split_lines(help_text, help_width)
            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
            for line in help_lines[1:]:
                parts.append('%*s%s\n' % (help_position, '', line))

        # or add a newline if the description doesn't end with one
        elif not action_header.endswith('\n'):
            parts.append('\n')

        # if there are any sub-actions, add their help as well
        for subaction in self._iter_indented_subactions(action):
            parts.append(self._format_action(subaction))

        # return a single string
        return self._join_parts(parts)

    def _format_action_invocation(self, action):
        if not action.option_strings:
            metavar, = self._metavar_formatter(action, action.dest)(1)
            return metavar

        else:
            parts = []

            # if the Optional doesn't take a value, format is:
            #    -s, --long
            if action.nargs == 0:
                parts.extend(action.option_strings)

            # if the Optional takes a value, format is:
            #    -s ARGS, --long ARGS
            else:
                default = action.dest.upper()
                args_string = self._format_args(action, default)
                for option_string in action.option_strings:
                    parts.append('%s %s' % (option_string, args_string))

            return ', '.join(parts)

    def _metavar_formatter(self, action, default_metavar):
        if action.metavar is not None:
            result = action.metavar
        elif action.choices is not None:
            choice_strs = [str(choice) for choice in action.choices]
            result = '{%s}' % ','.join(choice_strs)
        else:
            result = default_metavar

        def format(tuple_size):
            if isinstance(result, tuple):
                return result
            else:
                return (result, ) * tuple_size
        return format

    def _format_args(self, action, default_metavar):
        get_metavar = self._metavar_formatter(action, default_metavar)
        if action.nargs is None:
            result = '%s' % get_metavar(1)
        elif action.nargs == OPTIONAL:
            result = '[%s]' % get_metavar(1)
        elif action.nargs == ZERO_OR_MORE:
            result = '[%s [%s ...]]' % get_metavar(2)
        elif action.nargs == ONE_OR_MORE:
            result = '%s [%s ...]' % get_metavar(2)
        elif action.nargs == REMAINDER:
            result = '...'
        elif action.nargs == PARSER:
            result = '%s ...' % get_metavar(1)
        else:
            formats = ['%s' for _ in range(action.nargs)]
            result = ' '.join(formats) % get_metavar(action.nargs)
        return result

    def _expand_help(self, action):
        params = dict(vars(action), prog=self._prog)
        for name in list(params):
            if params[name] is SUPPRESS:
                del params[name]
        for name in list(params):
            if hasattr(params[name], '__name__'):
                params[name] = params[name].__name__
        if params.get('choices') is not None:
            choices_str = ', '.join([str(c) for c in params['choices']])
            params['choices'] = choices_str
        return self._get_help_string(action) % params

    def _iter_indented_subactions(self, action):
        try:
            get_subactions = action._get_subactions
        except AttributeError:
            pass
        else:
            self._indent()
            for subaction in get_subactions():
                yield subaction
            self._dedent()

    def _split_lines(self, text, width):
        text = self._whitespace_matcher.sub(' ', text).strip()
        return _textwrap.wrap(text, width)

    def _fill_text(self, text, width, indent):
        text = self._whitespace_matcher.sub(' ', text).strip()
        return _textwrap.fill(text, width, initial_indent=indent,
                              subsequent_indent=indent)

    def _get_help_string(self, action):
        return action.help


class RawDescriptionHelpFormatter(HelpFormatter):
    """Help message formatter which retains any formatting in descriptions.

    Only the name of this class is considered a public API. All the methods
    provided by the class are considered an implementation detail.
    """

    def _fill_text(self, text, width, indent):
        return ''.join([indent + line for line in text.splitlines(True)])


class RawTextHelpFormatter(RawDescriptionHelpFormatter):
    """Help message formatter which retains formatting of all help text.

    Only the name of this class is considered a public API. All the methods
    provided by the class are considered an implementation detail.
    """

    def _split_lines(self, text, width):
        return text.splitlines()


class ArgumentDefaultsHelpFormatter(HelpFormatter):
    """Help message formatter which adds default values to argument help.

    Only the name of this class is considered a public API. All the methods
    provided by the class are considered an implementation detail.
    """

    def _get_help_string(self, action):
        help = action.help
        if '%(default)' not in action.help:
            if action.default is not SUPPRESS:
                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
                if action.option_strings or action.nargs in defaulting_nargs:
                    help += ' (default: %(default)s)'
        return help


# =====================
# Options and Arguments
# =====================

def _get_action_name(argument):
    if argument is None:
        return None
    elif argument.option_strings:
        return  '/'.join(argument.option_strings)
    elif argument.metavar not in (None, SUPPRESS):
        return argument.metavar
    elif argument.dest not in (None, SUPPRESS):
        return argument.dest
    else:
        return None


class ArgumentError(Exception):
    """An error from creating or using an argument (optional or positional).

    The string value of this exception is the message, augmented with
    information about the argument that caused it.
    """

    def __init__(self, argument, message):
        self.argument_name = _get_action_name(argument)
        self.message = message

    def __str__(self):
        if self.argument_name is None:
            format = '%(message)s'
        else:
            format = 'argument %(argument_name)s: %(message)s'
        return format % dict(message=self.message,
                             argument_name=self.argument_name)


class ArgumentTypeError(Exception):
    """An error from trying to convert a command line string to a type."""
    pass


# ==============
# Action classes
# ==============

class Action(_AttributeHolder):
    """Information about how to convert command line strings to Python objects.

    Action objects are used by an ArgumentParser to represent the information
    needed to parse a single argument from one or more strings from the
    command line. The keyword arguments to the Action constructor are also
    all attributes of Action instances.

    Keyword Arguments:

        - option_strings -- A list of command-line option strings which
            should be associated with this action.

        - dest -- The name of the attribute to hold the created object(s)

        - nargs -- The number of command-line arguments that should be
            consumed. By default, one argument will be consumed and a single
            value will be produced.  Other values include:
                - N (an integer) consumes N arguments (and produces a list)
                - '?' consumes zero or one arguments
                - '*' consumes zero or more arguments (and produces a list)
                - '+' consumes one or more arguments (and produces a list)
            Note that the difference between the default and nargs=1 is that
            with the default, a single value will be produced, while with
            nargs=1, a list containing a single value will be produced.

        - const -- The value to be produced if the option is specified and the
            option uses an action that takes no values.

        - default -- The value to be produced if the option is not specified.

        - type -- The type which the command-line arguments should be converted
            to, should be one of 'string', 'int', 'float', 'complex' or a
            callable object that accepts a single string argument. If None,
            'string' is assumed.

        - choices -- A container of values that should be allowed. If not None,
            after a command-line argument has been converted to the appropriate
            type, an exception will be raised if it is not a member of this
            collection.

        - required -- True if the action must always be specified at the
            command line. This is only meaningful for optional command-line
            arguments.

        - help -- The help string describing the argument.

        - metavar -- The name to be used for the option's argument with the
            help string. If None, the 'dest' value will be used as the name.
    """

    def __init__(self,
                 option_strings,
                 dest,
                 nargs=None,
                 const=None,
                 default=None,
                 type=None,
                 choices=None,
                 required=False,
                 help=None,
                 metavar=None):
        self.option_strings = option_strings
        self.dest = dest
        self.nargs = nargs
        self.const = const
        self.default = default
        self.type = type
        self.choices = choices
        self.required = required
        self.help = help
        self.metavar = metavar

    def _get_kwargs(self):
        names = [
            'option_strings',
            'dest',
            'nargs',
            'const',
            'default',
            'type',
            'choices',
            'help',
            'metavar',
        ]
        return [(name, getattr(self, name)) for name in names]

    def __call__(self, parser, namespace, values, option_string=None):
        raise NotImplementedError(_('.__call__() not defined'))


class _StoreAction(Action):

    def __init__(self,
                 option_strings,
                 dest,
                 nargs=None,
                 const=None,
                 default=None,
                 type=None,
                 choices=None,
                 required=False,
                 help=None,
                 metavar=None):
        if nargs == 0:
            raise ValueError('nargs for store actions must be > 0; if you '
                             'have nothing to store, actions such as store '
                             'true or store const may be more appropriate')
        if const is not None and nargs != OPTIONAL:
            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
        super(_StoreAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=nargs,
            const=const,
            default=default,
            type=type,
            choices=choices,
            required=required,
            help=help,
            metavar=metavar)

    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)


class _StoreConstAction(Action):

    def __init__(self,
                 option_strings,
                 dest,
                 const,
                 default=None,
                 required=False,
                 help=None,
                 metavar=None):
        super(_StoreConstAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=0,
            const=const,
            default=default,
            required=required,
            help=help)

    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, self.const)


class _StoreTrueAction(_StoreConstAction):

    def __init__(self,
                 option_strings,
                 dest,
                 default=False,
                 required=False,
                 help=None):
        super(_StoreTrueAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            const=True,
            default=default,
            required=required,
            help=help)


class _StoreFalseAction(_StoreConstAction):

    def __init__(self,
                 option_strings,
                 dest,
                 default=True,
                 required=False,
                 help=None):
        super(_StoreFalseAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            const=False,
            default=default,
            required=required,
            help=help)


class _AppendAction(Action):

    def __init__(self,
                 option_strings,
                 dest,
                 nargs=None,
                 const=None,
                 default=None,
                 type=None,
                 choices=None,
                 required=False,
                 help=None,
                 metavar=None):
        if nargs == 0:
            raise ValueError('nargs for append actions must be > 0; if arg '
                             'strings are not supplying the value to append, '
                             'the append const action may be more appropriate')
        if const is not None and nargs != OPTIONAL:
            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
        super(_AppendAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=nargs,
            const=const,
            default=default,
            type=type,
            choices=choices,
            required=required,
            help=help,
            metavar=metavar)

    def __call__(self, parser, namespace, values, option_string=None):
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
        items.append(values)
        setattr(namespace, self.dest, items)


class _AppendConstAction(Action):

    def __init__(self,
                 option_strings,
                 dest,
                 const,
                 default=None,
                 required=False,
                 help=None,
                 metavar=None):
        super(_AppendConstAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=0,
            const=const,
            default=default,
            required=required,
            help=help,
            metavar=metavar)

    def __call__(self, parser, namespace, values, option_string=None):
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
        items.append(self.const)
        setattr(namespace, self.dest, items)


class _CountAction(Action):

    def __init__(self,
                 option_strings,
                 dest,
                 default=None,
                 required=False,
                 help=None):
        super(_CountAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=0,
            default=default,
            required=required,
            help=help)

    def __call__(self, parser, namespace, values, option_string=None):
        new_count = _ensure_value(namespace, self.dest, 0) + 1
        setattr(namespace, self.dest, new_count)


class _HelpAction(Action):

    def __init__(self,
                 option_strings,
                 dest=SUPPRESS,
                 default=SUPPRESS,
                 help=None):
        super(_HelpAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            default=default,
            nargs=0,
            help=help)

    def __call__(self, parser, namespace, values, option_string=None):
        parser.print_help()
        parser.exit()


class _VersionAction(Action):

    def __init__(self,
                 option_strings,
                 version=None,
                 dest=SUPPRESS,
                 default=SUPPRESS,
                 help="show program's version number and exit"):
        super(_VersionAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            default=default,
            nargs=0,
            help=help)
        self.version = version

    def __call__(self, parser, namespace, values, option_string=None):
        version = self.version
        if version is None:
            version = parser.version
        formatter = parser._get_formatter()
        formatter.add_text(version)
        parser.exit(message=formatter.format_help())


class _SubParsersAction(Action):

    class _ChoicesPseudoAction(Action):

        def __init__(self, name, help):
            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
            sup.__init__(option_strings=[], dest=name, help=help)

    def __init__(self,
                 option_strings,
                 prog,
                 parser_class,
                 dest=SUPPRESS,
                 help=None,
                 metavar=None):

        self._prog_prefix = prog
        self._parser_class = parser_class
        self._name_parser_map = {}
        self._choices_actions = []

        super(_SubParsersAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=PARSER,
            choices=self._name_parser_map,
            help=help,
            metavar=metavar)

    def add_parser(self, name, **kwargs):
        # set prog from the existing prefix
        if kwargs.get('prog') is None:
            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)

        # create a pseudo-action to hold the choice help
        if 'help' in kwargs:
            help = kwargs.pop('help')
            choice_action = self._ChoicesPseudoAction(name, help)
            self._choices_actions.append(choice_action)

        # create the parser and add it to the map
        parser = self._parser_class(**kwargs)
        self._name_parser_map[name] = parser
        return parser

    def _get_subactions(self):
        return self._choices_actions

    def __call__(self, parser, namespace, values, option_string=None):
        parser_name = values[0]
        arg_strings = values[1:]

        # set the parser name if requested
        if self.dest is not SUPPRESS:
            setattr(namespace, self.dest, parser_name)

        # select the parser
        try:
            parser = self._name_parser_map[parser_name]
        except KeyError:
            tup = parser_name, ', '.join(self._name_parser_map)
            msg = _('unknown parser %r (choices: %s)' % tup)
            raise ArgumentError(self, msg)

        # parse all the remaining options into the namespace
        # store any unrecognized options on the object, so that the top
        # level parser can decide what to do with them
        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
        if arg_strings:
            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)


# ==============
# Type classes
# ==============

class FileType(object):
    """Factory for creating file object types

    Instances of FileType are typically passed as type= arguments to the
    ArgumentParser add_argument() method.

    Keyword Arguments:
        - mode -- A string indicating how the file is to be opened. Accepts the
            same values as the builtin open() function.
        - bufsize -- The file's desired buffer size. Accepts the same values as
            the builtin open() function.
    """

    def __init__(self, mode='r', bufsize=None):
        self._mode = mode
        self._bufsize = bufsize

    def __call__(self, string):
        # the special argument "-" means sys.std{in,out}
        if string == '-':
            if 'r' in self._mode:
                return _sys.stdin
            elif 'w' in self._mode:
                return _sys.stdout
            else:
                msg = _('argument "-" with mode %r' % self._mode)
                raise ValueError(msg)

        # all other arguments are used as file names
        if self._bufsize:
            return open(string, self._mode, self._bufsize)
        else:
            return open(string, self._mode)

    def __repr__(self):
        args = [self._mode, self._bufsize]
        args_str = ', '.join([repr(arg) for arg in args if arg is not None])
        return '%s(%s)' % (type(self).__name__, args_str)

# ===========================
# Optional and Positional Parsing
# ===========================

class Namespace(_AttributeHolder):
    """Simple object for storing attributes.

    Implements equality by attribute names and values, and provides a simple
    string representation.
    """

    def __init__(self, **kwargs):
        for name in kwargs:
            setattr(self, name, kwargs[name])

    __hash__ = None

    def __eq__(self, other):
        return vars(self) == vars(other)

    def __ne__(self, other):
        return not (self == other)

    def __contains__(self, key):
        return key in self.__dict__


class _ActionsContainer(object):

    def __init__(self,
                 description,
                 prefix_chars,
                 argument_default,
                 conflict_handler):
        super(_ActionsContainer, self).__init__()

        self.description = description
        self.argument_default = argument_default
        self.prefix_chars = prefix_chars
        self.conflict_handler = conflict_handler

        # set up registries
        self._registries = {}

        # register actions
        self.register('action', None, _StoreAction)
        self.register('action', 'store', _StoreAction)
        self.register('action', 'store_const', _StoreConstAction)
        self.register('action', 'store_true', _StoreTrueAction)
        self.register('action', 'store_false', _StoreFalseAction)
        self.register('action', 'append', _AppendAction)
        self.register('action', 'append_const', _AppendConstAction)
        self.register('action', 'count', _CountAction)
        self.register('action', 'help', _HelpAction)
        self.register('action', 'version', _VersionAction)
        self.register('action', 'parsers', _SubParsersAction)

        # raise an exception if the conflict handler is invalid
        self._get_handler()

        # action storage
        self._actions = []
        self._option_string_actions = {}

        # groups
        self._action_groups = []
        self._mutually_exclusive_groups = []

        # defaults storage
        self._defaults = {}

        # determines whether an "option" looks like a negative number
        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')

        # whether or not there are any optionals that look like negative
        # numbers -- uses a list so it can be shared and edited
        self._has_negative_number_optionals = []

    # ====================
    # Registration methods
    # ====================
    def register(self, registry_name, value, object):
        registry = self._registries.setdefault(registry_name, {})
        registry[value] = object

    def _registry_get(self, registry_name, value, default=None):
        return self._registries[registry_name].get(value, default)

    # ==================================
    # Namespace default accessor methods
    # ==================================
    def set_defaults(self, **kwargs):
        self._defaults.update(kwargs)

        # if these defaults match any existing arguments, replace
        # the previous default on the object with the new one
        for action in self._actions:
            if action.dest in kwargs:
                action.default = kwargs[action.dest]

    def get_default(self, dest):
        for action in self._actions:
            if action.dest == dest and action.default is not None:
                return action.default
        return self._defaults.get(dest, None)


    # =======================
    # Adding argument actions
    # =======================
    def add_argument(self, *args, **kwargs):
        """
        add_argument(dest, ..., name=value, ...)
        add_argument(option_string, option_string, ..., name=value, ...)
        """

        # if no positional args are supplied or only one is supplied and
        # it doesn't look like an option string, parse a positional
        # argument
        chars = self.prefix_chars
        if not args or len(args) == 1 and args[0][0] not in chars:
            if args and 'dest' in kwargs:
                raise ValueError('dest supplied twice for positional argument')
            kwargs = self._get_positional_kwargs(*args, **kwargs)

        # otherwise, we're adding an optional argument
        else:
            kwargs = self._get_optional_kwargs(*args, **kwargs)

        # if no default was supplied, use the parser-level default
        if 'default' not in kwargs:
            dest = kwargs['dest']
            if dest in self._defaults:
                kwargs['default'] = self._defaults[dest]
            elif self.argument_default is not None:
                kwargs['default'] = self.argument_default

        # create the action object, and add it to the parser
        action_class = self._pop_action_class(kwargs)
        if not _callable(action_class):
            raise ValueError('unknown action "%s"' % action_class)
        action = action_class(**kwargs)

        # raise an error if the action type is not callable
        type_func = self._registry_get('type', action.type, action.type)
        if not _callable(type_func):
            raise ValueError('%r is not callable' % type_func)

        return self._add_action(action)

    def add_argument_group(self, *args, **kwargs):
        group = _ArgumentGroup(self, *args, **kwargs)
        self._action_groups.append(group)
        return group

    def add_mutually_exclusive_group(self, **kwargs):
        group = _MutuallyExclusiveGroup(self, **kwargs)
        self._mutually_exclusive_groups.append(group)
        return group

    def _add_action(self, action):
        # resolve any conflicts
        self._check_conflict(action)

        # add to actions list
        self._actions.append(action)
        action.container = self

        # index the action by any option strings it has
        for option_string in action.option_strings:
            self._option_string_actions[option_string] = action

        # set the flag if any option strings look like negative numbers
        for option_string in action.option_strings:
            if self._negative_number_matcher.match(option_string):
                if not self._has_negative_number_optionals:
                    self._has_negative_number_optionals.append(True)

        # return the created action
        return action

    def _remove_action(self, action):
        self._actions.remove(action)

    def _add_container_actions(self, container):
        # collect groups by titles
        title_group_map = {}
        for group in self._action_groups:
            if group.title in title_group_map:
                msg = _('cannot merge actions - two groups are named %r')
                raise ValueError(msg % (group.title))
            title_group_map[group.title] = group

        # map each action to its group
        group_map = {}
        for group in container._action_groups:

            # if a group with the title exists, use that, otherwise
            # create a new group matching the container's group
            if group.title not in title_group_map:
                title_group_map[group.title] = self.add_argument_group(
                    title=group.title,
                    description=group.description,
                    conflict_handler=group.conflict_handler)

            # map the actions to their new group
            for action in group._group_actions:
                group_map[action] = title_group_map[group.title]

        # add container's mutually exclusive groups
        # NOTE: if add_mutually_exclusive_group ever gains title= and
        # description= then this code will need to be expanded as above
        for group in container._mutually_exclusive_groups:
            mutex_group = self.add_mutually_exclusive_group(
                required=group.required)

            # map the actions to their new mutex group
            for action in group._group_actions:
                group_map[action] = mutex_group

        # add all actions to this container or their group
        for action in container._actions:
            group_map.get(action, self)._add_action(action)

    def _get_positional_kwargs(self, dest, **kwargs):
        # make sure required is not specified
        if 'required' in kwargs:
            msg = _("'required' is an invalid argument for positionals")
            raise TypeError(msg)

        # mark positional arguments as required if at least one is
        # always required
        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
            kwargs['required'] = True
        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
            kwargs['required'] = True

        # return the keyword arguments with no option strings
        return dict(kwargs, dest=dest, option_strings=[])

    def _get_optional_kwargs(self, *args, **kwargs):
        # determine short and long option strings
        option_strings = []
        long_option_strings = []
        for option_string in args:
            # error on strings that don't start with an appropriate prefix
            if not option_string[0] in self.prefix_chars:
                msg = _('invalid option string %r: '
                        'must start with a character %r')
                tup = option_string, self.prefix_chars
                raise ValueError(msg % tup)

            # strings starting with two prefix characters are long options
            option_strings.append(option_string)
            if option_string[0] in self.prefix_chars:
                if len(option_string) > 1:
                    if option_string[1] in self.prefix_chars:
                        long_option_strings.append(option_string)

        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
        dest = kwargs.pop('dest', None)
        if dest is None:
            if long_option_strings:
                dest_option_string = long_option_strings[0]
            else:
                dest_option_string = option_strings[0]
            dest = dest_option_string.lstrip(self.prefix_chars)
            if not dest:
                msg = _('dest= is required for options like %r')
                raise ValueError(msg % option_string)
            dest = dest.replace('-', '_')

        # return the updated keyword arguments
        return dict(kwargs, dest=dest, option_strings=option_strings)

    def _pop_action_class(self, kwargs, default=None):
        action = kwargs.pop('action', default)
        return self._registry_get('action', action, action)

    def _get_handler(self):
        # determine function from conflict handler string
        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
        try:
            return getattr(self, handler_func_name)
        except AttributeError:
            msg = _('invalid conflict_resolution value: %r')
            raise ValueError(msg % self.conflict_handler)

    def _check_conflict(self, action):

        # find all options that conflict with this option
        confl_optionals = []
        for option_string in action.option_strings:
            if option_string in self._option_string_actions:
                confl_optional = self._option_string_actions[option_string]
                confl_optionals.append((option_string, confl_optional))

        # resolve any conflicts
        if confl_optionals:
            conflict_handler = self._get_handler()
            conflict_handler(action, confl_optionals)

    def _handle_conflict_error(self, action, conflicting_actions):
        message = _('conflicting option string(s): %s')
        conflict_string = ', '.join([option_string
                                     for option_string, action
                                     in conflicting_actions])
        raise ArgumentError(action, message % conflict_string)

    def _handle_conflict_resolve(self, action, conflicting_actions):

        # remove all conflicting options
        for option_string, action in conflicting_actions:

            # remove the conflicting option
            action.option_strings.remove(option_string)
            self._option_string_actions.pop(option_string, None)

            # if the option now has no option string, remove it from the
            # container holding it
            if not action.option_strings:
                action.container._remove_action(action)


class _ArgumentGroup(_ActionsContainer):

    def __init__(self, container, title=None, description=None, **kwargs):
        # add any missing keyword arguments by checking the container
        update = kwargs.setdefault
        update('conflict_handler', container.conflict_handler)
        update('prefix_chars', container.prefix_chars)
        update('argument_default', container.argument_default)
        super_init = super(_ArgumentGroup, self).__init__
        super_init(description=description, **kwargs)

        # group attributes
        self.title = title
        self._group_actions = []

        # share most attributes with the container
        self._registries = container._registries
        self._actions = container._actions
        self._option_string_actions = container._option_string_actions
        self._defaults = container._defaults
        self._has_negative_number_optionals = \
            container._has_negative_number_optionals

    def _add_action(self, action):
        action = super(_ArgumentGroup, self)._add_action(action)
        self._group_actions.append(action)
        return action

    def _remove_action(self, action):
        super(_ArgumentGroup, self)._remove_action(action)
        self._group_actions.remove(action)


class _MutuallyExclusiveGroup(_ArgumentGroup):

    def __init__(self, container, required=False):
        super(_MutuallyExclusiveGroup, self).__init__(container)
        self.required = required
        self._container = container

    def _add_action(self, action):
        if action.required:
            msg = _('mutually exclusive arguments must be optional')
            raise ValueError(msg)
        action = self._container._add_action(action)
        self._group_actions.append(action)
        return action

    def _remove_action(self, action):
        self._container._remove_action(action)
        self._group_actions.remove(action)


class ArgumentParser(_AttributeHolder, _ActionsContainer):
    """Object for parsing command line strings into Python objects.

    Keyword Arguments:
        - prog -- The name of the program (default: sys.argv[0])
        - usage -- A usage message (default: auto-generated from arguments)
        - description -- A description of what the program does
        - epilog -- Text following the argument descriptions
        - parents -- Parsers whose arguments should be copied into this one
        - formatter_class -- HelpFormatter class for printing help messages
        - prefix_chars -- Characters that prefix optional arguments
        - fromfile_prefix_chars -- Characters that prefix files containing
            additional arguments
        - argument_default -- The default value for all arguments
        - conflict_handler -- String indicating how to handle conflicts
        - add_help -- Add a -h/-help option
    """

    def __init__(self,
                 prog=None,
                 usage=None,
                 description=None,
                 epilog=None,
                 version=None,
                 parents=[],
                 formatter_class=HelpFormatter,
                 prefix_chars='-',
                 fromfile_prefix_chars=None,
                 argument_default=None,
                 conflict_handler='error',
                 add_help=True):

        if version is not None:
            import warnings
            warnings.warn(
                """The "version" argument to ArgumentParser is deprecated. """
                """Please use """
                """"add_argument(..., action='version', version="N", ...)" """
                """instead""", DeprecationWarning)

        superinit = super(ArgumentParser, self).__init__
        superinit(description=description,
                  prefix_chars=prefix_chars,
                  argument_default=argument_default,
                  conflict_handler=conflict_handler)

        # default setting for prog
        if prog is None:
            prog = _os.path.basename(_sys.argv[0])

        self.prog = prog
        self.usage = usage
        self.epilog = epilog
        self.version = version
        self.formatter_class = formatter_class
        self.fromfile_prefix_chars = fromfile_prefix_chars
        self.add_help = add_help

        add_group = self.add_argument_group
        self._positionals = add_group(_('positional arguments'))
        self._optionals = add_group(_('optional arguments'))
        self._subparsers = None

        # register types
        def identity(string):
            return string
        self.register('type', None, identity)

        # add help and version arguments if necessary
        # (using explicit default to override global argument_default)
        if '-' in prefix_chars:
            default_prefix = '-'
        else:
            default_prefix = prefix_chars[0]
        if self.add_help:
            self.add_argument(
                default_prefix+'h', default_prefix*2+'help',
                action='help', default=SUPPRESS,
                help=_('show this help message and exit'))
        if self.version:
            self.add_argument(
                default_prefix+'v', default_prefix*2+'version',
                action='version', default=SUPPRESS,
                version=self.version,
                help=_("show program's version number and exit"))

        # add parent arguments and defaults
        for parent in parents:
            self._add_container_actions(parent)
            try:
                defaults = parent._defaults
            except AttributeError:
                pass
            else:
                self._defaults.update(defaults)

    # =======================
    # Pretty __repr__ methods
    # =======================
    def _get_kwargs(self):
        names = [
            'prog',
            'usage',
            'description',
            'version',
            'formatter_class',
            'conflict_handler',
            'add_help',
        ]
        return [(name, getattr(self, name)) for name in names]

    # ==================================
    # Optional/Positional adding methods
    # ==================================
    def add_subparsers(self, **kwargs):
        if self._subparsers is not None:
            self.error(_('cannot have multiple subparser arguments'))

        # add the parser class to the arguments if it's not present
        kwargs.setdefault('parser_class', type(self))

        if 'title' in kwargs or 'description' in kwargs:
            title = _(kwargs.pop('title', 'subcommands'))
            description = _(kwargs.pop('description', None))
            self._subparsers = self.add_argument_group(title, description)
        else:
            self._subparsers = self._positionals

        # prog defaults to the usage message of this parser, skipping
        # optional arguments and with no "usage:" prefix
        if kwargs.get('prog') is None:
            formatter = self._get_formatter()
            positionals = self._get_positional_actions()
            groups = self._mutually_exclusive_groups
            formatter.add_usage(self.usage, positionals, groups, '')
            kwargs['prog'] = formatter.format_help().strip()

        # create the parsers action and add it to the positionals list
        parsers_class = self._pop_action_class(kwargs, 'parsers')
        action = parsers_class(option_strings=[], **kwargs)
        self._subparsers._add_action(action)

        # return the created parsers action
        return action

    def _add_action(self, action):
        if action.option_strings:
            self._optionals._add_action(action)
        else:
            self._positionals._add_action(action)
        return action

    def _get_optional_actions(self):
        return [action
                for action in self._actions
                if action.option_strings]

    def _get_positional_actions(self):
        return [action
                for action in self._actions
                if not action.option_strings]

    # =====================================
    # Command line argument parsing methods
    # =====================================
    def parse_args(self, args=None, namespace=None):
        args, argv = self.parse_known_args(args, namespace)
        if argv:
            msg = _('unrecognized arguments: %s')
            self.error(msg % ' '.join(argv))
        return args

    def parse_known_args(self, args=None, namespace=None):
        # args default to the system args
        if args is None:
            args = _sys.argv[1:]

        # default Namespace built from parser defaults
        if namespace is None:
            namespace = Namespace()

        # add any action defaults that aren't present
        for action in self._actions:
            if action.dest is not SUPPRESS:
                if not hasattr(namespace, action.dest):
                    if action.default is not SUPPRESS:
                        default = action.default
                        if isinstance(action.default, basestring):
                            default = self._get_value(action, default)
                        setattr(namespace, action.dest, default)

        # add any parser defaults that aren't present
        for dest in self._defaults:
            if not hasattr(namespace, dest):
                setattr(namespace, dest, self._defaults[dest])

        # parse the arguments and exit if there are any errors
        try:
            namespace, args = self._parse_known_args(args, namespace)
            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
            return namespace, args
        except ArgumentError:
            err = _sys.exc_info()[1]
            self.error(str(err))

    def _parse_known_args(self, arg_strings, namespace):
        # replace arg strings that are file references
        if self.fromfile_prefix_chars is not None:
            arg_strings = self._read_args_from_files(arg_strings)

        # map all mutually exclusive arguments to the other arguments
        # they can't occur with
        action_conflicts = {}
        for mutex_group in self._mutually_exclusive_groups:
            group_actions = mutex_group._group_actions
            for i, mutex_action in enumerate(mutex_group._group_actions):
                conflicts = action_conflicts.setdefault(mutex_action, [])
                conflicts.extend(group_actions[:i])
                conflicts.extend(group_actions[i + 1:])

        # find all option indices, and determine the arg_string_pattern
        # which has an 'O' if there is an option at an index,
        # an 'A' if there is an argument, or a '-' if there is a '--'
        option_string_indices = {}
        arg_string_pattern_parts = []
        arg_strings_iter = iter(arg_strings)
        for i, arg_string in enumerate(arg_strings_iter):

            # all args after -- are non-options
            if arg_string == '--':
                arg_string_pattern_parts.append('-')
                for arg_string in arg_strings_iter:
                    arg_string_pattern_parts.append('A')

            # otherwise, add the arg to the arg strings
            # and note the index if it was an option
            else:
                option_tuple = self._parse_optional(arg_string)
                if option_tuple is None:
                    pattern = 'A'
                else:
                    option_string_indices[i] = option_tuple
                    pattern = 'O'
                arg_string_pattern_parts.append(pattern)

        # join the pieces together to form the pattern
        arg_strings_pattern = ''.join(arg_string_pattern_parts)

        # converts arg strings to the appropriate and then takes the action
        seen_actions = set()
        seen_non_default_actions = set()

        def take_action(action, argument_strings, option_string=None):
            seen_actions.add(action)
            argument_values = self._get_values(action, argument_strings)

            # error if this argument is not allowed with other previously
            # seen arguments, assuming that actions that use the default
            # value don't really count as "present"
            if argument_values is not action.default:
                seen_non_default_actions.add(action)
                for conflict_action in action_conflicts.get(action, []):
                    if conflict_action in seen_non_default_actions:
                        msg = _('not allowed with argument %s')
                        action_name = _get_action_name(conflict_action)
                        raise ArgumentError(action, msg % action_name)

            # take the action if we didn't receive a SUPPRESS value
            # (e.g. from a default)
            if argument_values is not SUPPRESS:
                action(self, namespace, argument_values, option_string)

        # function to convert arg_strings into an optional action
        def consume_optional(start_index):

            # get the optional identified at this index
            option_tuple = option_string_indices[start_index]
            action, option_string, explicit_arg = option_tuple

            # identify additional optionals in the same arg string
            # (e.g. -xyz is the same as -x -y -z if no args are required)
            match_argument = self._match_argument
            action_tuples = []
            while True:

                # if we found no optional action, skip it
                if action is None:
                    extras.append(arg_strings[start_index])
                    return start_index + 1

                # if there is an explicit argument, try to match the
                # optional's string arguments to only this
                if explicit_arg is not None:
                    arg_count = match_argument(action, 'A')

                    # if the action is a single-dash option and takes no
                    # arguments, try to parse more single-dash options out
                    # of the tail of the option string
                    chars = self.prefix_chars
                    if arg_count == 0 and option_string[1] not in chars:
                        action_tuples.append((action, [], option_string))
                        char = option_string[0]
                        option_string = char + explicit_arg[0]
                        new_explicit_arg = explicit_arg[1:] or None
                        optionals_map = self._option_string_actions
                        if option_string in optionals_map:
                            action = optionals_map[option_string]
                            explicit_arg = new_explicit_arg
                        else:
                            msg = _('ignored explicit argument %r')
                            raise ArgumentError(action, msg % explicit_arg)

                    # if the action expect exactly one argument, we've
                    # successfully matched the option; exit the loop
                    elif arg_count == 1:
                        stop = start_index + 1
                        args = [explicit_arg]
                        action_tuples.append((action, args, option_string))
                        break

                    # error if a double-dash option did not use the
                    # explicit argument
                    else:
                        msg = _('ignored explicit argument %r')
                        raise ArgumentError(action, msg % explicit_arg)

                # if there is no explicit argument, try to match the
                # optional's string arguments with the following strings
                # if successful, exit the loop
                else:
                    start = start_index + 1
                    selected_patterns = arg_strings_pattern[start:]
                    arg_count = match_argument(action, selected_patterns)
                    stop = start + arg_count
                    args = arg_strings[start:stop]
                    action_tuples.append((action, args, option_string))
                    break

            # add the Optional to the list and return the index at which
            # the Optional's string args stopped
            assert action_tuples
            for action, args, option_string in action_tuples:
                take_action(action, args, option_string)
            return stop

        # the list of Positionals left to be parsed; this is modified
        # by consume_positionals()
        positionals = self._get_positional_actions()

        # function to convert arg_strings into positional actions
        def consume_positionals(start_index):
            # match as many Positionals as possible
            match_partial = self._match_arguments_partial
            selected_pattern = arg_strings_pattern[start_index:]
            arg_counts = match_partial(positionals, selected_pattern)

            # slice off the appropriate arg strings for each Positional
            # and add the Positional and its args to the list
            for action, arg_count in zip(positionals, arg_counts):
                args = arg_strings[start_index: start_index + arg_count]
                start_index += arg_count
                take_action(action, args)

            # slice off the Positionals that we just parsed and return the
            # index at which the Positionals' string args stopped
            positionals[:] = positionals[len(arg_counts):]
            return start_index

        # consume Positionals and Optionals alternately, until we have
        # passed the last option string
        extras = []
        start_index = 0
        if option_string_indices:
            max_option_string_index = max(option_string_indices)
        else:
            max_option_string_index = -1
        while start_index <= max_option_string_index:

            # consume any Positionals preceding the next option
            next_option_string_index = min([
                index
                for index in option_string_indices
                if index >= start_index])
            if start_index != next_option_string_index:
                positionals_end_index = consume_positionals(start_index)

                # only try to parse the next optional if we didn't consume
                # the option string during the positionals parsing
                if positionals_end_index > start_index:
                    start_index = positionals_end_index
                    continue
                else:
                    start_index = positionals_end_index

            # if we consumed all the positionals we could and we're not
            # at the index of an option string, there were extra arguments
            if start_index not in option_string_indices:
                strings = arg_strings[start_index:next_option_string_index]
                extras.extend(strings)
                start_index = next_option_string_index

            # consume the next optional and any arguments for it
            start_index = consume_optional(start_index)

        # consume any positionals following the last Optional
        stop_index = consume_positionals(start_index)

        # if we didn't consume all the argument strings, there were extras
        extras.extend(arg_strings[stop_index:])

        # if we didn't use all the Positional objects, there were too few
        # arg strings supplied.
        if positionals:
            self.error(_('too few arguments'))

        # make sure all required actions were present
        for action in self._actions:
            if action.required:
                if action not in seen_actions:
                    name = _get_action_name(action)
                    self.error(_('argument %s is required') % name)

        # make sure all required groups had one option present
        for group in self._mutually_exclusive_groups:
            if group.required:
                for action in group._group_actions:
                    if action in seen_non_default_actions:
                        break

                # if no actions were used, report the error
                else:
                    names = [_get_action_name(action)
                             for action in group._group_actions
                             if action.help is not SUPPRESS]
                    msg = _('one of the arguments %s is required')
                    self.error(msg % ' '.join(names))

        # return the updated namespace and the extra arguments
        return namespace, extras

    def _read_args_from_files(self, arg_strings):
        # expand arguments referencing files
        new_arg_strings = []
        for arg_string in arg_strings:

            # for regular arguments, just add them back into the list
            if arg_string[0] not in self.fromfile_prefix_chars:
                new_arg_strings.append(arg_string)

                # # replace arguments referencing files with the file content
                # else:
                #     try:
                #         args_file = open(arg_string[1:])
                #         try:
                #             arg_strings = []
                #             for arg_line in args_file.read().splitlines():
                #                 for arg in self.convert_arg_line_to_args(arg_line):
                #                     arg_strings.append(arg)
                #             arg_strings = self._read_args_from_files(arg_strings)
                #             new_arg_strings.extend(arg_strings)
                #         finally:
                #             args_file.close()
                #     except IOError:
                #         err = _sys.exc_info()[1]
                #         self.error(str(err))

        # return the modified argument list
        return new_arg_strings

    def convert_arg_line_to_args(self, arg_line):
        return [arg_line]

    def _match_argument(self, action, arg_strings_pattern):
        # match the pattern for this action to the arg strings
        nargs_pattern = self._get_nargs_pattern(action)
        match = _re.match(nargs_pattern, arg_strings_pattern)

        # raise an exception if we weren't able to find a match
        if match is None:
            nargs_errors = {
                None: _('expected one argument'),
                OPTIONAL: _('expected at most one argument'),
                ONE_OR_MORE: _('expected at least one argument'),
            }
            default = _('expected %s argument(s)') % action.nargs
            msg = nargs_errors.get(action.nargs, default)
            raise ArgumentError(action, msg)

        # return the number of arguments matched
        return len(match.group(1))

    def _match_arguments_partial(self, actions, arg_strings_pattern):
        # progressively shorten the actions list by slicing off the
        # final actions until we find a match
        result = []
        for i in range(len(actions), 0, -1):
            actions_slice = actions[:i]
            pattern = ''.join([self._get_nargs_pattern(action)
                               for action in actions_slice])
            match = _re.match(pattern, arg_strings_pattern)
            if match is not None:
                result.extend([len(string) for string in match.groups()])
                break

        # return the list of arg string counts
        return result

    def _parse_optional(self, arg_string):
        # if it's an empty string, it was meant to be a positional
        if not arg_string:
            return None

        # if it doesn't start with a prefix, it was meant to be positional
        if not arg_string[0] in self.prefix_chars:
            return None

        # if the option string is present in the parser, return the action
        if arg_string in self._option_string_actions:
            action = self._option_string_actions[arg_string]
            return action, arg_string, None

        # if it's just a single character, it was meant to be positional
        if len(arg_string) == 1:
            return None

        # if the option string before the "=" is present, return the action
        if '=' in arg_string:
            option_string, explicit_arg = arg_string.split('=', 1)
            if option_string in self._option_string_actions:
                action = self._option_string_actions[option_string]
                return action, option_string, explicit_arg

        # search through all possible prefixes of the option string
        # and all actions in the parser for possible interpretations
        option_tuples = self._get_option_tuples(arg_string)

        # if multiple actions match, the option string was ambiguous
        if len(option_tuples) > 1:
            options = ', '.join([option_string
                                 for action, option_string, explicit_arg in option_tuples])
            tup = arg_string, options
            self.error(_('ambiguous option: %s could match %s') % tup)

        # if exactly one action matched, this segmentation is good,
        # so return the parsed action
        elif len(option_tuples) == 1:
            option_tuple, = option_tuples
            return option_tuple

        # if it was not found as an option, but it looks like a negative
        # number, it was meant to be positional
        # unless there are negative-number-like options
        if self._negative_number_matcher.match(arg_string):
            if not self._has_negative_number_optionals:
                return None

        # if it contains a space, it was meant to be a positional
        if ' ' in arg_string:
            return None

        # it was meant to be an optional but there is no such option
        # in this parser (though it might be a valid option in a subparser)
        return None, arg_string, None

    def _get_option_tuples(self, option_string):
        result = []

        # option strings starting with two prefix characters are only
        # split at the '='
        chars = self.prefix_chars
        if option_string[0] in chars and option_string[1] in chars:
            if '=' in option_string:
                option_prefix, explicit_arg = option_string.split('=', 1)
            else:
                option_prefix = option_string
                explicit_arg = None
            for option_string in self._option_string_actions:
                if option_string.startswith(option_prefix):
                    action = self._option_string_actions[option_string]
                    tup = action, option_string, explicit_arg
                    result.append(tup)

        # single character options can be concatenated with their arguments
        # but multiple character options always have to have their argument
        # separate
        elif option_string[0] in chars and option_string[1] not in chars:
            option_prefix = option_string
            explicit_arg = None
            short_option_prefix = option_string[:2]
            short_explicit_arg = option_string[2:]

            for option_string in self._option_string_actions:
                if option_string == short_option_prefix:
                    action = self._option_string_actions[option_string]
                    tup = action, option_string, short_explicit_arg
                    result.append(tup)
                elif option_string.startswith(option_prefix):
                    action = self._option_string_actions[option_string]
                    tup = action, option_string, explicit_arg
                    result.append(tup)

        # shouldn't ever get here
        else:
            self.error(_('unexpected option string: %s') % option_string)

        # return the collected option tuples
        return result

    def _get_nargs_pattern(self, action):
        # in all examples below, we have to allow for '--' args
        # which are represented as '-' in the pattern
        nargs = action.nargs

        # the default (None) is assumed to be a single argument
        if nargs is None:
            nargs_pattern = '(-*A-*)'

        # allow zero or one arguments
        elif nargs == OPTIONAL:
            nargs_pattern = '(-*A?-*)'

        # allow zero or more arguments
        elif nargs == ZERO_OR_MORE:
            nargs_pattern = '(-*[A-]*)'

        # allow one or more arguments
        elif nargs == ONE_OR_MORE:
            nargs_pattern = '(-*A[A-]*)'

        # allow any number of options or arguments
        elif nargs == REMAINDER:
            nargs_pattern = '([-AO]*)'

        # allow one argument followed by any number of options or arguments
        elif nargs == PARSER:
            nargs_pattern = '(-*A[-AO]*)'

        # all others should be integers
        else:
            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)

        # if this is an optional action, -- is not allowed
        if action.option_strings:
            nargs_pattern = nargs_pattern.replace('-*', '')
            nargs_pattern = nargs_pattern.replace('-', '')

        # return the pattern
        return nargs_pattern

    # ========================
    # Value conversion methods
    # ========================
    def _get_values(self, action, arg_strings):
        # for everything but PARSER args, strip out '--'
        if action.nargs not in [PARSER, REMAINDER]:
            arg_strings = [s for s in arg_strings if s != '--']

        # optional argument produces a default when not present
        if not arg_strings and action.nargs == OPTIONAL:
            if action.option_strings:
                value = action.const
            else:
                value = action.default
            if isinstance(value, basestring):
                value = self._get_value(action, value)
                self._check_value(action, value)

        # when nargs='*' on a positional, if there were no command-line
        # args, use the default if it is anything other than None
        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
                  not action.option_strings):
            if action.default is not None:
                value = action.default
            else:
                value = arg_strings
            self._check_value(action, value)

        # single argument or optional argument produces a single value
        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
            arg_string, = arg_strings
            value = self._get_value(action, arg_string)
            self._check_value(action, value)

        # REMAINDER arguments convert all values, checking none
        elif action.nargs == REMAINDER:
            value = [self._get_value(action, v) for v in arg_strings]

        # PARSER arguments convert all values, but check only the first
        elif action.nargs == PARSER:
            value = [self._get_value(action, v) for v in arg_strings]
            self._check_value(action, value[0])

        # all other types of nargs produce a list
        else:
            value = [self._get_value(action, v) for v in arg_strings]
            for v in value:
                self._check_value(action, v)

        # return the converted value
        return value

    def _get_value(self, action, arg_string):
        type_func = self._registry_get('type', action.type, action.type)
        if not _callable(type_func):
            msg = _('%r is not callable')
            raise ArgumentError(action, msg % type_func)

        # convert the value to the appropriate type
        try:
            result = type_func(arg_string)

        # ArgumentTypeErrors indicate errors
        except ArgumentTypeError:
            name = getattr(action.type, '__name__', repr(action.type))
            msg = str(_sys.exc_info()[1])
            raise ArgumentError(action, msg)

        # TypeErrors or ValueErrors also indicate errors
        except (TypeError, ValueError):
            name = getattr(action.type, '__name__', repr(action.type))
            msg = _('invalid %s value: %r')
            raise ArgumentError(action, msg % (name, arg_string))

        # return the converted value
        return result

    def _check_value(self, action, value):
        # converted value must be one of the choices (if specified)
        if action.choices is not None and value not in action.choices:
            tup = value, ', '.join(map(repr, action.choices))
            msg = _('invalid choice: %r (choose from %s)') % tup
            raise ArgumentError(action, msg)

    # =======================
    # Help-formatting methods
    # =======================
    def format_usage(self):
        formatter = self._get_formatter()
        formatter.add_usage(self.usage, self._actions,
                            self._mutually_exclusive_groups)
        return formatter.format_help()

    def format_help(self):
        formatter = self._get_formatter()

        # usage
        formatter.add_usage(self.usage, self._actions,
                            self._mutually_exclusive_groups)

        # description
        formatter.add_text(self.description)

        # positionals, optionals and user-defined groups
        for action_group in self._action_groups:
            formatter.start_section(action_group.title)
            formatter.add_text(action_group.description)
            formatter.add_arguments(action_group._group_actions)
            formatter.end_section()

        # epilog
        formatter.add_text(self.epilog)

        # determine help from format above
        return formatter.format_help()

    def format_version(self):
        import warnings
        warnings.warn(
            'The format_version method is deprecated -- the "version" '
            'argument to ArgumentParser is no longer supported.',
            DeprecationWarning)
        formatter = self._get_formatter()
        formatter.add_text(self.version)
        return formatter.format_help()

    def _get_formatter(self):
        return self.formatter_class(prog=self.prog)

    # =====================
    # Help-printing methods
    # =====================
    def print_usage(self, file=None):
        if file is None:
            file = _sys.stdout
        self._print_message(self.format_usage(), file)

    def print_help(self, file=None):
        if file is None:
            file = _sys.stdout
        self._print_message(self.format_help(), file)

    def print_version(self, file=None):
        import warnings
        warnings.warn(
            'The print_version method is deprecated -- the "version" '
            'argument to ArgumentParser is no longer supported.',
            DeprecationWarning)
        self._print_message(self.format_version(), file)

    def _print_message(self, message, file=None):
        if message:
            if file is None:
                file = _sys.stderr
            file.write(message)

    # ===============
    # Exiting methods
    # ===============
    def exit(self, status=0, message=None):
        if message:
            self._print_message(message, _sys.stderr)
        _sys.exit(status)

    def error(self, message):
        """error(message: string)

        Prints a usage message incorporating the message to stderr and
        exits.

        If you override this in a subclass, it should not return -- it
        should either exit or raise an exception.
        """
        self.print_usage(_sys.stderr)
        self.exit(2, _('%s: error: %s\n') % (self.prog, message))


import re
import sys


class CertificateError(ValueError):
    pass


def _dnsname_match(dn, hostname, max_wildcards=1):
    """Matching according to RFC 6125, section 6.4.3

    http://tools.ietf.org/html/rfc6125#section-6.4.3
    """
    pats = []
    if not dn:
        return False

    # Ported from python3-syntax:
    # leftmost, *remainder = dn.split(r'.')
    parts = dn.split(r'.')
    leftmost = parts[0]
    remainder = parts[1:]

    wildcards = leftmost.count('*')
    if wildcards > max_wildcards:
        # Issue #17980: avoid denials of service by refusing more
        # than one wildcard per fragment.  A survey of established
        # policy among SSL implementations showed it to be a
        # reasonable choice.
        raise CertificateError(
            "too many wildcards in certificate DNS name: " + repr(dn))

    # speed up common case w/o wildcards
    if not wildcards:
        return dn.lower() == hostname.lower()

    # RFC 6125, section 6.4.3, subitem 1.
    # The client SHOULD NOT attempt to match a presented identifier in which
    # the wildcard character comprises a label other than the left-most label.
    if leftmost == '*':
        # When '*' is a fragment by itself, it matches a non-empty dotless
        # fragment.
        pats.append('[^.]+')
    elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
        # RFC 6125, section 6.4.3, subitem 3.
        # The client SHOULD NOT attempt to match a presented identifier
        # where the wildcard character is embedded within an A-label or
        # U-label of an internationalized domain name.
        pats.append(re.escape(leftmost))
    else:
        # Otherwise, '*' matches any dotless string, e.g. www*
        pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))

    # add the remaining fragments, ignore any wildcards
    for frag in remainder:
        pats.append(re.escape(frag))

    pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
    return pat.match(hostname)


def match_hostname(cert, hostname):
    dnsnames = []
    for i in range(cert.get_extension_count()):
        ext = cert.get_extension(i)
        if ext.get_short_name() == 'subjectAltName':
            data = str(ext)
            san = [(key.strip(), value.strip()) for (key, value)
                   in (item.split(':') for item in data.split(','))]
            break
    else:
        san = ()

    for key, value in san:
        if key == 'DNS':
            if _dnsname_match(value, hostname):
                return
            dnsnames.append(value)

    if not dnsnames:
        # The subject is only checked when there is no DNS entry
        # in subjectAltName
        subject = cert.get_subject()
        subject = dict(subject.get_components())
        dnsname = subject.get('CN', '')
        if _dnsname_match(dnsname, hostname):
            return
        dnsnames.append(dnsname)

    if len(dnsnames) > 1:
        raise CertificateError("hostname %r "
                               "doesn't match either of %s"
                               % (hostname, ', '.join(map(repr, dnsnames))))
    elif len(dnsnames) == 1:
        raise CertificateError("hostname %r "
                               "doesn't match %r"
                               % (hostname, dnsnames[0]))
    else:
        raise CertificateError("no appropriate commonName or "
                               "subjectAltName fields were found")


######## end argparse }}} ###############
if __name__ == "__main__":
    try:
        sys.exit(main())
    except UnknownKernelException, uke:
        logerror(str(uke))
        sys.exit(1)
    # Python 2.4 doesn't contain BaseException thus SystemExit is a subclass
    # of Exception
    except SystemExit, e:
        raise e
    except Exception, e:
        print str(e)
        kcarelog.exception(e)
        sys.exit(1)

