#!/usr/bin/env python3
##############################################################################
# Copyright (c) Members of the EGEE Collaboration. 2004.
# See http://www.eu-egee.org/partners/ for details on the copyright
# holders.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at #
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################################################

import base64
import getopt
import logging
import os
import pickle
import re
import signal
import sys
import tempfile
import time


def parse_options():
    config = {}

    try:
        opts, args = getopt.getopt(sys.argv[1:], "dc:", ["config"])
    except getopt.GetoptError:
        sys.stderr.write("Error: Invalid option specified.\n")
        print_usage()
        sys.exit(2)
    for o, a in opts:
        if o in ("-c", "--config"):
            config['BDII_CONFIG_FILE'] = a
        if o in ("-d", "--daemon"):
            config['BDII_DAEMON'] = True

    if 'BDII_CONFIG_FILE' not in config:
        sys.stderr.write("Error: Configuration file not specified.\n")
        print_usage()
        sys.exit(1)

    if not os.path.exists(config['BDII_CONFIG_FILE']):
        sys.stderr.write("Error: Configuration file %s does not exist.\n" %
                         config['BDII_CONFIG_FILE'])
        sys.exit(1)

    return config


def get_config(config):
    for line in open(config['BDII_CONFIG_FILE']).readlines():
        index = line.find("#")
        if index > -1:
            line = line[:index]
        index = line.find("=")
        if index > -1:
            key = line[:index].strip()
            value = line[index+1:].strip()
            config[key] = value

    if 'SLAPD_CONF' in os.environ:
        config['SLAPD_CONF'] = os.environ['SLAPD_CONF']

    if 'BDII_DAEMON' not in config:
        config['BDII_DAEMON'] = False

    if 'BDII_RUN_DIR' not in config:
        config['BDII_RUN_DIR'] = '/run/bdii'

    if 'BDII_PID_FILE' not in config:
        config['BDII_PID_FILE'] = "%s/bdii-update.pid" % config['BDII_RUN_DIR']

    if 'BDII_HOSTNAME' not in config:
        config['BDII_HOSTNAME'] = 'localhost'

    for parameter in ['BDII_LOG_FILE', 'BDII_LOG_LEVEL', 'BDII_LDIF_DIR',
                      'BDII_PROVIDER_DIR', 'BDII_PLUGIN_DIR',
                      'BDII_READ_TIMEOUT']:
        if parameter not in config:
            sys.stderr.write(("Error: Configuration parameter %s is not"
                              " specified in the configuration file %s.\n") % (
                                  parameter, config['BDII_CONFIG_FILE']))
            sys.exit(1)

    for parameter in ['BDII_LDIF_DIR', 'BDII_PROVIDER_DIR', 'BDII_PLUGIN_DIR']:
        if not os.path.exists(config[parameter]):
            sys.stderr.write("Error: %s %s does not exist.\n" % (
                parameter, config[parameter]))
            sys.exit(1)
    if 'BDII_LOG_LEVEL' not in config:
        config['BDII_LOG_LEVEL'] = 'ERROR'
    else:
        log_levels = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']
        try:
            log_levels.index(config['BDII_LOG_LEVEL'])
        except ValueError:
            sys.stderr.write(("Error: Log level %s is not an"
                              " allowed level. %s\n") % (
                                  config['BDII_LOG_LEVEL'], log_levels))
            sys.exit(1)

    config['BDII_READ_TIMEOUT'] = int(config['BDII_READ_TIMEOUT'])

    if config['BDII_DAEMON'] is True:
        for parameter in ['BDII_PORT', 'BDII_BREATHE_TIME', 'BDII_VAR_DIR',
                          'BDII_ARCHIVE_SIZE', 'BDII_DELETE_DELAY',
                          'SLAPD_CONF']:
            if parameter not in config:
                sys.stderr.write(("Error: Configuration parameter %s is not"
                                  " specified in the configuration file %s.\n")
                                 % (parameter, config['BDII_CONFIG_FILE']))
                sys.exit(1)
        if os.path.exists(config['SLAPD_CONF']):
            config['BDII_PASSWD'] = {}
            config['BDII_PASSWD_FILE'] = {}
            if not os.path.exists(config['BDII_RUN_DIR']):
                os.makedirs(config['BDII_RUN_DIR'])
            rootdn = False
            rootpw = False
            filename = ""
            for line in open(config['SLAPD_CONF']):
                if line.find("rootdn") > -1:
                    rootdn = line.replace("rootdn", "").strip()
                    rootdn = rootdn.replace('"', '').replace(" ", "")
                    filename = rootdn.replace('o=', '')
                    if rootpw:
                        config['BDII_PASSWD'][rootdn] = rootpw
                        config['BDII_PASSWD_FILE'][rootdn] = "%s/%s" % (
                                config['BDII_RUN_DIR'], filename)
                        pf = os.open(config['BDII_PASSWD_FILE'][rootdn],
                                     os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
                                     0o600)
                        os.write(pf, rootpw)
                        os.close(pf)
                        rootdn = False
                        rootpw = False
                if line.find("rootpw") > -1:
                    rootpw = line.replace("rootpw", "").strip()
                    if rootdn:
                        config['BDII_PASSWD'][rootdn] = rootpw
                        config['BDII_PASSWD_FILE'][rootdn] = "%s/%s" % (
                               config['BDII_RUN_DIR'], filename)
                        pf = os.open(config['BDII_PASSWD_FILE'][rootdn],
                                     os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
                                     0o600)
                        os.write(pf, rootpw.encode())
                        os.close(pf)
                        rootdn = False
                        rootpw = False
        config['BDII_BREATHE_TIME'] = float(config['BDII_BREATHE_TIME'])
        config['BDII_ARCHIVE_SIZE'] = int(config['BDII_ARCHIVE_SIZE'])
        config['BDII_DELETE_DELAY'] = int(config['BDII_DELETE_DELAY'])

    return config


def print_usage():
    sys.stderr.write('''Usage: %s [ OPTIONS ]

     -c --config      BDII configuration file
     -d --daemon      Run BDII in daemon mode
''' % str(sys.argv[0]))


def create_daemon(log_file):
    try:
        pid = os.fork()
    except OSError as e:
        return((e.errno, e.strerror))

    if pid == 0:
        os.setsid()
        signal.signal(signal.SIGHUP, signal.SIG_IGN)
        try:
            pid = os.fork()
        except OSError as e:
            return((e.errno, e.strerror))
        if pid == 0:
            os.umask(0o022)
        else:
            os._exit(0)
    else:
        os._exit(0)
    try:
        maxfd = os.sysconf("SC_OPEN_MAX")
    except (AttributeError, ValueError):
        maxfd = 256
    for fd in range(3, maxfd):
        try:
            os.close(fd)
        except OSError:
            pass
    os.close(0)
    os.open("/dev/null", os.O_RDONLY)
    os.close(1)
    os.open("/dev/null", os.O_WRONLY)

    # connect stderr to log file
    e = os.open(log_file, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644)
    os.dup2(e, 2)
    os.close(e)
    sys.stderr = os.fdopen(2, 'a')

    # Write PID
    pid_file = open(config['BDII_PID_FILE'], 'w')
    pid_file.write("%s\n" % str(os.getpid()))
    pid_file.close()


def get_logger(log_file, log_level):
    log = logging.getLogger('bdii-update')
    hdlr = logging.StreamHandler(sys.stderr)
    formatter = logging.Formatter('%(asctime)s: [%(levelname)s] %(message)s')
    hdlr.setFormatter(formatter)
    log.addHandler(hdlr)
    log.setLevel(logging.__dict__.get(log_level))

    return log


def handler(signum, frame):
    if signum == 14:
        # Commit suicide
        process_group = os.getpgrp()
        os.killpg(process_group, signal.SIGTERM)
        sys.exit(1)


def read_ldif(source):
    # Get pipe file descriptors
    read_fd, write_fd = os.pipe()

    # Fork
    pid = os.fork()

    if pid:
        # Close write file descriptor as we don't need it.
        os.close(write_fd)

        read_fh = os.fdopen(read_fd)
        raw_ldif = read_fh.read()
        result = os.waitpid(pid, 0)
        if result[1] > 0:
            log.error("Timed out while reading %s", source)
            return ""
        raw_ldif = raw_ldif.replace("\n ", "")

        return raw_ldif

    else:
        # Close read file d
        os.close(read_fd)

        # Set process group
        os.setpgrp()

        # Setup signal handler
        signal.signal(signal.SIGALRM, handler)
        signal.alarm(config['BDII_READ_TIMEOUT'])

        # Open pipe to LDIF
        if source[:7] == 'ldap://':
            url = source.split('/')
            command = "ldapsearch -LLL -x -h %s -b %s 2>/dev/null" % (
                      url[2], url[3])
            pipe = os.popen(command)
        elif source[:7] == 'file://':
            pipe = open(source[7:])
        else:
            pipe = os.popen(source)

        raw_ldif = pipe.read()

        # Close LDIF pipe
        pipe.close()
        try:
            write_fh = os.fdopen(write_fd, 'w')
            write_fh.write(raw_ldif)
            write_fh.close()
        except IOError:
            log.error("Information provider %s terminated unexpectedly." %
                      source)
        # Disable the alarm
        signal.alarm(0)
        sys.exit(0)


def get_dns(ldif):
    dns = {}
    last_dn_index = len(ldif)
    while True:
        dn_index = ldif.rfind("dn:", 0, last_dn_index)
        if dn_index == -1:
            break
        end_dn_index = ldif.find("\n", dn_index, last_dn_index)
        dn = ldif[dn_index + 4:end_dn_index].lower()
        dn = re.sub("\\s*,\\s*", ",", dn)
        dn = re.sub("\\s*=\\s*", "=", dn)
        # Replace encoded slash
        dn = dn.replace("\\5c", "\\\\")
        # Replace encoded comma
        dn = dn.replace("\\2c", "\\,")
        # Replace encoded equals
        dn = dn.replace("\\3d", "\\=")
        # Replace encoded plus
        dn = dn.replace("\\2b", "\\+")
        # Replace encoded semi colon
        dn = dn.replace("\\3b", "\\;")
        # Replace encoded quote
        dn = dn.replace("\\22", "\\\"")
        # Replace encoded greater than
        dn = dn.replace("\\3e", "\\>")
        # Replace encoded less than
        dn = dn.replace("\\3c", "\\<")
        dns[dn] = (dn_index, last_dn_index, end_dn_index)
        last_dn_index = dn_index
    return dns


def group_dns(dns):
    grouped = {}
    for dn in dns:
        index = dn.rfind(",")
        root = dn[index + 1:].strip()
        if root in grouped:
            grouped[root].append(dn)
        else:
            if root in config['BDII_PASSWD']:
                grouped[root] = [dn]
            else:
                if "o=shadow" in config['BDII_PASSWD'] and root == "o=grid":
                    grouped[root] = [dn]
                elif root != "o=shadow":
                    log.error(("dn suffix %s in not specified in the slapd"
                               " configuration file.") % root)

    return grouped


def convert_entry(entry_string):
    entry = {}
    for line in entry_string.split("\n"):
        index = line.find(":")
        if index > -1:
            attribute = line[:index].lower()
            value = line[index + 1:].strip()
            if value and line[index + 1] == ":":
                value = base64.b64decode(line[index + 2:].strip()).decode()
            if attribute in entry:
                if value not in entry[attribute]:
                    entry[attribute].append(value)
            else:
                entry[attribute] = [value]
    return entry


# From RFC2849
# SAFE-CHAR                = %x01-09 / %x0B-0C / %x0E-7F
#                           ; any value <= 127 decimal except NUL, LF,
#                           ; and CR
# SAFE-INIT-CHAR           = %x01-09 / %x0B-0C / %x0E-1F /
#                            %x21-39 / %x3B / %x3D-7F
#                            ; any value <= 127 except NUL, LF, CR,
#                            ; SPACE, colon (":", ASCII 58 decimal)
#                            ; and less-than ("<" , ASCII 60 decimal)
# SAFE-STRING              = [SAFE-INIT-CHAR *SAFE-CHAR]
safe_string = re.compile("^[\x01-\x09\x0B-\x0C\x0E-\x1F\x21-\x39\x3B\x3D-\x7F]"
                         "[\x01-\x09\x0B-\x0C\x0E-\x7F]*$")


def needs_encoding(value):
    if not value:
        return False
    return safe_string.search(value) is None


def convert_back(entry):
    entry_string = "dn: %s\n" % entry["dn"][0]
    entry.pop("dn")
    for attribute in entry.keys():
        attribute = attribute.lower()
        for value in entry[attribute]:
            if needs_encoding(value):
                entry_string += "%s:: %s\n" % (attribute,
                                               base64.b64encode(value.encode()).decode())
            else:
                entry_string += "%s: %s\n" % (attribute, value)

    return entry_string


def ldif_diff(dn, old_entry, new_entry):
    add_attribute = {}
    delete_attribute = {}
    replace_attribute = {}

    old_entry = convert_entry(old_entry)
    new_entry = convert_entry(new_entry)
    dn_perserved_case = None
    for attribute in new_entry.keys():
        attribute = attribute.lower()
        if attribute == "dn":
            dn_perserved_case = new_entry['dn'][0]
            continue

        # If the old entry has the attribue we need to compare values
        if attribute in old_entry:

            # If the old entries are different find the modify.
            if not new_entry[attribute] == old_entry[attribute]:
                replace_attribute[attribute] = new_entry[attribute]

        # The old entry does not have the attribute so add it.
        else:
            add_attribute[attribute] = new_entry[attribute]

    # Checking for removed attributes
    for attribute in old_entry.keys():
        if (attribute.lower() == "dn"):
            continue
        if attribute not in new_entry:
            delete_attribute[attribute] = old_entry[attribute]

    # Create LDIF modify statement
    ldif = ['dn: %s' % dn_perserved_case]
    ldif.append('changetype: modify')
    for attribute in add_attribute.keys():
        attribute = attribute.lower()
        ldif.append('add: %s' % attribute)
        for value in add_attribute[attribute]:
            ldif.append('%s: %s' % (attribute, value))
        ldif.append('-')
    for attribute in replace_attribute.keys():
        attribute = attribute.lower()
        ldif.append('replace: %s' % attribute)
        for value in replace_attribute[attribute]:
            ldif.append('%s: %s' % (attribute, value))
        ldif.append('-')
    for attribute in delete_attribute.keys():
        attribute = attribute.lower()
        ldif.append('delete: %s' % attribute)
        ldif.append('-')

    if len(ldif) > 3:
        ldif = "\n".join(ldif) + "\n\n"
    else:
        ldif = ""

    return ldif


def modify_entry(entry, mods):
    mods = convert_entry(mods)
    entry = convert_entry(entry)
    if 'changetype' in mods:
        # Handle LDIF delete attribute
        if 'delete' in mods:
            for attribute in mods['delete']:
                attribute = attribute.lower()
                if attribute in entry:
                    if attribute in mods:
                        for value in mods[attribute]:
                            try:
                                entry[attribute].remove(value)
                                if len(entry[attribute]) == 0:
                                    entry.pop(attribute)
                            except ValueError:
                                pass
                            except KeyError:
                                pass
                    else:
                        entry.pop(attribute)

        # Handle LDIF replace attribute
        if 'replace' in mods:
            for attribute in mods['replace']:
                attribute = attribute.lower()
                if attribute in entry:
                    if attribute in mods:
                        entry[attribute] = mods[attribute]

        # Handle LDIF add attribute
        if 'add' in mods:
            for attribute in mods['add']:
                attribute = attribute.lower()
                if attribute not in entry:
                    log.debug("attribute: %s" % attribute)
                    entry[attribute] = mods[attribute]
                else:
                    entry[attribute].extend(mods[attribute])

    # Just old style just change
    else:
        for attribute in mods.keys():
            if attribute in entry:
                entry[attribute] = mods[attribute]

    entry_string = convert_back(entry)
    return entry_string


def fix(dns, ldif):
    response = []
    append = response.append

    for dn in dns.keys():
        entry = convert_entry(ldif[dns[dn][0]:dns[dn][1]])
        if dn[:11].lower() == "mds-vo-name":
            if 'objectclass' in entry:
                if 'mds' in [x.lower() for x in entry['objectclass']]:
                    if 'gluetop' in [x.lower() for x in entry['objectclass']]:
                        value = dn[12:dn.index(",")]
                        entry = {'dn': [dn],
                                 'objectclass': ['MDS'],
                                 'mds-vo-name': [value]}
        entry = convert_back(entry)
        append(entry)
    response = "".join(response)
    return response


def log_errors(error_file, dns):
    log.debug("Logging Errors")
    request = 0
    dn = None
    error_counter = 0
    for line in open(error_file).readlines():
        if line[:7] == 'request':
            request += 1
        else:
            if request > 1:
                try:
                    if not dn == dns[request - 2]:
                        error_counter += 1
                        dn = dns[request - 2]
                        log.warn("dn: %s" % dn)
                except IndexError:
                    log.error("Problem with error reporting ...")
                    log.error("Request Num: %i, Line: %s, dns: %i" %
                              (request, line, len(dns)))
            if len(line) > 5:
                log.warn(line.strip())
    return error_counter


def main(config, log):

    log.info("Starting Update Process")
    while True:
        log.info("Starting Update")
        stats = {}
        stats['update_start'] = time.time()

        new_ldif = ""

        log.info("Reading static LDIF files ...")
        stats['read_start'] = time.time()
        ldif_files = os.listdir(config['BDII_LDIF_DIR'])
        for file_name in ldif_files:
            if file_name[-5:] == '.ldif':
                if file_name[0] not in ('#', '.'):
                    file_url = "file://%s/%s" % (config['BDII_LDIF_DIR'],
                                                 file_name)
                    log.debug("Reading %s" % file_url[7:])
                    response = read_ldif(file_url)
                    new_ldif = new_ldif + response

        stats['read_stop'] = time.time()

        log.info("Running Providers")
        stats['providers_start'] = time.time()
        providers = os.listdir(config['BDII_PROVIDER_DIR'])
        for provider in providers:
            if provider[-1:] != '~' or (provider[0] in ('#', '.')):
                log.debug("Running %s/%s" % (config['BDII_PROVIDER_DIR'],
                                             provider))
                response = read_ldif("%s/%s" % (config['BDII_PROVIDER_DIR'],
                                                provider))
                new_ldif = new_ldif + response

        stats['providers_stop'] = time.time()

        new_dns = get_dns(new_ldif)
        ldif_modify = ""

        log.info("Running Plugins")
        stats['plugins_start'] = time.time()
        plugins = os.listdir(config['BDII_PLUGIN_DIR'])
        for plugin in plugins:
            if plugin[-1:] != '~' or (plugin[0] in ('#', '.')):
                log.debug("Running %s/%s" % (config['BDII_PLUGIN_DIR'],
                                             plugin))
                response = read_ldif("%s/%s" % (config['BDII_PLUGIN_DIR'],
                                                plugin))
                modify_dns = get_dns(response)
                for dn in modify_dns.keys():
                    if dn in new_dns:
                        mod_entry = modify_entry(
                                new_ldif[new_dns[dn][0]:new_dns[dn][1]],
                                response[modify_dns[dn][0]:modify_dns[dn][1]])
                        start = len(new_ldif)
                        end = start + len(mod_entry)
                        new_dns[dn] = (start, end)
                        new_ldif = new_ldif + mod_entry
                    else:
                        ldif_modify += response[
                                modify_dns[dn][0]:modify_dns[dn][1]
                                ]
        stats['plugins_stop'] = time.time()

        log.debug("Doing Fix")
        new_ldif = fix(new_dns, new_ldif)

        log.debug("Writing new_ldif to disk")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            dump_fh = open("%s/new.ldif" % (config['BDII_VAR_DIR']), 'w')
            dump_fh.write(new_ldif)
            dump_fh.close()

        if not config['BDII_DAEMON']:
            print(new_ldif)
            sys.exit(0)

        log.info("Reading old LDIF file ...")
        stats['read_old_start'] = time.time()
        old_ldif_file = "%s/old.ldif" % (config['BDII_VAR_DIR'])
        if os.path.exists(old_ldif_file):
            old_ldif = read_ldif("file://%s" % (old_ldif_file))
        else:
            old_ldif = ""
        stats['read_old_stop'] = time.time()

        log.debug("Starting Diff")
        ldif_add = []
        ldif_delete = []

        new_dns = get_dns(new_ldif)
        old_dns = get_dns(old_ldif)

        for dn in new_dns.keys():
            if dn in old_dns:
                old = old_ldif[old_dns[dn][0]:old_dns[dn][1]].strip()
                new = new_ldif[new_dns[dn][0]:new_dns[dn][1]].strip()

                # If the entries are different we need to compare them
                if not new == old:
                    entry = ldif_diff(dn, old, new)
                    ldif_modify += entry
            else:
                ldif_add.append(dn)

        # Checking for removed entries
        for dn in old_dns.keys():
            if dn not in new_dns:
                ldif_delete.append(old_ldif[old_dns[dn][0]
                                   + 4:old_dns[dn][2]].strip())

        log.debug("Finished Diff")

        log.debug("Sorting Add Keys")
        ldif_add.sort(key=lambda x: len(x))

        log.debug("Writing ldif_add to disk")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            dump_fh = open("%s/add.ldif" % (config['BDII_VAR_DIR']), 'w')
            for dn in ldif_add:
                dump_fh.write(new_ldif[new_dns[dn][0]:new_dns[dn][1]])
                dump_fh.write("\n")
            dump_fh.close()

        log.debug("Adding New Entries")
        stats['db_update_start'] = time.time()

        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            error_file = "%s/add.err" % config['BDII_VAR_DIR']
        else:
            error_file = tempfile.mktemp()

        roots = group_dns(ldif_add)
        suffixes = list(roots.keys())
        if "o=shadow" in suffixes:
            index = suffixes.index("o=shadow")
            if index > 0:
                suffixes[index] = suffixes[0]
                suffixes[0] = "o=shadow"

        add_error_counter = 0
        for root in suffixes:
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh = os.popen(("ldapadd -d 256 -x -c -h %s -p %s"
                                     " -D %s -y %s >/dev/null 2>%s") % (
                                         config['BDII_HOSTNAME'],
                                         config['BDII_PORT'],
                                         bind,
                                         config['BDII_PASSWD_FILE'][bind],
                                         error_file), 'w')
                for dn in roots[root]:
                    input_fh.write(new_ldif[new_dns[dn][0]:new_dns[dn][1]])
                    input_fh.write("\n")
                input_fh.close()
            except (IOError, KeyError):
                log.error("Could not add new entries to the database.")

            add_error_counter += log_errors(error_file, ldif_add)

            if not config['BDII_LOG_LEVEL'] == 'DEBUG':
                os.remove(error_file)

        log.debug("Writing ldif_modify to disk")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            dump_fh = open("%s/modify.ldif" % (config['BDII_VAR_DIR']), 'w')
            dump_fh.write(ldif_modify)
            dump_fh.close()

        log.debug("Modify New Entries")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            error_file = "%s/modify.err" % config['BDII_VAR_DIR']
        else:
            error_file = tempfile.mktemp()

        ldif_modify_dns = get_dns(ldif_modify)
        roots = group_dns(ldif_modify_dns)
        modify_error_counter = 0
        for root in roots.keys():
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh = os.popen(("ldapmodify -d 256 -x -c -h %s -p %s -D"
                                     " %s -y %s >/dev/null 2>%s") % (
                                         config['BDII_HOSTNAME'],
                                         config['BDII_PORT'],
                                         bind,
                                         config['BDII_PASSWD_FILE'][bind],
                                         error_file), 'w')
                for dn in roots[root]:
                    input_fh.write(ldif_modify[
                        ldif_modify_dns[dn][0]:ldif_modify_dns[dn][1]
                        ])
                    input_fh.write("\n")
                input_fh.close()
            except (IOError, KeyError):
                log.error("Could not modify entries in the database.")

            modify_error_counter += log_errors(error_file,
                                               list(ldif_modify_dns.keys()))

            if config['BDII_LOG_LEVEL'] != 'DEBUG':
                os.remove(error_file)

        log.debug("Sorting Delete Keys")
        ldif_delete.sort(key=lambda x: len(x))

        log.debug("Writing ldif_delete to disk")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            dump_fh = open("%s/delete.ldif" % config['BDII_VAR_DIR'], 'w')
            for dn in ldif_delete:
                dump_fh.write("%s\n" % (dn))
            dump_fh.close()

        # Delayed delete Function
        if config['BDII_DELETE_DELAY'] > 0:
            log.debug("Doing Delayed Delete")
            delete_timestamp = time.time()

            # Get DNs of entries to be deleted not yet in delayed delete so
            # their status can be updated
            new_delayed_delete_file = '%s/new_delayed_delete.pkl' % config[
                    'BDII_VAR_DIR'
                    ]
            try:
                nfh = open(new_delayed_delete_file, 'w')
                nfh.write("")
            except IOError:
                log.error("Unable to open new_delayed_delete file %s" %
                          new_delayed_delete_file)

            delayed_delete_file = '%s/delayed_delete.pkl' % config[
                    'BDII_VAR_DIR'
                    ]
            if os.path.exists(delayed_delete_file):
                file_handle = open(delayed_delete_file, 'rb')
                delay_delete = pickle.load(file_handle)
                file_handle.close()
            else:
                delay_delete = {}

            # Add remove cache timestamps that have been readded
            for dn in list(delay_delete.keys()):
                if dn not in ldif_delete:
                    log.debug("Removing %s from cache (readded)" % dn)
                    delay_delete.pop(dn)

            # Add current timestamp for new deletes
            for dn in ldif_delete:
                if dn not in delay_delete:
                    delay_delete[dn] = delete_timestamp
                    nfh.write("%s\n" % (dn))
            nfh.close()

            # Remove delayed deletes from LDIF or remove from cache
            for dn in list(delay_delete.keys()):
                if delay_delete[dn] + config[
                        'BDII_DELETE_DELAY'] >= delete_timestamp:
                    ldif_delete.remove(dn)
                else:
                    delay_delete.pop(dn)

            # Store Delayed Deletes
            log.debug("Storing delayed deletes")
            file_handle = open(delayed_delete_file, 'wb')
            pickle.dump(delay_delete, file_handle)
            file_handle.close()

        log.debug("Deleting Old Entries")
        if config['BDII_LOG_LEVEL'] == 'DEBUG':
            error_file = "%s/delete.err" % config['BDII_VAR_DIR']
        else:
            error_file = tempfile.mktemp()

        roots = group_dns(ldif_delete)
        delete_error_counter = 0
        for root in roots.keys():
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh = os.popen(("ldapdelete -d 256 -x -c -h %s -p %s"
                                     " -D %s -y %s >/dev/null 2>%s") % (
                                         config['BDII_HOSTNAME'],
                                         config['BDII_PORT'],
                                         bind,
                                         config['BDII_PASSWD_FILE'][bind],
                                         error_file), 'w')
                for dn in roots[root]:
                    input_fh.write("%s\n" % dn)
                    log.debug("Deleting %s" % dn)
                input_fh.close()
            except (IOError, KeyError):
                log.error("Could not delete old entries in the database.")

            delete_error_counter += log_errors(error_file, ldif_delete)

            if config['BDII_LOG_LEVEL'] != 'DEBUG':
                os.remove(error_file)

        roots = group_dns(new_dns)
        stats['query_start'] = time.time()
        if os.path.exists("%s/old.ldif" % config['BDII_VAR_DIR']):
            os.remove("%s/old.ldif" % config['BDII_VAR_DIR'])
        if os.path.exists("%s/old.err" % config['BDII_VAR_DIR']):
            os.remove("%s/old.err" % config['BDII_VAR_DIR'])
        for root in roots.keys():
            # Stop flapping due to o=shadow
            if root == "o=shadow":
                command = ("ldapsearch -LLL -x -h %s -p %s -b %s -s base"
                           " >> %s/old.ldif 2>> %s/old.err") % (
                                   config['BDII_HOSTNAME'],
                                   config['BDII_PORT'],
                                   root,
                                   config['BDII_VAR_DIR'],
                                   config['BDII_VAR_DIR'])
            else:
                command = ("ldapsearch -LLL -x -h %s -p %s -b %s"
                           " >> %s/old.ldif 2>> %s/old.err") % (
                                   config['BDII_HOSTNAME'],
                                   config['BDII_PORT'],
                                   root,
                                   config['BDII_VAR_DIR'],
                                   config['BDII_VAR_DIR'])
            result = os.system(command)
            if result > 0:
                log.error("Query to self failed.")
        stats['query_stop'] = time.time()
        out_file = "%s/archive/%s-snapshot.gz" % (
                config['BDII_VAR_DIR'],
                time.strftime('%y-%m-%d-%H-%M-%S'))
        log.debug("Creating GZIP file")
        os.system("gzip -c %s/old.ldif > %s" % (config['BDII_VAR_DIR'],
                                                out_file))

        infosys_output = ""
        if len(old_ldif) == 0:
            log.debug("ldapadd o=infosys compression")
            command = "ldapadd"

            infosys_output += "dn: o=infosys\n"
            infosys_output += "objectClass: organization\n"
            infosys_output += "o: infosys\n\n"
            infosys_output += "dn: CompressionType=zip,o=infosys\n"
            infosys_output += "objectClass: CompressedContent\n"
            infosys_output += "Hostname: %s\n" % config['BDII_HOSTNAME']
            infosys_output += "CompressionType: zip\n"
            infosys_output += "Data: file://%s\n\n" % out_file
        else:
            log.debug("ldapmodify o=infosys compression")
            command = "ldapmodify"

            infosys_output += "dn: CompressionType=zip,o=infosys\n"
            infosys_output += "changetype: Modify\n"
            infosys_output += "replace: Data\n"
            infosys_output += "Data: file://%s\n\n" % out_file
        try:
            output_fh = os.popen(("%s -x -c -h %s -p %s -D o=infosys -y %s"
                                  " >/dev/null") % (
                                      command,
                                      config['BDII_HOSTNAME'],
                                      config['BDII_PORT'],
                                      config['BDII_PASSWD_FILE']['o=infosys']),
                                 'w')
            output_fh.write(infosys_output)
            output_fh.close()
        except (IOError, KeyError):
            log.error("Could not add compressed data to the database.")

        old_files = os.popen("ls -t %s/archive" % config[
            'BDII_VAR_DIR']).readlines()
        log.debug("Deleting old GZIP files")
        for file in old_files[config['BDII_ARCHIVE_SIZE']:]:
            os.remove("%s/archive/%s" % (config['BDII_VAR_DIR'], file.strip()))

        stats['db_update_stop'] = time.time()
        stats['update_stop'] = time.time()

        stats['UpdateTime'] = int(stats['update_stop']
                                  - stats['update_start'])
        stats['ReadTime'] = int(stats['read_old_stop']
                                - stats['read_old_start'])
        stats['ProvidersTime'] = int(stats['providers_stop']
                                     - stats['providers_start'])
        stats['PluginsTime'] = int(stats['plugins_stop']
                                   - stats['plugins_start'])
        stats['QueryTime'] = int(stats['query_stop']
                                 - stats['query_start'])
        stats['DBUpdateTime'] = int(stats['db_update_stop']
                                    - stats['db_update_start'])
        stats['TotalEntries'] = len(old_dns)
        stats['NewEntries'] = len(ldif_add)
        stats['ModifiedEntries'] = len(ldif_modify_dns.keys())
        stats['DeletedEntries'] = len(ldif_delete)
        stats['FailedAdds'] = add_error_counter
        stats['FailedModifies'] = modify_error_counter
        stats['FailedDeletes'] = delete_error_counter

        for key in stats.keys():
            if key.find("_") == -1:
                log.info("%s: %i" % (key, stats[key]))

        infosys_output = ""
        if len(old_ldif) == 0:
            log.debug("ldapadd o=infosys updatestats")
            command = "ldapadd"

            infosys_output += "dn: Hostname=%s,o=infosys\n" % config[
                                'BDII_HOSTNAME']
            infosys_output += "objectClass: UpdateStats\n"
            infosys_output += "Hostname: %s\n" % config['BDII_HOSTNAME']
            for key in stats.keys():
                if key.find("_") == -1:
                    infosys_output += "%s: %i\n" % (key, stats[key])
            infosys_output += "\n"
        else:
            log.debug("ldapmodify o=infosys updatestats")
            command = "ldapmodify"

            infosys_output += "dn: Hostname=%s,o=infosys\n" % config[
                    'BDII_HOSTNAME']
            infosys_output += "changetype: Modify\n"
            for key in stats.keys():
                if key.find("_") == -1:
                    infosys_output += "replace: %s\n" % key
                    infosys_output += "%s: %i\n" % (key, stats[key])
                    infosys_output += "-\n"
            infosys_output += "\n"
        try:
            output_fh = os.popen(("%s -x -c -h %s -p %s -D o=infosys -y %s"
                                  " >/dev/null") % (
                                      command,
                                      config['BDII_HOSTNAME'],
                                      config['BDII_PORT'],
                                      config['BDII_PASSWD_FILE']['o=infosys']),
                                 'w')
            output_fh.write(infosys_output)
            output_fh.close()
        except (IOError, KeyError):
            log.error("Could not add stats entries to the database.")

        old_ldif = None
        new_ldif = None
        new_dns = None
        ldif_delete = None
        ldif_add = None
        ldif_modify = None

        log.info("Sleeping for %i seconds" % int(config['BDII_BREATHE_TIME']))
        time.sleep(config['BDII_BREATHE_TIME'])


if __name__ == '__main__':
    config = parse_options()
    config = get_config(config)
    if config['BDII_DAEMON']:
        create_daemon(config['BDII_LOG_FILE'])
        # Giving some time for the init.d script to finish
        time.sleep(3)
    else:
        # connect stderr to log file
        e = os.open(config['BDII_LOG_FILE'],
                    os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644)
        os.dup2(e, 2)
        os.close(e)
        sys.stderr = os.fdopen(2, 'a')

    log = get_logger(config['BDII_LOG_FILE'], config['BDII_LOG_LEVEL'])
    main(config, log)
