#!/usr/bin/env python

# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Copyright (c) 2005, 2007 XenSource Ltd.

#
# To add new entries to the bugtool, you need to:
#
# Create a new capability.  These declare the new entry to the GUI, including
# the expected size, time to collect, privacy implications, and whether the
# capability should be selected by default.  One capability may refer to
# multiple files, assuming that they can be reasonably grouped together, and
# have the same privacy implications.  You need:
#
#   A new CAP_ constant.
#   A cap() invocation to declare the capability.
#
# You then need to add calls to main() to collect the files.  These will
# typically be calls to the helpers file_output(), tree_output(), cmd_output(),
# or func_output().
#

# Special pylint disables for latest pylint on xen-bugtool itself (for the moment):
# The old Pylint-1.9.x thinks some of these are useless:
# pylint: disable=useless-suppression
# pylint: disable=missing-docstring,line-too-long,multiple-statements,unnecessary-pass
# pylint: disable=broad-exception-raised,missing-type-doc,useless-object-inheritance
# pylint: disable=undefined-variable,unnecessary-comprehension

from __future__ import print_function

import fcntl
import getopt
import glob
import io
import json
import logging
import os
import platform
import pprint
import re
import socket
import sys
import tarfile
import time
import traceback
import xml
import zipfile
from collections import OrderedDict
from contextlib import contextmanager
from hashlib import md5 as md5_new
from select import select
from signal import SIGHUP, SIGTERM, SIGUSR1
from subprocess import PIPE, Popen
from xml.dom.minidom import getDOMImplementation, parse
from xml.etree import ElementTree
from xml.etree.ElementTree import Element

import defusedxml.sax

if sys.version_info.major == 2:  # pragma: no cover
    from commands import getoutput  # pyright: ignore[reportMissingImports]
    from urllib2 import HTTPError, urlopen  # type:ignore[attr-defined]

    unicode_type = unicode  # pyright:ignore[reportUndefinedVariable] # pylint: disable=unicode-builtin
else:
    from subprocess import getoutput
    from urllib.request import HTTPError, urlopen

    from typing import TYPE_CHECKING

    if TYPE_CHECKING:  # Used for type checking only:
        from _typeshed import ReadableBuffer  # pylint: disable=unused-import

    unicode_type = str

# Fixed in 3.7: https://github.com/python/cpython/pull/12628
# Monkey-patch zipfile's __del__ function to be less stupid
#   Specifically, it calls close which further writes to the file, which
#   fails with ENOSPC if the root filesystem is full
if sys.version < "3.7":
    zipfile_del = zipfile.ZipFile.__del__  # type: ignore[attr-defined] # mypy,pyright

    def exceptionless_del(*argl, **kwargs):
        try:
            zipfile_del(*argl, **kwargs)
        except OSError:
            pass
    zipfile.ZipFile.__del__ = exceptionless_del  # type: ignore[attr-defined] # mypy,pyright

def xapi_local_session():
    import XenAPI  # Import on first use.
    return XenAPI.xapi_local()

OS_RELEASE = platform.release()

#
# Files & directories
#

BUG_DIR = "/var/opt/xen/bug-report"
PLUGIN_DIR = "/etc/xensource/bugtool"
XAPI_BLOBS = '/var/xapi/blobs'
GRUB_BIOS_CONFIG = '/boot/grub/grub.cfg'
GRUB_EFI_CONFIG = '/boot/efi/EFI/xenserver/grub.cfg'
BOOT_KERNEL = '/boot/vmlinuz-' + OS_RELEASE
BOOT_INITRD = '/boot/initrd-' + OS_RELEASE + '.img'
PROC_PARTITIONS = '/proc/partitions'
FCOE_BLACKLIST_FILE = '/etc/sysconfig/fcoe-blacklist'
FCOE_CONFIG_DIR = '/etc/fcoe/'
FSTAB = '/etc/fstab'
PROC_MOUNTS = '/proc/mounts'
ISCSI_CONF = '/etc/iscsi/iscsid.conf'
ISCSI_INITIATOR = '/etc/iscsi/initiatorname.iscsi'
LVM_CACHE = '/etc/lvm/cache/.cache'
LVM_CONFIG = '/etc/lvm/lvm.conf'
PROC_BUDDYINFO = '/proc/buddyinfo'
PROC_CPUINFO = '/proc/cpuinfo'
PROC_MEMINFO = '/proc/meminfo'
PROC_PAGETYINFO = '/proc/pagetypeinfo'
PROC_SLABINFO = '/proc/slabinfo'
PROC_VMSTAT = '/proc/vmstat'
PROC_ZONEINFO = '/proc/zoneinfo'
PROC_IOMEM = '/proc/iomem'
PROC_IOPORTS = '/proc/ioports'
PROC_INTERRUPTS = '/proc/interrupts'
PROC_SOFTIRQS = '/proc/softirqs'
PROC_SCSI = '/proc/scsi/scsi'
FIRSTBOOT_DIR = '/etc/firstboot.d'
PROC_VERSION = '/proc/version'
PROC_MDSTAT  = '/proc/mdstat'
PROC_MODULES = '/proc/modules'
PROC_DEVICES = '/proc/devices'
PROC_FILESYSTEMS = '/proc/filesystems'
PROC_CMDLINE = '/proc/cmdline'
PROC_CONFIG = '/proc/config.gz'
PROC_USB_DEV = '/proc/bus/usb/devices'
PROC_XEN_BALLOON = '/proc/xen/balloon'
PROC_NET_BONDING_DIR = '/proc/net/bonding'
PROC_NET_VLAN_DIR = '/proc/net/vlan'
PROC_NET_SOFTNET_STAT = '/proc/net/softnet_stat'
PROC_NET_NETSTAT = '/proc/net/netstat'
PROC_XSVERSION = '/proc/xsversion'
ETC_MDADM_CONF = '/etc/mdadm.conf'
ETC_MDADM_MDADM_CONF = '/etc/mdadm/mdadm.conf'
AD_USERS = '/etc/security/hcp_ad_users.conf'
AD_GROUPS = '/etc/security/hcp_ad_groups.conf'
MODPROBE_CONF = '/etc/modprobe.conf'
MODPROBE_DIR = '/etc/modprobe.d'
BOOT_TIME_CPUS = '/etc/xensource/boot_time_cpus'
BOOT_TIME_MEMORY = '/etc/xensource/boot_time_memory'
SYSCONFIG_CLOCK = '/etc/sysconfig/clock'
SYSCONFIG_HWCONF = '/etc/sysconfig/hwconf'
SYSCONFIG_NETWORK = '/etc/sysconfig/network'
SYSCONFIG_NETWORK_SCRIPTS = '/etc/sysconfig/network-scripts'
SYSCONFIG_RENAME_DATA_DIR = '/etc/sysconfig/network-scripts/interface-rename-data'
NET_UDEV_RULES = '/etc/udev/rules.d/60-net.rules'
IFCFG_RE = re.compile(r'^.*/ifcfg-.*')
ROUTE_RE = re.compile(r'^.*/route-.*')
NETWORK_DBCACHE = '/var/xapi/network.dbcache'
NETWORKD_DB = '/var/xapi/networkd.db'
RESOLV_CONF = '/etc/resolv.conf'
MULTIPATH_CONF = '/etc/multipath.conf'
MULTIPATH_CONFD = '/etc/multipath/conf.d/*'
NSSWITCH_CONF = '/etc/nsswitch.conf'
CHRONY_CONF = '/etc/chrony.conf'
IPTABLES_CONFIG = '/etc/sysconfig/iptables-config'
HOSTS = '/etc/hosts'
HOSTS_ALLOW = '/etc/hosts.allow'
HOSTS_DENY = '/etc/hosts.deny'
DHCP_LEASE_DIR = '/var/lib/xcp/dhclient'
OPENVSWITCH_CORE_DIR = '/var/xen/openvswitch'
OPENVSWITCH_CONF = '/etc/ovs-vswitchd.conf'
OPENVSWITCH_CONF_DB = '/run/openvswitch/conf.db'
OPENVSWITCH_VSWITCHD_PID = '/var/run/openvswitch/ovs-vswitchd.pid'
VAR_LOG_DIR = '/var/log/'
XENSOURCE_INVENTORY = '/etc/xensource-inventory'
OEM_CONFIG_DIR = '/var/xsconfig'
OEM_CONFIG_FILES_RE = re.compile(r'^.*xensource-inventory$')
OEM_DB_FILES_RE = re.compile(r'^.*state\.db')
INITIAL_INVENTORY = '/opt/xensource/etc/initial-inventory'
VENDORKERNEL_INVENTORY = '/etc/vendorkernel-inventory'
STATIC_VDIS = '/etc/xensource/static-vdis'
POOL_CONF = '/etc/xensource/pool.conf'
NETWORK_CONF = '/etc/xensource/network.conf'
XAPI_CONF = '/etc/xapi.conf'
XAPI_CONF_DIR = '/etc/xapi.conf.d'
XENOPSD_CONF = '/etc/xenopsd.conf'
XAPI_SSL_CONF = '/etc/xensource/xapi-ssl.conf'
XAPI_GLOBS_CONF = '/etc/xensource/xapi_globs.conf'
SSHD_CONFIG = '/etc/ssh/sshd_config'
DB_CONF = '/etc/xensource/db.conf'
DB_CONF_RIO = '/etc/xensource/db.conf.rio'
DB_DEFAULT_FIELDS = '/etc/xensource/db-default-fields'
DB_SCHEMA_SQL = '/etc/xensource/db_schema.sql'
SYSTEMD_CONF_DIR = '/etc/systemd'
CGRULES_CONF = '/etc/cgrules.conf'
XAPI_LOCAL_DB = '/var/xapi/local.db'
HOST_CRASHDUMPS_DIR = '/var/crash'
HOST_CRASHDUMP_LOGS_EXCLUDES_RE = re.compile(r".*/(\.sacrificial-space-for-logs|coredump\.bin)$")
XAPI_DEBUG_DIR = '/var/xapi/debug'
INSTALLED_REPOS_DIR = '/etc/xensource/installed-repos'
UPDATE_APPLIED_DIR = '/var/update/applied'
OEM_XENSERVER_LOGS_RE = re.compile(r'^.*xensource\.log$')
XHA_LOG = '/var/log/xha.log'
XHAD_CONF = '/etc/xensource/xhad.conf'
YUM_LOG = '/var/log/yum.log'
YUM_REPOS_DIR = '/etc/yum.repos.d'
PAM_DIR = '/etc/pam.d'
FIST_RE = re.compile(r'.*/fist_')
# -------------------------------------------------------------------------------
# LWIDENTITY files and LIKEWISE_DIR are added as basis for backporting to 8.2 CU1

# LWIDENTITY_JOIN_LOG is a temporary file created by lwidentity-join
LWIDENTITY_JOIN_LOG = "/tmp/lwidentity.join.log"

# HOSTS_LWIDENTITY_ORIG is a backup of /etc/hosts created by lwidentity-join
HOSTS_LWIDENTITY_ORIG = "/etc/hosts.lwidentity.orig"

# Likewise configuration files
LIKEWISE_DIR = "/var/lib/likewise"
# -------------------------------------------------------------------------------
KRB5_CONF = '/etc/krb5.conf'
SAMBA_CONFIG_DIR = '/etc/samba'
# SAMBA_DATA_DIR = '/var/lib/samba'
QEMU_DIR = '/var/lib/xen'
QEMU_RESUME_RE = re.compile(r'.*/qemu-resume\..*')
INTERFACE_RENAME_LOG = '/var/log/interface-rename.log'
SYS_NETBACK_DEBUG = '/sys/kernel/debug/xen-netback'
SYS_EFIVARS = '/sys/firmware/efi/efivars'
BLKTAP_DEVICE_PATH = '/dev/blktap'
SYS_KERNEL_NOTES = '/sys/kernel/notes'
SIGNING_KEY_INFO_DIR = '/etc/pki/rpm-gpg'
XAPI_CLUSTERD = '/var/opt/xapi-clusterd/db'
SWTPM_IC = '/var/lib/swtpm-localca/issuercert.pem'
SWTPM_RCA ='/var/lib/swtpm-localca/swtpm-localca-rootca-cert.pem'
SWTPM_CS = '/var/lib/swtpm-localca/certserial'
NRPE_CONF = '/etc/nagios/nrpe.cfg'
NRPE_DIR = '/etc/nrpe.d'
XEN_BUGTOOL_LOG = 'xen-bugtool.log'
CRON_DIRS = '/etc/cron*'
CRON_SPOOL = '/var/spool/cron'
SNMP_XS_CONF = '/etc/snmp/snmp.xs.conf'
SNMPD_XS_CONF = '/etc/snmp/snmpd.xs.conf'
SNMPD_CONF = '/var/lib/net-snmp/snmpd.conf'
SYSCONFIG_SNMPD = '/etc/sysconfig/snmpd'

#
# External programs
#

ACPIDUMP = 'acpidump'
ARPTABLES = 'arptables'
BIN_STATIC_VDIS = 'static-vdis'
BIOSDEVNAME = 'biosdevname'
BRCTL = 'brctl'
CHRONYC = 'chronyc'
DCBTOOL = 'dcbtool'
DF = 'df'
DU = 'du'
DMESG = 'dmesg'
DMIDECODE = 'dmidecode'
DMSETUP = 'dmsetup'
EBTABLES = 'ebtables'
EFIBOOTMGR = 'efibootmgr'
ETHTOOL = 'ethtool'
FCOEADM = 'fcoeadm'
FDISK = 'fdisk'
HA_QUERY_LIVESET = '/opt/xensource/debug/debug_ha_query_liveset'
HDPARM = 'hdparm'
IP = "ip"
IPTABLES = 'iptables'
ISCSIADM = 'iscsiadm'
KPATCH = 'kpatch'
LIST_DOMAINS = 'list_domains'
LLDPTOOL = 'lldptool'
LOSETUP = 'losetup'
LS = 'ls'
LSBLK = 'lsblk'
LSPCI = 'lspci'
LVDISPLAY = 'lvdisplay'
LVS = 'lvs'
MD5SUM = 'md5sum'
MDADM = 'mdadm'
MODINFO = 'modinfo'
MULTIPATHD = 'multipathd'
NSTAT = 'nstat'
SS = 'ss'
OVS_APPCTL = 'ovs-appctl'
OVS_DPCTL = 'ovs-dpctl'
OVS_OFCTL = 'ovs-ofctl'
OVS_VSCTL = 'ovs-vsctl'
PS = 'ps'
PVS = 'pvs'
QLOGIC_FW = 'strings /lib/firmware/ql2*.bin | grep -i ver'
RPM = 'rpm'
SG_MAP = 'sg_map'
SYSCTL = 'sysctl'
SYSTEMCTL = 'systemctl'
TC = 'tc'
ULIMIT = 'ulimit -a'
UPTIME = 'uptime'
VGS = 'vgs'
VGSCAN = 'vgscan'
XE = 'xe'
XENPM = 'xenpm'
XEN_CPUID = 'xen-cpuid'
XEN_LIVEPATCH = 'xen-livepatch'
XEN_MICROCODE = 'xen-ucode'
XENSTORE_LS = 'xenstore-ls'
XL = 'xl'
ZCAT = 'zcat'

#
# PII -- Personally identifiable information.  Of particular concern are
# things that would identify customers, or their network topology.
# Passwords are never to be included in any bug report, regardless of any PII
# declaration.
#
# NO            -- No PII will be in these entries.
# YES           -- PII will likely or certainly be in these entries.
# MAYBE         -- The user may wish to audit these entries for PII.
# IF_CUSTOMIZED -- If the files are unmodified, then they will contain no PII,
# but since we encourage customers to edit these files, PII may have been
# introduced by the customer.  This is used in particular for the networking
# scripts in dom0.
#

PII_NO            = 'no'
PII_YES           = 'yes'
PII_MAYBE         = 'maybe'
PII_IF_CUSTOMIZED = 'if_customized'
KEY      = 0
PII      = 1
MIN_SIZE = 2
MAX_SIZE = 3
MIN_TIME = 4
MAX_TIME = 5
MIME     = 6
CHECKED  = 7
HIDDEN   = 8
VERBOSITY = 9

MIME_DATA = 'application/data'
MIME_TEXT = 'text/plain'

INVENTORY_XML_ROOT = "system-status-inventory"
INVENTORY_XML_SUMMARY = 'system-summary'
INVENTORY_XML_ELEMENT = 'inventory-entry'
CAP_XML_ROOT = "system-status-capabilities"
CAP_XML_ELEMENT = 'capability'


CAP_BLOBS                = 'blobs'
CAP_BOOT_LOADER          = 'boot-loader'
CAP_DEVICE_MODEL         = 'device-model'
CAP_DISK_INFO            = 'disk-info'
CAP_FCOE                 = 'fcoe'
CAP_FIRSTBOOT            = 'firstboot'
CAP_HARDWARE_INFO        = 'hardware-info'
CAP_HDPARM_T             = 'hdparm-t'
CAP_HIGH_AVAILABILITY    = 'high-availability'
CAP_HOST_CRASHDUMP_LOGS  = 'host-crashdump-logs'
CAP_KERNEL_INFO          = 'kernel-info'
CAP_LOSETUP_A            = 'loopback-devices'
CAP_MULTIPATH            = 'multipath'
CAP_NETWORK_CONFIG       = 'network-config'
CAP_NETWORK_STATUS       = 'network-status'
CAP_OEM                  = 'oem'
CAP_PAM                  = 'pam'
CAP_PROCESS_LIST         = 'process-list'
CAP_PERSISTENT_STATS     = 'persistent-stats'
CAP_BLOCK_SCHEDULER      = 'block-scheduler'
CAP_SYSTEM_LOAD          = 'system-load'
CAP_SYSTEM_LOGS          = 'system-logs'
CAP_SYSTEM_SERVICES      = 'system-services'
CAP_TAPDISK_LOGS         = 'tapdisk-logs'
CAP_VTPM                 = 'vtpm'
CAP_XAPI_DEBUG           = 'xapi-debug'
CAP_XAPI_SUBPROCESS      = 'xapi-subprocess'
CAP_XEN_BUGTOOL          = 'xen-bugtool'
CAP_XENRT                = 'xenrt'
CAP_XENSERVER_CONFIG     = 'xenserver-config'
CAP_XENSERVER_DOMAINS    = 'xenserver-domains'
CAP_XENSERVER_DATABASES  = 'xenserver-databases'
CAP_XENSERVER_INSTALL    = 'xenserver-install'
CAP_XENSERVER_LOGS       = 'xenserver-logs'
CAP_XEN_INFO             = 'xen-info'
CAP_XHA_LIVESET          = 'xha-liveset'
CAP_YUM                  = 'yum'
CAP_CRON                 = 'cron'

KB = 1024
MB = 1024 * 1024

# max size of vswitch database
CAP_NETWORK_CONFIG_OVERHEAD = 10 * MB

# max size of xenserver databases
CAP_XENSERVER_DATABASES_SIZE_OVERHEAD = 2 * MB
# max capture time of xenserver databases
CAP_XENSERVER_DATABASES_TIME_OVERHEAD = 40

caps = {}
cap_sizes = {}
unlimited_data = False
unlimited_time = False
dbg = False

def cap(key, pii=PII_MAYBE, min_size=-1, max_size=-1, min_time=-1,
        max_time=-1, mime=MIME_TEXT, checked=True, hidden=False, verbosity=9):
    if  os.getenv('XEN_RT') and max_time > 0:
        max_time *= 5
    caps[key] = (key, pii, min_size, max_size, min_time, max_time, mime,
                 checked, hidden, verbosity)
    cap_sizes[key] = 0


cap(CAP_BLOBS,               PII_NO,                    max_size=5*MB)
cap(CAP_BOOT_LOADER,         PII_NO,                    max_size=3*KB,
    max_time=10)
cap(CAP_DEVICE_MODEL,        PII_YES,   min_size=200*KB, max_size=8*MB)
cap(CAP_DISK_INFO,           PII_MAYBE,                 max_size=8*MB,
    max_time=120)
cap(CAP_FCOE,                PII_YES,   max_size=4*KB, max_time=10)
cap(CAP_FIRSTBOOT,           PII_YES,   min_size=60*KB, max_size=80*KB)
cap(CAP_HARDWARE_INFO,       PII_MAYBE,                 max_size=800*KB,
    max_time=60)
cap(CAP_HDPARM_T,            PII_NO,    min_size=0,     max_size=5*KB,
    min_time=20, max_time=90, checked=False, hidden=True)
cap(CAP_HIGH_AVAILABILITY,   PII_MAYBE,                 max_size=5*MB)
cap(CAP_HOST_CRASHDUMP_LOGS, PII_MAYBE)
cap(CAP_KERNEL_INFO,         PII_MAYBE,                 max_size=360*KB,
    max_time=50)
cap(CAP_LOSETUP_A,           PII_MAYBE,                 max_size=KB, max_time=5)
cap(CAP_MULTIPATH,           PII_MAYBE,                 max_size=20*KB,
    max_time=10)
cap(CAP_NETWORK_CONFIG,      PII_IF_CUSTOMIZED,
                                        min_size=0,     max_size=100*KB)
cap(CAP_NETWORK_STATUS,      PII_YES,                   max_size=20*KB,
    max_time=30)
cap(CAP_PAM,                 PII_YES,                   max_size=50*MB)
cap(CAP_PERSISTENT_STATS,    PII_NO,                    max_size=50*MB,
    max_time=60, checked=False, hidden=True)
cap(CAP_PROCESS_LIST,        PII_YES,                   max_size=30*KB,
    max_time=60)
cap(CAP_SYSTEM_LOAD,         PII_MAYBE,                 max_size=70*MB, max_time=30)
cap(CAP_SYSTEM_LOGS,         PII_MAYBE,                 max_size=50*MB,
    max_time=10)
cap(CAP_SYSTEM_SERVICES,     PII_NO,                    max_size=128*KB,
    max_time=20)
cap(CAP_TAPDISK_LOGS,        PII_NO,                    max_size=10*MB)
cap(CAP_VTPM,                PII_NO,                    max_size=5*KB)
cap(CAP_XAPI_DEBUG,          PII_MAYBE,                 max_size=10*MB)
cap(CAP_XAPI_SUBPROCESS,     PII_NO,                    max_size=5*KB,
    max_time=10)
cap(CAP_XEN_BUGTOOL,         PII_NO,    min_size=0)
cap(CAP_XENRT,               PII_NO,    min_size=0,     max_size=500*MB,
    checked=False, hidden=True)
cap(CAP_XENSERVER_CONFIG,    PII_MAYBE,                 max_size=80*KB,
    max_time=10)
cap(CAP_XENSERVER_DOMAINS,   PII_NO,                    max_size=1*KB,
    max_time=10)
cap(CAP_XENSERVER_DATABASES, PII_YES,                   max_size=10*KB,
    max_time=5)
cap(CAP_XENSERVER_INSTALL,   PII_MAYBE, min_size=10*KB, max_size=4*MB)
cap(CAP_XENSERVER_LOGS,      PII_MAYBE, min_size=0,     max_size=70*MB)
cap(CAP_XEN_INFO,            PII_MAYBE,                 max_size=20*KB,
    max_time=10)
cap(CAP_XHA_LIVESET,         PII_MAYBE,                 max_size=10*KB,
    max_time=10)
cap(CAP_YUM,                 PII_IF_CUSTOMIZED,         max_size=100*KB,
    max_time=30)
cap(CAP_CRON,                PII_IF_CUSTOMIZED,         max_size=100*KB,
    max_time=30)
cap(CAP_BLOCK_SCHEDULER,           PII_NO,                    max_size=100*KB,
    max_time=30)

ANSWER_YES_TO_ALL = False
SILENT_MODE = False
entries = None
data = {}
"""The global data dictionary of xen-bugtool
It is filled by the various *_output functions, and then used to create the
inventory.xml and the tarball or zipfile.

The keys are the filenames of the collected data, and the values are
dictionaries with the following keys:
  - "cap": The capability of the data: one of the CAP_ constants
  - "path": If created by file_output: The path to the file to collect
  - "func": If created by func_output: A function that returns the file data
  - "cmd_args": If crated by cmd_output: The command to return the file data
  - "filter": An optional filter function to pass the file data through
"""

directory_specifications = OrderedDict()
dev_null = open('/dev/null', 'r+')


class StringIOmtime(io.BytesIO):
    """Byte buffer object with mtime for TarOutput/ZipOutput"""

    def __init__(self, buf=b""):  # type: (StringIOmtime, bytes) -> None
        """Create a StringIOmtime object with mtime for TarOutput/ZipOutput"""
        io.BytesIO.__init__(self, buf)
        self.mtime = time.time()

    def write(self, s):  # type: (StringIOmtime, ReadableBuffer) -> int
        """Write to the StringIOmtime object and update the mtime"""
        self.mtime = time.time()
        return io.BytesIO.write(self, no_unicode(s))


def no_unicode(x):
    return x.encode("utf-8") if isinstance(x, unicode_type) else x


@contextmanager
def log_exceptions():
    """Context manager to log exceptions."""
    try:
        yield
    except Exception as e:  # pylint: disable=broad-except
        log("%s, %s" % (e, traceback.format_exc()))


def log_internal_error(_):
    """Log an internal error and return an empty string as bugtool output data"""
    log("bugtool: Internal error: " + traceback.format_exc())
    return ""  # For now, no diagnostic output in the individual output files


def log(x, print_output=True):
    if print_output:
        output(x)

    with open(XEN_BUGTOOL_LOG, "a") as logFile:
        print(x, file=logFile)

def output(x):
    if not SILENT_MODE:
        print(x)

def output_ts(x):
    output("[%s]  %s" % (time.strftime("%x %X %Z"), x))

def cmd_output(cap, args, label = None, filter = None):
    if cap in entries:
        if not label:
            if isinstance(args, list):
                a = [aa for aa in args]
                a[0] = os.path.basename(a[0])
                label = ' '.join(a)
            else:
                label = args
        data[label] = {'cap': cap, 'cmd_args': args, 'filter': filter}

def dir_list(cap, path_list, recursive = False):
    flags = '-l'
    if recursive:
        flags = '-lR'

    pl = []
    for path in path_list:
        pl.extend(glob.glob(path))

    for p in pl:
        cmd_output(cap, [LS, flags, p])

def file_output(cap, path_list):
    if cap in entries:
        pl = []
        for path in path_list:
            pl.extend(glob.glob(path))

        for p in pl:
            try:
                s = os.stat(p)
                if unlimited_data or caps[cap][MAX_SIZE] == -1 or \
                        cap_sizes[cap] < caps[cap][MAX_SIZE] or s.st_size == 0:
                    data[p] = {'cap': cap, 'filename': p}
                    cap_sizes[cap] += s.st_size
                else:
                    log("Omitting %s, size constraint of %s exceeded" % (p, cap))
            except:
                pass

def tree_output(cap, path, pattern = None, negate = False):
    if cap in entries:
        if path in directory_specifications:
            directory_specifications[path].append((cap, pattern, negate))
        else:
            directory_specifications[path] = [(cap, pattern, negate)]


def traverse_directory_specifications(directory_specs, requested_capabilities):
    """Lookup the defined directories on the requested pattern and negate.

    :param directory_specs: Directories to lookup with cap, pattern, and negate.
    :param requested_capabilities: The list of requested capabilities.
    """
    for directory, tree_output_entries in directory_specs.items():
        # Multiple tree_output calls may have appended multiple output entries:
        for directory_items in tree_output_entries:
            # Unpack the stored capability, pattern and negate flag of each row:
            capability, pattern, negate = directory_items
            # If the capability is in the requested inventory entries, check it:
            if capability in requested_capabilities:
                if os.path.isdir(directory):
                    lookup_tree_recursively(capability, directory, pattern, negate)


def lookup_tree_recursively(cap, path, pattern, negate):
    """Lookup the directory at the path for files matching pattern recursively.

    :param cap (str): The inventory entry to associate with all matching files.
    :param path (str): The path to start traversing from.
    :param pattern (str or None): The pattern to match the filenames against.
    :param negate (bool): If True, negate the pattern matching.
    """
    try:
        for f in os.listdir(path):
            fn = os.path.join(path, f)
            if matches(fn, pattern, negate) and os.path.isfile(fn):
                file_output(cap, [fn])
            elif os.path.isdir(fn):
                lookup_tree_recursively(cap, fn, pattern, negate)
    except Exception as e:
        logging.info("Lookup for " + cap + ": %s" % e)


def func_output(cap, label, func):
    if cap in entries:
        data[label] = {'cap': cap, 'func': func}


def get_recent_logs(logs, verbosity):
    """Return list of filenames, sorted by mtime. On verbosity<9, only the first x"""

    # Get the mtime of each logfile in a list of tuples and sort the tuples by mtime:
    logs = sorted([(os.stat(e).st_mtime, e) for e in logs])
    # On verbosity<9, return the latest <verbosity> elements, otherwise all logfiles:
    return [x[1] for x in (logs[-verbosity:] if verbosity < 9 else logs)]


def include_inventory(archive, dir):
    """Add the inventory.xml to the archive, filled from the current data"""

    info = StringIOmtime(make_inventory(data, dir))
    archive.add_path_with_data(construct_filename(dir, "inventory.xml", {}), info)


def run_procs_and_capture_collected_output(collect, subdir, archive):
    """Prepare and run processes defined in the passed bugtool data dictionary.

    :param collect (dict): Dict with instructions on which processes to run.
    :param subdir (str): The subdirectory where the output files will be stored.
    :param archive (obj): The archive object used to store the output files.
    """

    # Prepare the process_list dictionary
    process_lists = {}
    for k, v in collect.items():
        name = construct_filename(subdir, k, v)
        cap = v['cap']
        if "cmd_args" in v:
            v['output'] = StringIOmtime()
            if cap not in process_lists:
                process_lists[cap] = []
            process_lists[cap].append(
                ProcOutputAndArchive(
                    v["cmd_args"],
                    caps[cap][MAX_TIME],
                    name,
                    archive,
                    v,
                )
            )

    # Run the processes in the process_list dictionary in parallel
    run_procs(process_lists.values())

    # collect all output from processes and free the allocated memory
    for k, v in data.items():
        if 'output' in v:
            archive.add_path_with_data(construct_filename(subdir, k, v), v["output"])
            v['md5'] = md5sum(v)
            del v['output']


def collect_data(subdir, archive):
    """Collect all requested data and archive it in the passed output archive

    :param subdir: The toplevel directory in which to store the output files.
    :param archive: The archive object used to store the output files.
    """
    # Run processes first as some (rrd-cli save_rrds) may create/update files:
    run_procs_and_capture_collected_output(data, subdir, archive)

    # Afterwards, traverse the directory specifications for files to add
    traverse_directory_specifications(directory_specifications, entries)

    # Then, loop over the files which were found and collect their contents
    for k, v in data.items():
        if "cmd_args" in v:
            continue  # commands processing has been moved to a different loop
        name = construct_filename(subdir, k, v)
        cap = v["cap"]
        filename = v.get("filename")
        if filename and (filename.startswith("/proc/") or filename.startswith("/sys/")):
            # proc files must be read into memory
            try:
                f = open(v["filename"], "rb")
                s = f.read(unlimited_data and -1 or caps[cap][MAX_SIZE])
                f.close()
                if unlimited_data or caps[cap][MAX_SIZE] == -1 or \
                        cap_sizes[cap] < caps[cap][MAX_SIZE] or len(s) == 0:
                    v['output'] = StringIOmtime(s)
                    archive.add_path_with_data(name, v['output'])
                    v['md5'] = md5sum(v)
                    del v['output']
                    cap_sizes[cap] += len(s)
                else:
                    log("Omitting %s, size constraint of %s exceeded" % (v['filename'], cap))
            except IOError as e:
                if e.errno != 2:
                    log("IOError reading %s: %s" % (filename, e))
        elif "func" in v:
            try:
                s = no_unicode(v["func"](cap))
            except Exception:
                backtrace = traceback.format_exc()  # type: str
                log(backtrace)
                s = backtrace.encode()
            if unlimited_data or caps[cap][MAX_SIZE] == -1 or \
                    cap_sizes[cap] < caps[cap][MAX_SIZE]:
                v['output'] = StringIOmtime(s)
                archive.add_path_with_data(name, v['output'])
                v['md5'] = md5sum(v)
                del v['output']
                cap_sizes[cap] += len(s)
            else:
                log("Omitting %s, size constraint of %s exceeded" % (k, cap))
        elif filename:
            try:
                archive.addRealFile(name, filename)
            except:
                pass


def usage():
    return '''Usage: xenserver-status-report [OPTION]...
Capture information to help diagnose bugs.

 --capabilities      output capabilities informations
 --output            specify output format (tar, tar.bz2 or zip)
 -s, --silent        silent mode
 --entries=<list>    specify which capabilities (separated by commas)
 -y, --yestoall      confirm every file automatically
 --outfd=<file>      specify output file
 -a, --all           enable all capabilities
 -u, --unlimited     do not limit file size and execution time
 -d, --debug         enable debug output
 --help              this help'''


def main(argv=None):  # pylint: disable=too-many-statements,too-many-branches
    global ANSWER_YES_TO_ALL, SILENT_MODE
    global entries, dbg
    global unlimited_data, unlimited_time

    output_type = 'tar.bz2'
    output_fd = -1

    # Set a default PATH
    path = ['/opt/xensource/bin', '/usr/local/sbin', '/usr/local/bin',
            '/usr/sbin', '/usr/bin', '/root/bin']
    if 'PATH' in os.environ:
        for element in os.environ['PATH'].split(':'):
            if element not in path:
                path.append(element)
    os.environ['PATH'] = ':'.join(path)

    if argv is None:
        argv = sys.argv

    # Ensure we have a clean bugtool log file
    try:
        os.remove(XEN_BUGTOOL_LOG)
    except:
        pass

    log(" ".join(argv), print_output=False)
    log("PATH=%s" % os.environ['PATH'], print_output=False)

    try:
        (options, params) = getopt.gnu_getopt(
            argv, 'adsuy', ['capabilities', 'silent', 'yestoall', 'entries=',
                            'output=', 'outfd=', 'all', 'unlimited', 'debug',
                            'help'])
    except getopt.GetoptError as opterr:
        logging.fatal("xen-bugtool: %s", opterr)
        logging.fatal(usage())
        return 2

    for (k, v) in options:
        if k == '--help':
            print(usage())
            return 0

    # we need access to privileged files, exit if we are not running as root
    if os.getuid() != 0:
        logging.fatal("Error: xen-bugtool must be run as root")
        return 1

    try:
        load_plugins(True)
    except:
        pass

    inventory = readKeyValueFile(XENSOURCE_INVENTORY)
    if "OEM_BUILD_NUMBER" in inventory:
        cap(CAP_OEM,                 PII_MAYBE,                 max_size=5*MB,
            max_time=90)

    entries = [cap_key for cap_key in caps if caps[cap_key][CHECKED]]
    if os.getenv('XEN_RT'):
        entries += [e for e in [CAP_BLOBS, CAP_BOOT_LOADER,  # pragma: no cover
                   CAP_DEVICE_MODEL, CAP_DISK_INFO, CAP_FCOE, CAP_FIRSTBOOT,
                   CAP_HARDWARE_INFO, CAP_HOST_CRASHDUMP_LOGS, CAP_KERNEL_INFO, CAP_LOSETUP_A,
                   CAP_NETWORK_CONFIG, CAP_NETWORK_STATUS, CAP_PROCESS_LIST, CAP_HIGH_AVAILABILITY,
                   CAP_PAM, CAP_MULTIPATH,
                   CAP_SYSTEM_LOGS, CAP_SYSTEM_SERVICES, CAP_TAPDISK_LOGS,
                   CAP_XAPI_DEBUG, CAP_XAPI_SUBPROCESS, CAP_VTPM,
                   CAP_XENRT, CAP_XENSERVER_CONFIG, CAP_XENSERVER_DOMAINS, CAP_XENSERVER_DATABASES,
                   CAP_XENSERVER_INSTALL, CAP_XENSERVER_LOGS, CAP_XEN_INFO, CAP_XHA_LIVESET, CAP_YUM] \
                   if e not in entries]

    update_capabilities()

    for (k, v) in options:
        if k == '--capabilities':
            print_capabilities()
            return 0

        if k == '--output':
            if  v in ['tar', 'tar.bz2', 'zip']:
                output_type = v
            else:
                logging.fatal("Invalid output format '%s'", v)
                return 2

        # "-s" or "--silent" means suppress output (except for the final
        # output filename at the end)
        if k in ['-s', '--silent']:
            SILENT_MODE = True

        if k == '--entries' and v != '':
            # parse verbosity option in entries string
            entries = []
            items = v.split(',')
            for item in items:
                item = item.split(':')
                entries.append(item[0])
                if item[0] in caps and len(item) > 1:
                    update_cap(item[0], VERBOSITY, min(9, max(1, int(item[1]))))
            for key in entries:
                if key not in caps:
                    logging.warning("--entries=%s is not known!", key)
                    entries.remove(key)

        # If the user runs the script with "-y" or "--yestoall" we don't ask
        # all the really annoying questions.
        if k in ['-y', '--yestoall']:
            ANSWER_YES_TO_ALL = True

        if k == '--outfd':
            output_fd = int(v)
            try:
                old = fcntl.fcntl(output_fd, fcntl.F_GETFD)
                fcntl.fcntl(output_fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
            except:
                logging.fatal("Invalid output file descriptor: %d", output_fd)
                return 2

        elif k in ['-a', '--all']:
            entries = list(caps.keys())
        elif k in ['-u', '--unlimited']:
            unlimited_data = True
            unlimited_time = True
        elif k in ['-d', '--debug']:
            dbg = True
            ProcOutput.debug = True
            logging.getLogger().setLevel(logging.DEBUG)  # Activates logging.debug("log messages")

    if len(params) != 1:
        logging.fatal("Invalid additional arguments: %s", str(params))
        return 2

    if output_fd != -1 and output_type != 'tar':
        logging.fatal("Option '--outfd' only valid with '--output=tar'")
        return 2

    if ANSWER_YES_TO_ALL:
        output("Warning: '--yestoall' argument provided, will not prompt for individual files.")

    output('''
This application will collate the Xen dmesg output, details of the
hardware configuration of your machine, information about the build of
Xen that you are using, plus, if you allow it, various logs.

The collated information will be saved as a .%s for archiving or
sending to a Technical Support Representative.

The logs may contain private information, and if you are at all
worried about that, you should exit now, or you should explicitly
exclude those logs from the archive.

''' % output_type)

    # assemble potential data
    tree_output(CAP_BLOBS, XAPI_BLOBS)

    file_output(CAP_BOOT_LOADER, [GRUB_BIOS_CONFIG])
    file_output(CAP_BOOT_LOADER, [GRUB_EFI_CONFIG])
    cmd_output(CAP_BOOT_LOADER, [LS, '-lR', '/boot'])
    cmd_output(CAP_BOOT_LOADER, [MD5SUM, BOOT_KERNEL, BOOT_INITRD], label='vmlinuz-initrd.md5sum')
    cmd_output(CAP_BOOT_LOADER, [EFIBOOTMGR, '-v'])

    file_output(CAP_CRON, [CRON_DIRS + "/*"])
    file_output(CAP_CRON, [os.path.join(CRON_SPOOL, '*')])

    tree_output(CAP_DEVICE_MODEL, QEMU_DIR, QEMU_RESUME_RE)

    cmd_output(CAP_DISK_INFO, [FDISK, '-l'])
    file_output(CAP_DISK_INFO, [PROC_PARTITIONS, PROC_MOUNTS])
    file_output(CAP_DISK_INFO, [FSTAB, ISCSI_CONF, ISCSI_INITIATOR])
    cmd_output(CAP_DISK_INFO, [DF, '-alT'])
    cmd_output(CAP_DISK_INFO, [DF, '-alTi'])
    cmd_output(CAP_DISK_INFO, [DU, '-ax', '/'])
    for d in disk_list():
        cmd_output(CAP_DISK_INFO, [HDPARM, '-I', '/dev/%s' % d])
    if len(pidof('iscsid')) != 0:
        cmd_output(CAP_DISK_INFO, [ISCSIADM, '-m', 'node'])
        cmd_output(CAP_DISK_INFO, [ISCSIADM, '-m', 'session', '-P', '3'])
        cmd_output(CAP_DISK_INFO, [ISCSIADM, '-m', 'iface'])
    cmd_output(CAP_DISK_INFO, [VGSCAN])
    cmd_output(CAP_DISK_INFO, [PVS])
    cmd_output(CAP_DISK_INFO, [VGS])
    cmd_output(CAP_DISK_INFO, [LVS])
    file_output(CAP_DISK_INFO, [LVM_CACHE, LVM_CONFIG])
    cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_host'])
    cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_disk'])
    cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/fc_transport'])
    cmd_output(CAP_DISK_INFO, [SG_MAP, '-x'])
    func_output(CAP_DISK_INFO, 'scsi-hosts', dump_scsi_hosts)
    cmd_output(CAP_DISK_INFO, [LVDISPLAY, '--map'])
    cmd_output(CAP_BLOCK_SCHEDULER, [LSBLK, '-io', 'type,name,sched,tran,rota,log-sec,rq-size,vendor,model'], label='lsblk')
    file_output(CAP_DISK_INFO, ['/sys/block/sd*/device/scsi_disk/*/provisioning_mode',
                                '/sys/block/sd*/device/scsi_disk/*/thin_provisioning'])

    # mdadm information
    cmd_output(CAP_DISK_INFO, [MDADM, '--detail-platform'])
    cmd_output(CAP_DISK_INFO, [MDADM, '--detail', '--scan'])
    file_output(CAP_DISK_INFO, [PROC_MDSTAT, ETC_MDADM_CONF, ETC_MDADM_MDADM_CONF])
    for a in mdadm_arrays():
        cmd_output(CAP_DISK_INFO, [MDADM, '--query', '--detail', a])

    tree_output(CAP_FIRSTBOOT, FIRSTBOOT_DIR)

    file_output(CAP_HARDWARE_INFO, [PROC_CPUINFO, PROC_MEMINFO, PROC_IOMEM, PROC_IOPORTS, PROC_INTERRUPTS])
    file_output(CAP_HARDWARE_INFO, [PROC_SOFTIRQS])
    file_output(CAP_HARDWARE_INFO, [PROC_BUDDYINFO])
    file_output(CAP_HARDWARE_INFO, [PROC_PAGETYINFO])
    file_output(CAP_HARDWARE_INFO, [PROC_SLABINFO])
    file_output(CAP_HARDWARE_INFO, [PROC_VMSTAT])
    file_output(CAP_HARDWARE_INFO, [PROC_ZONEINFO])
    cmd_output(CAP_HARDWARE_INFO, [DMIDECODE])
    cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-n'])
    cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-tv'])
    cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-vv'])
    cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-nm'])
    cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-nnm'])
    cmd_output(CAP_HARDWARE_INFO, [ACPIDUMP])
    file_output(CAP_HARDWARE_INFO, [PROC_USB_DEV, PROC_SCSI])
    file_output(CAP_HARDWARE_INFO, [BOOT_TIME_CPUS, BOOT_TIME_MEMORY])
    file_output(CAP_HARDWARE_INFO, [SYSCONFIG_HWCONF])
    cmd_output(CAP_HARDWARE_INFO, [LS, '-lR', '/dev'])
    cmd_output(CAP_HARDWARE_INFO, [XENPM, 'get-cpu-topology'])
    cmd_output(CAP_HARDWARE_INFO, [XENPM, 'get-cpufreq-states'])
    cmd_output(CAP_HARDWARE_INFO, [XENPM, 'get-cpuidle-states'])
    tree_output(CAP_HARDWARE_INFO, SYS_EFIVARS)
    # FIXME IDE?

    for d in disk_list():
        cmd_output(CAP_HDPARM_T, [HDPARM, '-tT', '/dev/%s' % d])

    file_output(CAP_HIGH_AVAILABILITY, [XHAD_CONF, XHA_LOG])

    tree_output(CAP_HOST_CRASHDUMP_LOGS, HOST_CRASHDUMPS_DIR,
                HOST_CRASHDUMP_LOGS_EXCLUDES_RE, True)

    file_output(CAP_KERNEL_INFO, [PROC_VERSION, PROC_MODULES, PROC_DEVICES,
                                  PROC_FILESYSTEMS, PROC_CMDLINE])
    cmd_output(CAP_KERNEL_INFO, [ZCAT, PROC_CONFIG], label='config')
    cmd_output(CAP_KERNEL_INFO, [SYSCTL, '-A'])
    file_output(CAP_KERNEL_INFO, [MODPROBE_CONF])
    tree_output(CAP_KERNEL_INFO, MODPROBE_DIR)
    func_output(CAP_KERNEL_INFO, 'modinfo', module_info)
    cmd_output(CAP_KERNEL_INFO, QLOGIC_FW, label='qlogic_fw')
    cmd_output(CAP_KERNEL_INFO, ULIMIT, label='ulimit-a')
    cmd_output(CAP_KERNEL_INFO, [KPATCH, 'list'])
    file_output(CAP_KERNEL_INFO, [SYS_KERNEL_NOTES])
    file_output(CAP_KERNEL_INFO, [PROC_XSVERSION])

    cmd_output(CAP_LOSETUP_A, [LOSETUP, '-a'])

    file_output(CAP_MULTIPATH, [MULTIPATH_CONF, MULTIPATH_CONFD])
    cmd_output(CAP_MULTIPATH, [DMSETUP, 'table'])
    cmd_output(CAP_MULTIPATH, [DMSETUP, 'info'])
    func_output(CAP_MULTIPATH, 'multipathd_topology', multipathd_topology)

    file_output(CAP_NETWORK_CONFIG, [NET_UDEV_RULES])
    tree_output(CAP_NETWORK_CONFIG, SYSCONFIG_RENAME_DATA_DIR)
    file_output(CAP_NETWORK_CONFIG, [INTERFACE_RENAME_LOG])
    file_output(CAP_NETWORK_CONFIG, [NETWORK_CONF])
    file_output(CAP_NETWORK_CONFIG, [NETWORK_DBCACHE])
    file_output(CAP_NETWORK_CONFIG, [NETWORKD_DB])
    tree_output(CAP_NETWORK_CONFIG, SYSCONFIG_NETWORK_SCRIPTS, IFCFG_RE)
    tree_output(CAP_NETWORK_CONFIG, SYSCONFIG_NETWORK_SCRIPTS, ROUTE_RE)
    file_output(CAP_NETWORK_CONFIG, [SYSCONFIG_NETWORK, RESOLV_CONF, NSSWITCH_CONF, HOSTS])
    file_output(CAP_NETWORK_CONFIG, [CHRONY_CONF, IPTABLES_CONFIG, HOSTS_ALLOW, HOSTS_DENY])
    file_output(CAP_NETWORK_CONFIG, [OPENVSWITCH_CONF, OPENVSWITCH_CONF_DB])

    cmd_output(CAP_NETWORK_STATUS, [IP, '-s', 'addr'])
    cmd_output(CAP_NETWORK_STATUS, [IP, 'route'])
    cmd_output(CAP_NETWORK_STATUS, [IP, 'neighbour'])
    cmd_output(CAP_NETWORK_STATUS, [SS, '-s'])
    cmd_output(CAP_NETWORK_STATUS, [IP, 'maddr'])
    cmd_output(CAP_NETWORK_STATUS, [NSTAT, '-a'])
    cmd_output(CAP_NETWORK_STATUS, [SS, '-nampio'])
    tree_output(CAP_NETWORK_STATUS, DHCP_LEASE_DIR)
    cmd_output(CAP_NETWORK_STATUS, [ARPTABLES, '-nvL'])
    cmd_output(CAP_NETWORK_STATUS, [EBTABLES, '-L'])
    cmd_output(CAP_NETWORK_STATUS, [IPTABLES, '-nvL'])
    cmd_output(CAP_NETWORK_STATUS, [BRCTL, 'show'])
    cmd_output(CAP_NETWORK_STATUS, [BIOSDEVNAME, '-d', '-x'])
    for p in os.listdir('/sys/class/net/'):
        if os.path.isdir('/sys/class/net/%s/bridge' % p):
            cmd_output(CAP_NETWORK_STATUS, [BRCTL, 'showmacs', p])
        else:
            try:
                f = open('/sys/class/net/%s/type' % p, 'r')
                t = f.readline()
                f.close()
                if int(t) == 1:
                    # ARPHRD_ETHER
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-S', p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-k', p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-i', p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-c', p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-g', p])
                    cmd_output(CAP_NETWORK_STATUS, [ETHTOOL, '-l', p])
            except:
                pass
    tree_output(CAP_NETWORK_STATUS, PROC_NET_BONDING_DIR)
    tree_output(CAP_NETWORK_STATUS, PROC_NET_VLAN_DIR)
    cmd_output(CAP_NETWORK_STATUS, [TC, '-s', 'qdisc'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'activity'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'clients'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'ntpdata'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'rtcdata'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'serverstats'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'smoothing'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'sources', '-v'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'sourcestats', '-v'])
    cmd_output(CAP_NETWORK_STATUS, [CHRONYC, 'tracking'])
    file_output(CAP_NETWORK_STATUS, [PROC_NET_SOFTNET_STAT])
    file_output(CAP_NETWORK_STATUS, [PROC_NET_NETSTAT])
    tree_output(CAP_NETWORK_STATUS, OPENVSWITCH_CORE_DIR)
    if os.path.exists(OPENVSWITCH_VSWITCHD_PID) and CAP_NETWORK_STATUS in entries:
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list', 'open_vswitch'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list', 'bridge'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list', 'port'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list', 'interface'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list-br'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'upcall/show'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'memory/show'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'coverage/show'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'dpif/show'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list', 'controller'])
        for b in br_list():
            cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list-ports', b])
            cmd_output(CAP_NETWORK_STATUS, [OVS_VSCTL, 'list-ifaces', b])
            cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'fdb/show', b])
            cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'mdb/show', b])
            # Assumed br has one-to-one mapping to dp
            cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'dpif/dump-flows', b])
            cmd_output(CAP_NETWORK_STATUS, [OVS_OFCTL, 'show', b])
            cmd_output(CAP_NETWORK_STATUS, [OVS_OFCTL, 'dump-flows', b])
        cmd_output(CAP_NETWORK_STATUS, [OVS_DPCTL, 'show'])
        cmd_output(CAP_NETWORK_STATUS, [OVS_DPCTL, 'show', '-s'])
        for d in dp_list():
            cmd_output(CAP_NETWORK_STATUS, [OVS_DPCTL, 'dump-flows', d])
        cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'bond/list'])
        for b in bond_list():
            cmd_output(CAP_NETWORK_STATUS, [OVS_APPCTL, 'bond/show', b])
    tree_output(CAP_NETWORK_STATUS, SYS_NETBACK_DEBUG)

    cmd_output(CAP_FCOE, [FCOEADM, '-i'])
    cmd_output(CAP_FCOE, [FCOEADM, '-t'])
    tree_output(CAP_FCOE, FCOE_CONFIG_DIR)
    file_output(CAP_FCOE, [FCOE_BLACKLIST_FILE])

    for p in os.listdir('/sys/class/net/'):
        if p.startswith('eth') and p.isalnum():
            cmd_output(CAP_FCOE, [DCBTOOL, 'gc', p, 'dcb'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gc', p, 'pg'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gc', p, 'pfc'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gc', p, 'app:0'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gc', p, 'll:0'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'go', p, 'pg'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'go', p, 'pfc'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'go', p, 'app:0'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'go', p, 'll:0'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gp', p, 'pg'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gp', p, 'pfc'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gp', p, 'app:0'])
            cmd_output(CAP_FCOE, [DCBTOOL, 'gp', p, 'll:0'])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p, '-n'])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p, '-V', 'APP', '-c'])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p, '-V', 'PFC', '-c'])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p, '-V', 'ETS-CFG', '-c'])
            cmd_output(CAP_FCOE, [LLDPTOOL, '-t', '-i', p, '-V', 'ETS-REC', '-c'])

    tree_output(CAP_PAM, PAM_DIR)
    file_output(CAP_PAM, [KRB5_CONF, SSHD_CONFIG])
    tree_output(CAP_PAM, LIKEWISE_DIR)
    file_output(CAP_PAM, [AD_USERS, AD_GROUPS])
    tree_output(CAP_PAM, SAMBA_CONFIG_DIR)
    # tree_output(CAP_PAM, SAMBA_DATA_DIR) # Explictly skip samba data as it contains credentials
    tree_output(CAP_PAM, VAR_LOG_DIR + "samba")

    with log_exceptions():
        dump_xapi_rrds(entries)

    cmd_output(CAP_PROCESS_LIST, [PS, 'wwwaxf', '-eo', 'pid,tty,stat,time,nice,psr,pcpu,pmem,nwchan,wchan:25,args'], label='process-tree')
    func_output(CAP_PROCESS_LIST, 'fd_usage', fd_usage)

    def get_log_range(verbosity):
        return range(1, verbosity) if verbosity < 9 else range(1, 20)

    file_output(CAP_SYSTEM_LOGS,
         [ VAR_LOG_DIR + x for x in
           [ 'crit.log', 'kern.log', 'daemon.log', 'user.log', 'syslog', 'messages',
             'monitor_memory.log', 'secure', 'debug', 'dmesg', 'boot.msg', 'blktap.log',
             'xen-dmesg', 'wtmp', 'xen/hypervisor.log', 'mcelog'] +
           [ f % n for n in get_log_range(caps[CAP_SYSTEM_LOGS][VERBOSITY]) \
                 for f in ['crit.log.%d', 'crit.log.%d.gz',
                           'kern.log.%d', 'kern.log.%d.gz',
                           'daemon.log.%d', 'daemon.log.%d.gz',
                           'user.log.%d', 'user.log.%d.gz',
                           'messages.%d', 'messages.%d.gz',
                           'monitor_memory.log.%d', 'monitor_memory.log.%d.gz',
                           'secure.%d', 'secure.%d.gz',
                           'xen/hypervisor.log.%d', 'xen/hypervisor.log.%d.gz',
                           'blktap.log.%d', 'wtmp.%d.gz']]])
    if not os.path.exists('/var/log/dmesg') and not os.path.exists('/var/log/boot.msg'):
        cmd_output(CAP_SYSTEM_LOGS, [DMESG])
    file_output(CAP_SYSTEM_LOGS, [LWIDENTITY_JOIN_LOG, HOSTS_LWIDENTITY_ORIG])

    cmd_output(CAP_SYSTEM_SERVICES, [SYSTEMCTL, 'status'])

    #file_output(CAP_TAPDISK_LOGS,
    #            glob.glob("/sys/class/blktap2/blktap*/debug"))
    if CAP_TAPDISK_LOGS in entries:
        generate_tapdisk_logs()

    file_output(CAP_VTPM, [SWTPM_IC, SWTPM_RCA, SWTPM_CS])
    tree_output(CAP_XAPI_DEBUG, XAPI_DEBUG_DIR)

    func_output(CAP_XAPI_SUBPROCESS, 'xapi_subprocesses', dump_xapi_subprocess_info)

    tree_output(CAP_XENRT, '/tmp', FIST_RE)

    file_output(CAP_XENRT, ['/var/lib/systemd/coredump/*'])
    tree_output(CAP_XENRT, '/tmp', re.compile(r'^.*xen\.qemu-dm\.'))

    file_output(CAP_XENSERVER_CONFIG, [INITIAL_INVENTORY])
    file_output(CAP_XENSERVER_CONFIG, [POOL_CONF, XAPI_CONF, XAPI_SSL_CONF, XAPI_GLOBS_CONF,
                                       XENSOURCE_INVENTORY, VENDORKERNEL_INVENTORY, XENOPSD_CONF])
    tree_output(CAP_XENSERVER_CONFIG, XAPI_CONF_DIR)
    cmd_output(CAP_XENSERVER_CONFIG, [LS, '-lR', '/opt/xensource'])
    cmd_output(CAP_XENSERVER_CONFIG, [BIN_STATIC_VDIS, 'list'])
    tree_output(CAP_XENSERVER_CONFIG, OEM_CONFIG_DIR, OEM_CONFIG_FILES_RE)
    tree_output(CAP_XENSERVER_CONFIG, STATIC_VDIS)
    cmd_output(CAP_XENSERVER_CONFIG, [LS, '-lR', STATIC_VDIS])
    file_output(CAP_XENSERVER_CONFIG, [SYSCONFIG_CLOCK])
    tree_output(CAP_XENSERVER_CONFIG, SYSTEMD_CONF_DIR)
    file_output(CAP_XENSERVER_CONFIG, [CGRULES_CONF])
    file_output(CAP_XENSERVER_CONFIG, [NRPE_CONF])
    tree_output(CAP_XENSERVER_CONFIG, NRPE_DIR)


    func_output(CAP_XENSERVER_DATABASES, 'xapi-db.xml', dump_filtered_xapi_db)
    func_output(CAP_XENSERVER_DATABASES, 'xapi-clusterd-db', filter_xapi_clusterd_db)
    cmd_output(CAP_XENSERVER_DATABASES, [XENSTORE_LS, '-f'], filter=filter_xenstore_secrets)
    file_output(CAP_XENSERVER_DATABASES, [DB_CONF, DB_CONF_RIO, DB_DEFAULT_FIELDS, DB_SCHEMA_SQL])
    tree_output(CAP_XENSERVER_DATABASES, OEM_CONFIG_DIR, OEM_DB_FILES_RE)
    file_output(CAP_XENSERVER_DATABASES, [XAPI_LOCAL_DB])
    cmd_output(CAP_XENSERVER_DATABASES, [XE, 'pool-dump-database', 'file-name='],
               label="xapi-db-dumped.xml", filter=filter_db_pii)

    cmd_output(CAP_XENSERVER_DOMAINS, [LIST_DOMAINS])
    cmd_output(CAP_XENSERVER_DOMAINS, [XL, "vcpu-list"])

    tree_output(CAP_XENSERVER_INSTALL, VAR_LOG_DIR + 'installer')
    file_output(CAP_XENSERVER_INSTALL,
                [ VAR_LOG_DIR + x for x in
                  [ 'firstboot-SR-commands-log',
                    'upgrade-commands-log', 'generate-iscsi-iqn-log']] +
                [ '/root/' + x for x in
                  [ 'blockdevs-log', 'cmdline-log', 'devcontents-log',
                    'dmesg-log', 'install-log', 'lspci-log', 'modules-log',
                    'pci-log', 'processes-log', 'tty-log', 'uname-log',
                    'vgscan-log']])
    tree_output(CAP_XENSERVER_INSTALL, INSTALLED_REPOS_DIR)
    tree_output(CAP_XENSERVER_INSTALL, UPDATE_APPLIED_DIR)
    func_output(CAP_XENSERVER_CONFIG, 'snmp_xs_conf', filter_snmp_xs_conf)
    func_output(CAP_XENSERVER_CONFIG, 'snmpd_xs_conf', filter_snmpd_xs_conf)
    func_output(CAP_XENSERVER_CONFIG, 'snmpd_conf', filter_snmpd_conf)
    file_output(CAP_XENSERVER_CONFIG, [SYSCONFIG_SNMPD])

    try:
        load_plugins()
    except:
        pass

    xenserver_logs = \
                [ VAR_LOG_DIR + x for x in
                  ['xensource.log', 'audit.log', 'xenstored-access.log',
                   'SMlog', 'isl_trace.log',  'xen/xenstored-trace.log',
                   'xen/xen-hotplug.log', 'xen/domain-builder-ng.log',
                   'squeezed.log', 'openvswitch/ovs-brcompatd.log',
                   'openvswitch/ovs-vswitchd.log', 'openvswitch/ovsdb-server.log',
                   'telemetry/telemetry.log', 'telemetry/license_server.log'] +
                  [ f % n for n in get_log_range(caps[CAP_XENSERVER_LOGS][VERBOSITY]) \
                        for f in ['xensource.log.%d', 'xensource.log.%d.gz',
                                  'SMlog.%d', 'SMlog.%d.gz',
                                  'isl_trace.log.%d', 'isl_trace.log.%d.gz',
                                  'audit.log.%d', 'audit.log.%d.gz',
                                  'xenstored-access.log.%d', 'xenstored-access.log.%d.gz',
                                  'xen/xenstored-access.log.%d', 'xen/xenstored-access.log.%d.gz',
                                  'squeezed.log.%d', 'squeezed.log.%d.gz',
                                  'openvswitch/ovs-brcompatd.log.%d', 'openvswitch/ovs-brcompatd.log.%d.gz',
                                  'openvswitch/ovs-vswitchd.log.%d', 'openvswitch/ovs-vswitchd.log.%d.gz',
                                  'openvswitch/ovsdb-server.log.%d', 'openvswitch/ovsdb-server.log.%d.gz',
                                  'telemetry/telemetry.log.%d', 'telemetry/telemetry.log.%d.gz',
                                  'telemetry/license_server.log.%d', 'telemetry/license_server.log.%d.gz']]]

    # Collect SAR data (binary and text, and add today's report with sar -A)
    cmd_output(CAP_SYSTEM_LOAD, ['sar', '-A'])
    sar_data = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_SYSTEM_LOAD][VERBOSITY])
    update_cap_size(CAP_SYSTEM_LOAD, size_of_all(sar_data))
    file_output(CAP_SYSTEM_LOAD, sar_data)

    qemu_logs = get_recent_logs(glob.glob('/tmp/qemu.[0-9]*'), caps[CAP_XENSERVER_LOGS][VERBOSITY])
    update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + qemu_logs))
    file_output(CAP_XENSERVER_LOGS, xenserver_logs)
    file_output(CAP_XENSERVER_LOGS, qemu_logs)
    tree_output(CAP_XENSERVER_LOGS, OEM_CONFIG_DIR, OEM_XENSERVER_LOGS_RE)

    cmd_output(CAP_XEN_INFO, [XL, 'dmesg'])
    cmd_output(CAP_XEN_INFO, [XL, 'info'])
    cmd_output(CAP_XEN_INFO, [XL, 'info', '-n'])
    cmd_output(CAP_XEN_INFO, [XEN_CPUID, '-v'])
    cmd_output(CAP_XEN_INFO, [XEN_CPUID, '-p'])
    cmd_output(CAP_XEN_INFO, [XEN_LIVEPATCH, 'list'])
    cmd_output(CAP_XEN_INFO, [XEN_MICROCODE, 'show-cpu-info'])
    file_output(CAP_XEN_INFO, [PROC_XEN_BALLOON])

    cmd_output(CAP_XHA_LIVESET, [HA_QUERY_LIVESET])

    file_output(CAP_YUM, [YUM_LOG])
    tree_output(CAP_YUM, YUM_REPOS_DIR)
    cmd_output(CAP_YUM, [RPM, '-qa'])
    tree_output(CAP_YUM, SIGNING_KEY_INFO_DIR)

    # permit the user to filter out data
    for k in sorted(data.keys()):
        if not ANSWER_YES_TO_ALL and not yes("Include '%s'? [Y/n]: " % k):
            del data[k]
    for k in sorted(directory_specifications.keys()):
        if not ANSWER_YES_TO_ALL and not yes("Include tree '%s'? [Y/n]: " % k):
            del directory_specifications[k]

    subdir = os.getenv('XENRT_BUGTOOL_BASENAME')
    if subdir:
        subdir = os.path.basename(subdir)
        if subdir == '..' or subdir == '.':
            subdir = None
    if not subdir:
        subdir = "bug-report-%s" % time.strftime("%Y%m%d%H%M%S")

    # create archive
    if output_fd == -1 and not os.path.exists(BUG_DIR):
        try:
            os.makedirs(BUG_DIR)
        except:
            pass

    if output_fd == -1:
        output_ts('Creating output file')

    if output_type.startswith('tar'):
        archive = TarOutput(subdir, output_type, output_fd)
    else:
        archive = ZipOutput(subdir)
    archive.declare_subarchive(SYSTEMD_CONF_DIR, subdir + SYSTEMD_CONF_DIR + ".tar")

    # collect selected data now
    output_ts('Running commands to collect data')
    collect_data(subdir, archive)

    # after all is done, include all log() entries from the XEN_BUGTOOL_LOG file
    if CAP_XEN_BUGTOOL in entries:
        archive.addRealFile(
            construct_filename(subdir, XEN_BUGTOOL_LOG, {}), XEN_BUGTOOL_LOG
        )

    # include inventory
    include_inventory(archive, subdir)

    if archive.close():
        res = 0
    else:
        res = 1

    clean_tapdisk_logs()

    try:
        os.remove(XEN_BUGTOOL_LOG)
    except:
        pass

    logging.debug("Category sizes (max, actual):\n")
    for key in entries:
        logging.debug("    %s (%d, %d)", key, caps[key][MAX_SIZE], cap_sizes[key])
    return res

def find_tapdisk_logs():
    return glob.glob('/var/log/blktap/*.log*')

def find_tapback_logs():
    return glob.glob('/var/log/tapback/tapback*')

def generate_tapdisk_logs():
    for pid in pidof('tapback'):
        try:
            os.kill(pid, SIGHUP)
        except:
            pass
    for pid in pidof('tapdisk2'):
        try:
            os.kill(pid, SIGUSR1)
            output_ts("Including logs for tapdisk process %d" % pid)
        except :
            pass
    # give processes a second to write their logs
    time.sleep(1)
    file_output(CAP_TAPDISK_LOGS, find_tapdisk_logs() + find_tapback_logs())

def clean_tapdisk_logs():
    for filename in find_tapdisk_logs():
        try:
            os.remove(filename)
        except :
            pass

def dump_xapi_subprocess_info(cap):
    """Check which fds are open by xapi and its subprocesses to diagnose faults like CA-10543.
       Returns a string containing a pretty-printed pstree-like structure. """
    all_pids = [proc_entry for proc_entry in os.listdir("/proc") if proc_entry.isdigit()]
    def readlines(filename):
        lines = ''
        try:
            f = open(filename, "r")
            lines = f.readlines()
            f.close()
        except:
            pass
        return lines
    def cmdline(pid):
        all = readlines("/proc/" + pid + "/cmdline")
        if all == []:
            return ""
        else:
            return all[0].replace('\x00', ' ')
    def parent(pid):
        for i in readlines("/proc/" + pid + "/status"):
            if i.startswith("PPid:"):
                return i.split()[-1]
        return None
    def pstree(pid):
        result = { "cmdline": cmdline(pid) }
        child_pids = [child_pid for child_pid in all_pids if parent(child_pid) == pid]
        children = { }
        for child in child_pids:
            children[child] = pstree(child)
        result['children'] = children
        fds = { }
        for fd in os.listdir("/proc/" + pid + "/fd"):
            try:
                fds[fd] = os.readlink("/proc/" + pid + "/fd/" + fd)
            except:
                pass
        result['fds'] = fds
        return result
    xapis = [pid for pid in all_pids if cmdline(pid).startswith("/opt/xensource/bin/xapi")]
    xapis = [pid for pid in xapis if parent(pid) == "1"]
    result = {}
    for xapi in xapis:
        result[xapi] = pstree(xapi)
    pp = pprint.PrettyPrinter(indent=4)
    return pp.pformat(result)


def dump_xapi_rrds(requested_entries):
    """
    Query xcp-rrdd to get the RRDs for all VMs and the host or pool master.

    Due to triggering memory leaks, it was disabled by default (unless -a is used)
    in 2013, it and was superseded by collecting the compressed rrd files instead.
    It's capability for --entries=persistent-stats is also hidden from the user.
    """
    if CAP_PERSISTENT_STATS not in requested_entries:
        return
    socket.setdefaulttimeout(5)
    session = xapi_local_session()
    session.xenapi.login_with_password('', '', '', 'xenserver-status-report')
    this_host = session.xenapi.session.get_this_host(session._session)
    # better way to find pool master?
    pool = list(session.xenapi.pool.get_all_records().values())[0]
    i_am_master = (this_host == pool['master'])

    for vm in session.xenapi.VM.get_all_records().values():
        # CA-376326: Skip templates and VMs that are not resident on a host:
        if vm['is_a_template'] or vm.get("resident_on") == "OpaqueRef:NULL":
            continue
        if vm['resident_on'] == this_host or (i_am_master and vm['power_state'] in ['Suspended', 'Halted']):
            try:
                rrd = urlopen(
                    "http://localhost/vm_rrd?session_id=%s&uuid=%s"
                    % (session._session, vm["uuid"]),
                    timeout=5,
                )
            except HTTPError as e:
                log("Failed to fetch RRD for VM %s: %s" % (vm["uuid"], e))
                continue
            try:
                name = "xapi_rrd-%s.out" % vm['uuid']
                vm_rrds = rrd.read()

                def create_lambda(x=None):
                    return lambda _: x

                func_output(CAP_PERSISTENT_STATS, name, create_lambda(vm_rrds))
            finally:
                rrd.close()

    try:
        rrd = urlopen("http://localhost/host_rrd?session_id=%s" % session._session)
        output = rrd.read()
        func_output(CAP_PERSISTENT_STATS, 'xapi_rrd-host', lambda _: output)
    finally:
        rrd.close()

    session.xenapi.session.logout()


class XapiDBContentHandler(xml.sax.ContentHandler):
    STRIP_STR = "REMOVED"
    def __init__(self):
        xml.sax.ContentHandler.__init__(self)
        self.table = None
        self.root = None
        self.elements_stack = []

    def _filter_secret_table(self, attrs):
        attrs["value"] = self.STRIP_STR

    def _filter_cluster_table(self, attrs):
        attrs["cluster_token"] = self.STRIP_STR

    def _filter_vm_table(self, attrs):
        # Remove private efi variables
        if "EFI-variables" in attrs["NVRAM"]:
            attrs["NVRAM"] = "(('EFI-variables'%.'{}'))".format(self.STRIP_STR)
        # Remove EFI-variables from snapshot
        metadata = attrs["snapshot_metadata"]
        s = re.sub(r"(?P<start>\'NVRAM\'\%\.\'\((\(\'[^\']+\%\.\'[^.]+\'\)\%\.)*\(\\\'EFI-variables\\\'\%\.\\\')[^\']+(?P<end>\\\')", r"\g<start>REMOVED\g<end>", metadata)
        attrs["snapshot_metadata"] = s

    def _filter(self, attrs):
        table_filters = {
            "secret": self._filter_secret_table,
            "VM": self._filter_vm_table,
            "Cluster": self._filter_cluster_table
        }

        if self.table not in table_filters:
            return

        table_filters[self.table](attrs)

    def startElement(self, name, attrs):
        if name == "table":
            self.table = attrs["name"]

        if name == "row":
            self._filter(attrs._attrs)

        element = Element(name, attrib=attrs._attrs)
        self.elements_stack.append(element)

    def endElement(self, name):
        element = self.elements_stack.pop()
        if self.elements_stack:
            # Last element in the stack is the parent of current one
            self.elements_stack[-1].append(element)
        else:
            self.root = element

        if name == "table":
            self.table = None

    def output(self):
        if self.root is not None:
            return ElementTree.tostring(self.root, encoding="UTF-8")
        return ""


class DBFilter:
    """Filter a Xapi XML database."""
    def __init__(self, raw_xml):
        # remove values for any keys containing the word 'password'
        # example input snippet to filter:
        #   '%.(\'incoming_chappassword\'%.\'TC12818outgoingpasswd\'))" host="host"'
        raw_xml = raw_xml if isinstance(raw_xml, str) else raw_xml.decode()
        nopass_xml = re.sub(r"\('(\w*(?:password)\w*)'%\.'\w*'\)", r"('\1'%.'REMOVED')", raw_xml)

        self.content_handler = XapiDBContentHandler()
        try:
            defusedxml.sax.parseString(no_unicode(nopass_xml), self.content_handler)
        except xml.sax._exceptions.SAXParseException:
            pass  # parse command output line by line, first line just headers

    def output(self):
        return self.content_handler.output()


def filter_db_pii(s, state):
    dbfilter = DBFilter(s)
    return dbfilter.output()

clipboard_match = re.compile(r'^/local/domain/(\d+)/data/((set)|(report))_clipboard')
def filter_xenstore_secrets(s, state):
    match = clipboard_match.search(s.decode())
    if match:
        return '/local/domain/%s/data/%s_clipboard = <filtered for security>\n' % (match.group(1), match.group(2))
    else:
        return s

def filter_xapi_clusterd_db(cap):
    clusterd_data = {}

    if not os.path.exists(XAPI_CLUSTERD):
        return ""

    try:
        with open(XAPI_CLUSTERD, 'r') as f:
            clusterd_data = json.load(f)
    except Exception as e:  # pylint: disable=broad-exception-caught
        return log_internal_error(e)
    try:
        config_keys = ['cluster_config', 'old_cluster_config']
        for config_key in config_keys:
            if config_key in clusterd_data.keys():
                conf = clusterd_data[config_key]
                if "pems" in conf and "blobs" in conf["pems"]:
                    conf["pems"]["blobs"] = "REMOVED"
                if "authkey" in conf:
                    conf["authkey"] = "REMOVED"

        if "token" in clusterd_data:
            clusterd_data["token"] = "REMOVED"
        output_str = json.dumps(clusterd_data)
    except Exception as e:  # pylint: disable=broad-exception-caught
        return log_internal_error(e)

    return output_str

def dump_filtered_xapi_db(cap):
    db_file = None

    # determine DB file name
    c = open(DB_CONF, 'r')
    try:
        for line in c:
            l = line.rstrip('\n')
            if (l[0], l[-1]) == ('[', ']'):
                db_file = l[1:-1]
                break
    finally:
        c.close()

    try:
        with open(db_file, 'r') as file_obj:
            raw_xml = file_obj.read()
            dbfilter = DBFilter(raw_xml)
            return dbfilter.output()
    except Exception as e:
        output_ts("Failed to filter xapi database %s" % (str(e)))
        return ""

def dump_scsi_hosts(cap):
    output = ''
    l = os.listdir('/sys/class/scsi_host')
    l.sort()

    for h in l:
        procname = ''
        try:
            f = open('/sys/class/scsi_host/%s/proc_name' % h)
            procname = f.readline().strip("\n")
            f.close()
        except:
            pass
        modelname = None
        try:
            f = open('/sys/class/scsi_host/%s/model_name' % h)
            modelname = f.readline().strip("\n")
            f.close()
        except:
            pass

        output += "%s:\n" %h
        output += "    %s%s\n" % (procname, modelname and (" -> %s" % modelname) or '')

    return output

def module_info(cap):
    output = io.BytesIO()
    modules = open(PROC_MODULES, 'r')
    procs = []

    for line in modules:
        module = line.split()[0]
        procs.append(ProcOutput([MODINFO, module], caps[cap][MAX_TIME], output))
    modules.close()

    run_procs([procs])

    return output.getvalue()


def multipathd_topology(cap):
    pipe = Popen(
        [MULTIPATHD, "-k"],
        universal_newlines=sys.version_info > (3, 0),
        bufsize=1,
        stdin=PIPE,
        stdout=PIPE,
        stderr=dev_null,
    )
    stdout, _ = pipe.communicate("show topology")

    return stdout

def filter_snmp_xs_conf(_):
    """Filter /etc/snmp/snmp.xs.conf with keys and community removed"""
    return snmp_regex_filter(SNMP_XS_CONF, r'(\"(community|\w*_key)\"\s*:\s*\")\S+(\",*)', r'\1REMOVED\3')

def filter_snmpd_xs_conf(_):
    """Filter /etc/snmp/snmpd.xs.conf with the com2sec community removed"""
    return snmp_regex_filter(SNMPD_XS_CONF, r"(com2sec(\s+\S+){2}\s+)\S+", r"\1REMOVED")

def filter_snmpd_conf(_):
    """Filter /var/lib/net-snmp/snmpd.conf with the usmUser fields authKey and privKey removed"""
    return snmp_regex_filter(SNMPD_CONF, r"(usmUser(\s+\S+){7}\s+)\S+(\s+\S+\s+)\S+(\s+\S+)", r"\1REMOVED\3REMOVED\4")

def snmp_regex_filter(replace_file, regex_str, replace_str):
    try:
        with open(replace_file, "r") as file:
            return re.sub(regex_str, replace_str, file.read())
    except Exception as e:
        return "Failed to filter %s %s" % (replace_file, str(e))


def dp_list():
    output = io.BytesIO()
    procs = [ProcOutput([OVS_DPCTL, 'dump-dps'], caps[CAP_NETWORK_STATUS][MAX_TIME], output)]

    run_procs([procs])

    if not procs[0].timed_out:
        return output.getvalue().decode().splitlines()
    return []

def br_list():
    output = io.BytesIO()
    procs = [ProcOutput([OVS_VSCTL, 'list-br'], caps[CAP_NETWORK_STATUS][MAX_TIME], output)]

    run_procs([procs])

    if not procs[0].timed_out:
        return output.getvalue().decode().splitlines()
    return []

def bond_list():
    output = io.BytesIO()
    procs = [ProcOutput([OVS_APPCTL, 'bond/list'], caps[CAP_NETWORK_STATUS][MAX_TIME], output)]

    run_procs([procs])

    if not procs[0].timed_out:
        bonds = output.getvalue().decode().splitlines()[1:]
        return [x.split('\t')[0] for x in bonds]
    return []

def fd_usage(cap):
    output = ''
    fd_dict = {}
    for d in [p for p in os.listdir('/proc') if p.isdigit()]:
        try:
            with open('/proc/'+d+'/cmdline') as fh:
                name = fh.readline()
                num_fds = len(os.listdir(os.path.join('/proc/'+d+'/fd')))
                if num_fds > 0:
                    if not num_fds in fd_dict:
                        fd_dict[num_fds] = []
                    fd_dict[num_fds].append(name.replace('\0', ' ').strip())
        except:
            output += "Error: Pid %s disappeared\n" % d
    keys = list(fd_dict.keys())
    keys.sort(key=int, reverse=True)
    for k in keys:
        output += "%s: %s\n" % (k, str(fd_dict[k]))
    return output

def load_plugins(just_capabilities = False):
    def getText(nodelist):
        rc = ""
        for node in nodelist:
            if node.nodeType == node.TEXT_NODE:
                rc += node.data
        if sys.version_info > (3, 0):
            return rc
        return rc.encode()

    def getBoolAttr(el, attr, default = False):
        ret = default
        val = el.getAttribute(attr).lower()
        if val in ['true', 'false', 'yes', 'no']:
            ret = val in ['true', 'yes']
        return ret

    for dir in [d for d in os.listdir(PLUGIN_DIR) if os.path.isdir(os.path.join(PLUGIN_DIR, d))]:
        if dir not in caps:
            if not os.path.exists("%s/%s.xml" % (PLUGIN_DIR, dir)):
                continue
            xmldoc = parse("%s/%s.xml" % (PLUGIN_DIR, dir))
            assert xmldoc.documentElement.tagName == "capability"

            pii, min_size, max_size, min_time, max_time, mime = \
                 PII_MAYBE, -1,-1,-1,-1, MIME_TEXT

            if xmldoc.documentElement.getAttribute("pii") in [PII_NO, PII_YES, PII_MAYBE, PII_IF_CUSTOMIZED]:
                pii = xmldoc.documentElement.getAttribute("pii")
            if xmldoc.documentElement.getAttribute("min_size") != '':
                min_size = int(xmldoc.documentElement.getAttribute("min_size"))
            if xmldoc.documentElement.getAttribute("max_size") != '':
                max_size = int(xmldoc.documentElement.getAttribute("max_size"))
            if xmldoc.documentElement.getAttribute("min_time") != '':
                min_time = int(xmldoc.documentElement.getAttribute("min_time"))
            if xmldoc.documentElement.getAttribute("max_time") != '':
                max_time = int(xmldoc.documentElement.getAttribute("max_time"))
            if xmldoc.documentElement.getAttribute("mime") in [MIME_DATA, MIME_TEXT]:
                mime = xmldoc.documentElement.getAttribute("mime")
            checked = getBoolAttr(xmldoc.documentElement, 'checked', True)
            hidden = getBoolAttr(xmldoc.documentElement, 'hidden', False)

            cap(dir, pii, min_size, max_size, min_time, max_time, mime, checked, hidden)

        if just_capabilities:
            continue

        plugdir = os.path.join(PLUGIN_DIR, dir)
        for file in [f for f in os.listdir(plugdir) if f.endswith('.xml')]:
            xmldoc = parse(os.path.join(plugdir, file))
            assert xmldoc.documentElement.tagName == "collect"

            for el in xmldoc.documentElement.getElementsByTagName("*"):
                if el.tagName == "files":
                    file_output(dir, getText(el.childNodes).split())
                elif el.tagName == "list":
                    recursive = getBoolAttr(el, 'recursive')
                    dir_list(dir, getText(el.childNodes).split(), recursive)
                elif el.tagName == "directory":
                    pattern = el.getAttribute("pattern")
                    if pattern == '': pattern = None
                    negate = getBoolAttr(el, 'negate')
                    tree_output(dir, getText(el.childNodes), pattern and re.compile(pattern) or None, negate)
                elif el.tagName == "command":
                    label = el.getAttribute("label")
                    if label == '': label = None
                    cmd_output(dir, getText(el.childNodes), label)

def removeNoError(filename):
    try:
        os.remove(filename)
    except OSError:
        pass

class TarSubArchive(io.BytesIO):
    """Utility class for adding tarfile subarchives to the output archive"""

    def __init__(self, basepath, tar_filename):
        """
        Create the object, defining the base path of the files and the tar file name.
        :param basepath: path below which all files shall be captured into the subarchive
        :param tar_filename: Name of the tar file in the parent archive
        """
        self.basepath = basepath
        self.mtime = time.time()
        self.name = tar_filename
        self.file = tarfile.open(fileobj=self, mode="w|", dereference=True)

    def add_file_with_path(self, name, filename):
        """
        Add a file to the subarchive
        :param name: Recorded path of the the file in the tar archive for extraction
        :param filename: Real file name of the file to be added to the tar archive
        """
        with open(filename, "rb") as buffered_reader:
            self.file.addfile(self.file.gettarinfo(filename, name), buffered_reader)

class ArchiveWithTarSubarchives(object):
    """Base class for TarOutput and ZipOutput with support to create sub-archives"""

    def __init__(self):
        """Initialize the defined sub-archives to be an empty list"""
        self.subarchives = []

    def declare_subarchive(self, basepath, tar_filename):
        """
        Declare a subarchive by defining the base path of the files and the tar file name.
        :param basepath: path below which all files shall be captured into the subarchive
        :param tar_filename: Name of the tar file in the parent archive
        """
        self.subarchives.append(TarSubArchive(basepath, tar_filename))

    def add_path_to_subarchive(self, name, filename):
        """If filename belongs to a subarchive, add the path to it as name and return True"""
        for subarchive in self.subarchives:
            if filename.startswith(subarchive.basepath):
                subarchive.add_file_with_path(name, filename)
                return True
        return False

    def add_subarchives(self):
        """Close all subarchives and add them the final output archive(tar or ZIP file)"""
        for subarchive in self.subarchives:
            if subarchive.file.getmembers():
                subarchive.file.close()
                self.add_path_with_data(subarchive.name, subarchive)
            else:
                subarchive.file.close()

    def add_path_with_data(self, _name, _data):
        """Implemented by the subclasses TarOutput/ZipOutput to add paths with data"""
        pass

class TarOutput(ArchiveWithTarSubarchives):
    def __init__(self, subdir, suffix, output_fd):
        super(TarOutput, self).__init__()
        self.output_fd = output_fd
        self.subdir = subdir
        mode = 'w|'
        if suffix == 'tar.bz2':
            mode = 'w|bz2'
        self.filename = "%s/%s.%s" % (BUG_DIR, subdir, suffix)

        if output_fd == -1:
            self.tf = tarfile.open(self.filename, mode)
        else:
            try:  # Python3.6 does not support "ab" if the file is not seekable:
                binary_fileobj = os.fdopen(output_fd, "ab")
            except OSError:  # pragma: no cover
                binary_fileobj = os.fdopen(output_fd, "wb")
            self.tf = tarfile.open(name=None, mode="w|", fileobj=binary_fileobj)

    def _getTi(self, filename):
        ti = tarfile.TarInfo(filename)
        ti.uname = 'root'
        ti.gname = 'root'
        return ti

    def addRealFile(self, name, filename):
        """Read file contents for adding to the output tar file or a subarchive of it"""
        if self.add_path_to_subarchive(name, filename):
            return
        ti = self._getTi(name)
        s = os.stat(filename)
        ti.mtime = s.st_mtime
        ti.size = s.st_size
        with open(filename, "rb") as buffered_reader:
            self.tf.addfile(ti, buffered_reader)


    def add_path_with_data(self, name, data):  # type:(str, StringIOmtime) -> None
        ti = self._getTi(name)
        ti.mtime = data.mtime
        ti.size = len(data.getvalue())
        data.seek(0)
        self.tf.addfile(ti, data)

    def close(self):
        """Add all subarchives to the output tar file and write it"""
        self.add_subarchives()
        try:
            self.tf.close()
            if self.output_fd == -1:
                output ('Writing tarball %s successful.' % self.filename)
                if SILENT_MODE:
                    print(self.filename)
            return True
        except Exception as e:
            if self.output_fd == -1:
                output ("Error closing tar file '%s': '%s'" % (self.filename, e))
            else:
                output ("Error closing tar file descriptor: '%s'" % e)

            output ("Cleaning up incomplete file '%s'" % self.filename)
            removeNoError(self.filename)
            return False

class ZipOutput(ArchiveWithTarSubarchives):
    def __init__(self, subdir):
        super(ZipOutput, self).__init__()
        self.subdir = subdir
        self.filename = "%s/%s.zip" % (BUG_DIR, subdir)
        self.zf = zipfile.ZipFile(self.filename, 'w', zipfile.ZIP_DEFLATED)

    def addRealFile(self, name, filename):
        """Read file contents for adding to the output ZIP or a subarchive of it"""
        if self.add_path_to_subarchive(name, filename):
            return
        if os.stat(filename).st_size < 50:
            compress_type = zipfile.ZIP_STORED
        else:
            compress_type = zipfile.ZIP_DEFLATED
        self.zf.write(filename, name, compress_type)

    def add_path_with_data(self, name, data):  # type:(str, StringIOmtime) -> None
        self.zf.writestr(name, data.getvalue())

    def close(self):
        """Add all subarchives to the output ZIP file and write it"""
        self.add_subarchives()
        try:
            self.zf.close()
            output ('Writing archive %s successful.' % self.filename)
            if SILENT_MODE:
                print(self.filename)
            return True
        except Exception as e:
            output ("Error closing zip file '%s': %s" % (self.filename, e))
            output ("Cleaning up incomplete file '%s'" % self.filename)
            removeNoError(self.filename)
            return False


def make_inventory(inventory, subdir):
    document = getDOMImplementation().createDocument(
        None, INVENTORY_XML_ROOT, None)

    # create summary entry
    s = document.createElement(INVENTORY_XML_SUMMARY)
    user = os.getenv('SUDO_USER', os.getenv('USER'))
    if user:
        s.setAttribute('user', user)
    s.setAttribute('date', time.strftime('%c'))
    s.setAttribute('hostname', platform.node())
    s.setAttribute('uname', ' '.join(platform.uname()))
    s.setAttribute("uptime", getoutput(UPTIME))
    document.getElementsByTagName(INVENTORY_XML_ROOT)[0].appendChild(s)

    for inventory_key, inventory_item in inventory.items():
        inventory_entry(document, subdir, inventory_key, inventory_item)
    return document.toprettyxml(encoding="utf-8")

def inventory_entry(document, subdir, k, v):
    try:
        el = document.createElement(INVENTORY_XML_ELEMENT)
        el.setAttribute('capability', v['cap'])
        el.setAttribute('filename', construct_filename(subdir, k, v))
        el.setAttribute('md5sum', md5sum(v))
        document.getElementsByTagName(INVENTORY_XML_ROOT)[0].appendChild(el)
    except:
        pass


def md5sum_file(filename):
    m = md5_new()
    f = open(filename, 'rb')
    while True:
        data = f.read(4096)
        if not data:
            break
        m.update(data)
    f.close()
    return m.hexdigest()


def md5sum(d):
    if "md5" in d:
        return d['md5']
    elif "filename" in d:
        return md5sum_file(d['filename'])
    elif "output" in d:
        m = md5_new()
        m.update(d['output'].getvalue())
        return m.hexdigest()
    raise Exception('Cannot compute md5 for this entry')


def construct_filename(subdir, k, v):
    s = v.get('filename')
    if s:
        if s.startswith('/'):
            s = s[1:]
    else:
        s = k.replace(' ', '-')
        s = s.replace('--', '-')
        s = s.replace('/', '%')
        if s.find('.') == -1:
            s += '.out'

    return os.path.join(subdir, s)


def update_capabilities():
    from xen.lowlevel.xc import Error as xcError, xc  # Import on first use.

    update_cap_size(CAP_HOST_CRASHDUMP_LOGS,
                    size_of_dir(HOST_CRASHDUMPS_DIR, HOST_CRASHDUMP_LOGS_EXCLUDES_RE, True))
    update_cap_size(CAP_XAPI_DEBUG, size_of_dir(XAPI_DEBUG_DIR))

    # compute max time & size based on number of PIFs and VIFs
    netdevs = os.listdir('/sys/class/net')
    num_vifs = len([vif for vif in netdevs if vif.startswith('vif')])
    num_pifs = len([eth for eth in netdevs if eth.startswith('eth')])
    max_time = caps[CAP_NETWORK_STATUS][MAX_TIME] * (num_pifs + num_vifs)
    update_cap_time(CAP_NETWORK_STATUS, max_time)
    max_size = caps[CAP_NETWORK_STATUS][MAX_SIZE] * (num_pifs + num_vifs)
    update_cap_size(CAP_NETWORK_STATUS, max_size)
    max_size = caps[CAP_NETWORK_CONFIG][MAX_SIZE] * num_pifs + CAP_NETWORK_CONFIG_OVERHEAD
    update_cap_size(CAP_NETWORK_CONFIG, max_size)

    # update FCOE capabilities based on number of PIFs
    update_cap_time(CAP_FCOE, caps[CAP_FCOE][MAX_TIME] * num_pifs)
    update_cap_size(CAP_FCOE, caps[CAP_FCOE][MAX_SIZE] * num_pifs)

    # compute max time & size based on number of domains, VBDs and VIFs
    num_vbds = 0
    if os.path.exists(BLKTAP_DEVICE_PATH):
        num_vbds = len([vbd for vbd in os.listdir(BLKTAP_DEVICE_PATH) if vbd.startswith('blktap')])
    try:
        num_doms = len(xc().domain_getinfo())
    except xcError:
        num_doms = 0
    max_time = (caps[CAP_XENSERVER_DATABASES][MAX_TIME] * (num_doms + num_vbds + num_vifs) +
                CAP_XENSERVER_DATABASES_TIME_OVERHEAD)
    update_cap_time(CAP_XENSERVER_DATABASES, max_time)
    max_size = (caps[CAP_XENSERVER_DATABASES][MAX_SIZE] * (num_doms + num_vbds + num_vifs) +
                CAP_XENSERVER_DATABASES_SIZE_OVERHEAD)
    update_cap_size(CAP_XENSERVER_DATABASES, max_size)


def update_cap_size(cap, size):
    update_cap(cap, MIN_SIZE, size)
    update_cap(cap, MAX_SIZE, size)


def update_cap_time(cap, time):
    update_cap(cap, MAX_TIME, time)


def update_cap(cap, k, v):
    l = list(caps[cap])
    l[k] = v
    caps[cap] = tuple(l)


def size_of_dir(d, pattern = None, negate = False):
    if os.path.isdir(d):
        return size_of_all([os.path.join(d, fn) for fn in os.listdir(d)],
                           pattern, negate)
    else:
        return 0


def size_of_all(files, pattern = None, negate = False):
    return sum([size_of(f, pattern, negate) for f in files])


def matches(f, pattern, negate):
    if negate:
        return not matches(f, pattern, False)
    else:
        return pattern is None or pattern.match(f)


def size_of(f, pattern, negate):
    if os.path.isfile(f) and matches(f, pattern, negate):
        return os.stat(f)[6]
    else:
        return size_of_dir(f, pattern, negate)


def print_capabilities():
    document = getDOMImplementation().createDocument(
        "ns", CAP_XML_ROOT, None)
    for key in caps:
        if not caps[key][HIDDEN]:
            capability(document, key)
    print(document.toprettyxml())

def capability(document, key):
    c = caps[key]
    el = document.createElement(CAP_XML_ELEMENT)
    el.setAttribute('key', c[KEY])
    el.setAttribute('pii', c[PII])
    el.setAttribute('min-size', str(c[MIN_SIZE]))
    el.setAttribute('max-size', str(c[MAX_SIZE]))
    el.setAttribute('min-time', str(c[MIN_TIME]))
    el.setAttribute('max-time', str(c[MAX_TIME]))
    el.setAttribute('content-type', c[MIME])
    el.setAttribute('default-checked', c[CHECKED] and 'yes' or 'no')
    document.getElementsByTagName(CAP_XML_ROOT)[0].appendChild(el)


def yes(prompt):
    if sys.version_info.major == 2:  # pragma: no cover
        yn = raw_input(prompt)  # pyright: ignore[reportUndefinedVariable]
    else:
        yn = input(prompt)

    return len(yn) == 0 or yn.lower()[0] == 'y'


def mdadm_arrays():
    output = io.BytesIO()
    run_procs([[ProcOutput([MDADM, '--detail', '--scan'],
                           caps[CAP_DISK_INFO][MAX_TIME], output)]])
    # Format along the lines of:
    # ARRAY /dev/device metadata=metadata [name=name] UUID=uuid
    try:
        for line in output.getvalue().decode().split("\n"):
            parts = line.split(" ")

            if len(parts) < 2 or parts[0] != "ARRAY":
                continue

            yield parts[1]
    except:
        pass

partition_re = re.compile(r'(.*[0-9]+$)|(^xvd)')

def disk_list():
    """Return a list of native disks, excluding partitions and virtual disks."""
    disks = []
    try:
        with open(PROC_PARTITIONS, "r") as f:
            f.readline()
            f.readline()
            for line in f:
                major, _, _, name = line.split()
                if int(major) < 254 and not partition_re.match(name):
                    disks.append(name)
    except:
        pass
    return disks


class ProcOutput:
    debug = False

    def __init__(self, command, max_time, inst=None, filter=None):
        self.command = command
        self.max_time = max_time
        self.start_time = None
        self.inst = inst
        self.running = False
        self.status = None
        self.timed_out = False
        self.failed = False
        self.filter = filter
        self.filter_state = {}

    def __del__(self):
        self.terminate()

    def cmdAsStr(self):
        return isinstance(self.command, list) and ' '.join(self.command) or self.command

    def run(self):
        self.timed_out = False
        try:
            if ProcOutput.debug:
                output_ts("Starting '%s'" % self.cmdAsStr())
            self.proc = Popen(
                self.command,
                # Python3 would issue the warning that line buffering
                # is not available in binary mode, remove this later:
                bufsize=1 if sys.version_info < (3, 0) else -1,
                stdin=dev_null,
                stdout=PIPE,
                stderr=dev_null,
                shell=isinstance(self.command, str),
            )
            old = fcntl.fcntl(self.proc.stdout.fileno(), fcntl.F_GETFD)
            fcntl.fcntl(self.proc.stdout.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
            self.running = True
            self.failed = False
        except Exception as e:
            output_ts("'%s' failed: %s" % (self.cmdAsStr(), e))
            self.running = False
            self.failed = True

    def terminate(self):
        if self.running:
            try:
                self.proc.stdout.close()
                os.kill(self.proc.pid, SIGTERM)
            except:
                pass
            self.proc = None
            self.running = False
            self.status = SIGTERM

    def read_line(self):
        assert self.running
        assert self.proc
        assert self.proc.stdout
        line = self.proc.stdout.readline()
        if not line:
            # process exited
            self.proc.stdout.close()
            self.status = self.proc.wait()
            self.proc = None
            self.running = False
        else:
            if self.filter:
                line = self.filter(line, self.filter_state)
            if self.inst:
                self.inst.write(line)

class ProcOutputAndArchive(ProcOutput):
    def __init__(self, command, max_time, name, archive, data):
        self.data = data
        self.name = name
        self.archive = archive
        ProcOutput.__init__(self, command, max_time, data['output'], data['filter'])

    def collectData(self):
        self.archive.add_path_with_data(self.name, self.data['output'])
        self.data['md5'] = md5sum(self.data)
        self.data['output'].close()
        del self.data['output']

    def terminate(self):
        if self.running:
            ProcOutput.terminate(self)
            if not self.running:
                self.collectData()

    def read_line(self):
        if self.running:
            ProcOutput.read_line(self)
            if not self.running:
                self.collectData()

def run_proc_group(pp):
    while True:
        pipes = []
        active_procs = []

        for p in pp:
            if p.running:
                active_procs.append(p)
                pipes.append(p.proc.stdout)
                break
            elif p.status == None and not p.failed and not p.timed_out:
                p.run()
                if p.running:
                    p.start_time = int(time.time())
                    active_procs.append(p)
                    pipes.append(p.proc.stdout)
                    break

        if len(pipes) == 0:
            # all finished
            break

        i, _, _ = select(pipes, [], [], 1.0)
        now = int(time.time())

        # handle process output
        for p in active_procs:
            if p.proc.stdout in i:
                p.read_line()

            # handle timeout
            if not unlimited_time and p.running and now > (p.start_time + p.max_time):
                output_ts("'%s' timed out" % p.cmdAsStr())
                if p.inst:
                    p.inst.write("\n** timeout **\n")
                p.timed_out = True
                p.terminate()

def run_procs(procs):
    for pp in procs:
        run_proc_group(pp)

def pidof(name):
    pids = []

    for d in [p for p in os.listdir('/proc') if p.isdigit()]:
        try:
            if os.path.basename(os.readlink('/proc/%s/exe' % d)) == name:
                pids.append(int(d))
        except:
            pass

    return pids


def readKeyValueFile(filename, allowed_keys = None, strip_quotes = True):
    """ Reads a KEY=Value style file (e.g. xensource-inventory). Returns a
    dictionary of key/values in the file.  Not designed for use with large files
    as the file is read entirely into memory."""

    f = open(filename, "r")
    lines = [x.strip("\n") for x in f.readlines()]
    f.close()

    # remove lines contain
    if allowed_keys:
        lines = [l for l in lines if True in [l.startswith(y) for y in allowed_keys]]

    defs = [ (l[:l.find("=")], l[(l.find("=") + 1):]) for l in lines ]

    if strip_quotes:
        def quotestrip(x):
            return x.strip("'")
        defs = [ (a, quotestrip(b)) for (a,b) in defs ]

    return dict(defs)


if __name__ == "__main__":
    logging.basicConfig(format="%(message)s")
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\nInterrupted.")
        sys.exit(3)
