#!/usr/bin/python3
#
# Copyright (c) 2017-2020 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# either version 2.1 of the License, or (at your option) any
# later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with the product.  If not, see
# <http://www.gnu.org/licenses/>.
#
# Our contact details: Virtuozzo International GmbH, Vordergasse 59, 8200
# Schaffhausen, Switzerland.
#

import glob
import os
import subprocess
import sys
import shutil
import configparser
from xml.dom import minidom

import tarfile
import mmap
import hashlib
import json
import time
import socket
import atexit
import tempfile
import re

try:
    import requests
except:
    import urllib.request

CONFIG="/etc/vz/reporter-vz-prlrep.conf"
LOG_TAIL_LINES = 250

# The compression ratio of gzip varies empirically
# between 5 and 20 for most of ACI logs
# Using the lower compression ratio for estimation
ESTIMATED_MIN_COMPRESS_RATIO = 5

STORAGE_BACKEND_LOGS = [
    '0011_init_coredns.log',
    'backup.log',
    'celery.log',
    'celery-beat.log',
    'celery_periodic.log',
    'celery-bouncer.log',
    'celery_snmp.log',
    'consul.log',
    'ha.log',
    'init_backend.log',
    'init_grafana.log',
    'init_postinstall.log',
    'messages.log',
    'keystone-manage.log',
    'keystone-wsgi-admin.log',
    'keystone-wsgi-public.log',
    'software-updates.log'
]

STORAGE_AGENT_LOGS = [
    'agent-script.log',
    'messages.log',
    'register-agent.log',
    'software-updates.log',
    'oneshot-scripts.log',
    'uwsgi.log'
]

COMMON_LOGS = [
    '/etc/opt/pva/pva-release',
    '/etc/abrt/abrt.conf',
    '/etc/vstorage/abgw.config',
    '/etc/vstorage/cs.config',
    '/etc/vstorage/mds.config',
    '/etc/vstorage/vstorage-mount.conf',
    '/etc/vz/disp_helper.json',
    '/etc/vz/problem_contact.conf',
    '/etc/vz/reporter-vz-prlrep.conf',
    '/etc/vz/vcmmd.conf',
    '/var/log/anaconda/anaconda.log',
    '/var/log/libreport-plugin-problem-report.log',
    '/var/log/openvswitch/ovs-vswitchd.log',
    '/var/log/openvswitch/ovsdb-server.log',
    '/var/log/prepare_vstorage_drive.log',
    '/var/log/vstorage/vstorage-iscsid.log',
    '/var/log/vstorage/vstorage-tcm.log',
    '/var/log/vstorage/vstorage-tcm-monitor.log',
    '/var/log/vstorage/vstorage-target-monitor.log',
    '/var/log/ganesha/ganesha.log',
    '/var/log/vstorage/vstorage-target-manager.log',
    '/var/log/shaman/shaman-monitor.log',
    '/var/log/shaman/shaman.log'
]

XML_LOGS = [
    '/var/log/shaman.log',
    '/var/log/prl-disp.log',
    '/var/log/prl.log',
    '/var/log/yum.log',
    '/var/log/cpufeatures.log',
    '/var/log/ploop.log',
    '/var/log/vcmmd.log',
    '/var/log/messages'
]

def print_log(message, log):
    try:
        logFile = open(log, 'a')
        logFile.write(message + '\n')
        logFile.close()
    except:
        print("Can't write to log file: " + log)
    print(message)

reporterLog = '/var/log/libreport-plugin-problem-report.log'
reportFile = None

dump_dir = os.environ.get('DUMP_DIR')
if not dump_dir or not os.path.isdir(dump_dir):
    print_log("Error: Can't access folder with logs!", reporterLog)
    sys.exit(1)

# python traces
btrace_file = dump_dir + "/backtrace"
cmdline_file = dump_dir + "/cmdline"
messages_file = dump_dir + "/messages"
# binary traces
corebt_file = dump_dir + "/core_backtrace"
vmcore_file = dump_dir + "/vmcore-dmesg.txt"

def get_all_processes(path):
    try:
        p = subprocess.Popen(['ps', 'auxf'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        allProcesses = open(path + '/AllProcesses.txt', 'w')
        allProcesses.write('\n======= ps auxf =======\n')
        while True:
            line = p.stdout.readline()
            if not line:
                break
            line = re.sub("webcp_password=[a-zA-Z0-9_+!@#%&*{};:,./<>?\|\-=]+", "webcp_password=*****", line)
            line = re.sub("password=[a-zA-Z0-9_+!@#%&*{};:,./<>?\|\-=]+", "password=*****", line)
            allProcesses.write(line)
        allProcesses.close()
        return True
    except:
        return False

def get_swan_log(path):
    try:
        out = subprocess.check_output("journalctl -u strongswan.service -n 50000 -o json | jq -r '\
((.__REALTIME_TIMESTAMP | tonumber / 1e6) | strflocaltime(\"%F %T\"))  + \" | \" + \
((.THREAD // \" \") | . + \" \" * (2 - length)) + \" <\" + \
((.IKE_SA_UNIQUE_ID // \"---\") | \" \" * (3 - length) + . ) + \"> \" + \
((.GROUP // \" \") | \"[\" + \" \" * (3 - length) + . + \"] \" ) +\
.MESSAGE'", shell=True)
        with open(path + '/StrongSwan.txt', 'w') as f:
            f.write(out)
        return True
    except:
        return False

def get_license(path):
    try:
        p = subprocess.Popen(['vzlicview'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        allProcesses = open(path + '/vzlicview.txt', 'w')
        allProcesses.write(p.stdout.read())
        allProcesses.close()
        return True
    except:
        return False

def get_storage_license(path):
    if not os.path.isdir("/etc/vstorage/clusters"):
        return False
    clusters = os.listdir("/etc/vstorage/clusters")
    if not clusters:
        return False
    try:
        for cluster_name in clusters:
            p = subprocess.Popen(['vstorage', '-c', cluster_name, 'view-license'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
            allProcesses = open(path + '/vzlic_storage.txt', 'a')
            allProcesses.write(p.stdout.read())
            allProcesses.close()
        return True
    except:
        return False


def collect_mds_log_tail(path):
    if not os.path.isdir("/etc/vstorage/clusters"):
        return None
    clusters = os.listdir("/etc/vstorage/clusters")
    if not clusters:
        return None

    file_path = path + '/' + 'mds_log_tail.txt'

    try:
        for cluster_name in clusters:
            p = subprocess.Popen(['vstorage', '-c', cluster_name, 'mds-log-tail'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
            allProcesses = open(file_path, 'a')
            allProcesses.write(p.stdout.read())
            allProcesses.close()
        return file_path
    except:
        return None


def get_installed_software(path):
    try:
        p = subprocess.Popen(['rpm', '-qa'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        installedSoftware = open(path + '/InstalledSoftware.txt', 'w')
        installedSoftware.write(p.stdout.read())
        installedSoftware.close()
        return True
    except:
        return False

def get_rpm_state(path, pkg_name):
    try:
        p = subprocess.Popen(['rpm', '-V', pkg_name], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        res = open(path + '/rpm_v', 'w')
        res.write(p.stdout.read())
        res.close()
        return True
    except:
        return False

def get_network_cfg(path):
    try:
        p = subprocess.Popen(['ip', 'a'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        network_cfg = open(path + '/network_cfg.txt', 'w')
        network_cfg.write(p.stdout.read())
        network_cfg.close()
        return True
    except:
        return False

def get_log(log, path):
    try:
        shutil.copy(log, path + '/' + os.path.basename(log) + '.txt')
        return True
    except:
        return False

def get_more_host_info(path):
    try:
        p = subprocess.Popen(['uname', '-a'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        moreHostInfo = open(path + '/MoreHostInfo.xml', 'w')
        moreHostInfo.write(p.stdout.read())
        moreHostInfo.close()
        return True
    except:
        return False

def get_mount_info():
    try:
        p = subprocess.Popen(['mount'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding='utf-8')
        mountInfo = p.stdout.read()
        return mountInfo
    except:
        return ''

def get_product_name():
    try:
        if (os.path.isfile('/etc/virtuozzo-release')):
            productName = open('/etc/virtuozzo-release', 'r').read().replace('\n', '')
        elif (os.path.isfile('/etc/hci-release')):
            productName = open('/etc/hci-release', 'r').read().replace('\n', '')
        else:
            productName = open('/etc/system-release', 'r').read().replace('\n', '')
        return productName
    except:
        return ''

def generate_report_xml(path, pkg_name):
    doc = minidom.Document()
    top = doc.createElement('VzProblemReport')
    doc.appendChild(top)
    childType = doc.createElement('Type')
    childTypeText = doc.createTextNode('17')
    childType.appendChild(childTypeText)
    top.appendChild(childType)
    childProductName = doc.createElement('ProductName')
    childProductNameText = doc.createTextNode(get_product_name())
    childProductName.appendChild(childProductNameText)
    top.appendChild(childProductName)
    macs = subprocess.check_output("ip addr | awk '/[a-f\d]{2}:[a-f\d]{2}:[a-f\d]{2}:[a-f\d]{2}:[a-f\d]{2}:[a-f\d]{2}/{print $2}'", shell=True).split()
    if not macs:
        childHwid = doc.createElement('HWID')
        top.appendChild(childHwid)
    else:
        for i, mac in enumerate(macs):
            childHwid = doc.createElement('HWID%d' % i)
            childHwidText = doc.createTextNode(hashlib.sha512((str(mac) + 'aY!wB&42YBpSj5p4').encode('utf-8')).hexdigest()[0:32].upper())
            childHwid.appendChild(childHwidText)
            top.appendChild(childHwid)
    if get_network_cfg(path):
        childNetworkCfg = doc.createElement('NetworkConfiguration')
        childNetworkCfgText = doc.createCDATASection('network_cfg.txt')
        childNetworkCfg.appendChild(childNetworkCfgText)
        top.appendChild(childNetworkCfg)
    if get_log('/proc/meminfo', path):
        childCfg = doc.createElement('MemInfo')
        childCfgText = doc.createCDATASection('meminfo.txt')
        childCfg.appendChild(childCfgText)
        top.appendChild(childCfg)
    if get_installed_software(path):
        childInstalledSoftware = doc.createElement('InstalledSoftware')
        childInstalledSoftwareText = doc.createCDATASection('InstalledSoftware.txt')
        childInstalledSoftware.appendChild(childInstalledSoftwareText)
        top.appendChild(childInstalledSoftware)
    if pkg_name:
        if get_rpm_state(path, pkg_name):
            rpmV = doc.createElement('RpmVerify')
            rpmVText = doc.createCDATASection('rpm_v.txt')
            rpmV.appendChild(rpmVText)
            top.appendChild(rpmV)
    if get_license(path):
        childLic = doc.createElement('License')
        childLicText = doc.createCDATASection('vzlicview.txt')
        childLic.appendChild(childLicText)
        top.appendChild(childLic)
    if get_storage_license(path):
        childLicStor = doc.createElement('StorageLicense')
        childLicStorText = doc.createCDATASection('vzlic_storage.txt')
        childLicStor.appendChild(childLicStorText)
        top.appendChild(childLicStor)
    if get_more_host_info(path):
        childMoreHostInfo = doc.createElement('MoreHostInfo')
        childMoreHostInfoText = doc.createCDATASection('MoreHostInfo.xml')
        childMoreHostInfo.appendChild(childMoreHostInfoText)
        top.appendChild(childMoreHostInfo)
    if get_all_processes(path):
        childAllProcesses = doc.createElement('AllProcesses')
        childAllProcessesText = doc.createCDATASection('AllProcesses.txt')
        childAllProcesses.appendChild(childAllProcessesText)
        top.appendChild(childAllProcesses)
    if get_swan_log(path):
        childStrongSwan = doc.createElement('StrongSwan')
        childStrongSwanText = doc.createCDATASection('StrongSwan.txt')
        childStrongSwan.appendChild(childStrongSwanText)
        top.appendChild(childStrongSwan)
    childMountInfo = doc.createElement('MountInfo')
    childMountInfoText = doc.createCDATASection(get_mount_info())
    childMountInfo.appendChild(childMountInfoText)
    top.appendChild(childMountInfo)
    logsFound = 0
    for log in XML_LOGS:
        if get_log(log, path):
            if logsFound == 0:
                childSystemLogs = doc.createElement('SystemLogs')
                childSystemLogs.setAttribute('id', '0')
                top.appendChild(childSystemLogs)
            childLog = doc.createElement('SystemLog')
            childLog.setAttribute('id', str(logsFound))
            childLog.setAttribute('dyn_lists', '')
            childSystemLogs.appendChild(childLog)
            childLogName = doc.createElement('Name')
            childLogNameText = doc.createTextNode(log)
            childLogName.appendChild(childLogNameText)
            childLog.appendChild(childLogName)
            childLogData = doc.createElement('Data')
            childLogDataText = doc.createCDATASection(os.path.basename(log))
            childLogData.appendChild(childLogDataText)
            childLog.appendChild(childLogData)
            logsFound = logsFound + 1;
            childSystemLogs.setAttribute('dyn_lists', 'SystemLog ' + str(logsFound))
    reportXML = open(path + '/Report.xml', 'w')
    reportXML.write(doc.toprettyxml(indent="  "))
    reportXML.close()

def estimate_compressed_file_size(path):
    if path.endswith(".log"):
        return os.path.getsize(path) / ESTIMATED_MIN_COMPRESS_RATIO
    return os.path.getsize(path)

def estimate_compressed_folder_size(d):
    return sum( estimate_compressed_file_size(os.path.join(dirpath, filename)) for dirpath, dirnames, filenames in os.walk( d ) for filename in filenames )

def get_vz_private_path():
    d = '/vz/private/'
    private_list = []
    # get list from /vz/private
    if os.path.exists(d):
       private_paths = [os.path.join(d, o) for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]
       for pr_dir in private_paths:
           if os.path.exists(pr_dir + '/dump/'):
              private_list.append(pr_dir + '/dump')
    return private_list


def get_max_report_size():
    MaxCrashReportsSize  = 5000
    if not os.path.isfile("/etc/abrt/abrt.conf"):
        return 5000

    with open("/etc/abrt/abrt.conf") as myfile:
        for line in myfile:
            name, var = line.partition("=")[::2]
            if name.startswith('MaxCrashReportsSize'):
                MaxCrashReportsSize = var.strip()
                break
    return MaxCrashReportsSize


def hash_checker(trace_hash):
    headers = {'User-Agent': 'reporter-vz-prlrep'}
    url = "https://report.virtuozzo.com/check_hash.php?backtrace_hash={trace_hash}".format(trace_hash=trace_hash)

    status = None
    page = None

    try:
        resp = requests.get(url, headers=headers)
        status = resp.status_code
        page = resp.content
    except:
        try:
            resp = urllib.request.urlopen(urllib.request.Request(url, headers=headers))
            status = resp.getcode()
            page = resp.read()
            resp.close()
        except:
            return True

    if status == 404:
        print_log('requested page [{}] not found'.format(url), reporterLog)
    elif status == 200:
        server_hash = page.strip()
        if server_hash != b'0':
            print_log(("report number: %s" % server_hash), reporterLog)
            return False

    return True

def service_active(service):
    """Return True if service is running/waiting"""
    cmd = '/bin/systemctl is-active %s' % service
    proc = subprocess.Popen(cmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    proc.communicate()
    if proc.returncode == 0:
       return True
    return False


def format_abrt_stacktrace(data):
    def fmt_frame(arg):
        (n, fr) = arg
        fr = dict(fr)
        fr.setdefault('function_name', '?')
        fr.setdefault('file_name', '?')
        fr.setdefault('address', 0)
        return ('#%d ' % n + "0x%(address)016x %(function_name)s (%(file_name)s)" % fr)

    for thr in data['stacktrace']:
        if 'crash_thread' in thr:
            break
        else:
            return ''
    return '\n'.join(map(fmt_frame, enumerate(thr['frames'])))

'''
Cleanup all temp resources at exit
'''
def _cleanup():
    # Delete temp dump directory if we created it
    if dump_dir and dump_dir != os.environ.get('DUMP_DIR'):
        try:
            os.unlink(dump_dir)
        except:
            print_log("Error: Can't remove temp dump folder " + dump_dir, reporterLog)
            pass
    if reportFile and os.path.exists(reportFile):
        try:
            os.remove(reportFile)
        except:
            print_log("Error: Can't remove temp report file " + reportFile, reporterLog)
            pass

'''
Check if we have caught a problem already known for report server.

If yes, then just exit.

If no - additionally check if this is hw-related problem and report it only for our own hosts.

If we have a hw problem with "MCE" in log but mcelog service is not running then just drop the report,
failed mcelog also triggers abrt crash handler but it is useless.

Return True if we found hw problem, False if not.
'''
def known_issues(dump_dir, component):
    trace_hash = ''
    # need to check fie size for mmap
    if os.path.exists(btrace_file) and os.path.getsize(btrace_file) and os.path.exists(cmdline_file) and os.path.getsize(cmdline_file) > 0:
        f = open(btrace_file)
        backtrace = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
        cmdf = open(cmdline_file)
        cmdline_check = mmap.mmap(cmdf.fileno(), 0, access=mmap.ACCESS_READ)
        if backtrace.find(b'/var/cache/yum') != -1 and cmdline_check.find(b'readykernel') != -1:
            print_log("/var/cache/yum detected in %s and readykernel in %s" % (btrace_file, cmdline_file), reporterLog)
            print_log("let's exit due we know what's wrong here", reporterLog)
            sys.exit(0)
        if backtrace.find(b'yumRepo.py') != -1 and (backtrace.find(b'Cannot retrieve metalink') != -1 or backtrace.find(b'Check uncompressed DB failed') != -1) \
                or "yum" in component and (backtrace.find(b'database is locked') != -1 or backtrace.find(b'decompress:OSError') != -1):
            print_log("Yum problems due to stale metadata, ignoring ....", reporterLog)
            sys.exit(0)
        if backtrace.find(b'vzbuild') != -1 and backtrace.find('uncommitted changes') != -1 or backtrace.find(b'vzbuildError') != -1:
            print_log("vzbuild issue, ignoring ....", reporterLog)
            sys.exit(0)
        if backtrace.find(b'katool') != -1 and backtrace.find(b'credentials not specified') != -1:
            print_log("katool legitimate exception, ignoring ....", reporterLog)
            sys.exit(0)
        if backtrace.find(b'kolla') != -1 and backtrace.find(b'ConnectionError') != -1:
            print_log("kolla known issue, exiting ....", reporterLog)
            sys.exit(0)
        # need to make sure that trace with python there
        if "Python" in dump_dir:
            print_log("python string in backtrace file found, let's check hash", reporterLog)
            trace_hash = hashlib.sha256(btrace_file.encode('utf-8')).hexdigest()
            # false here mean that such HASH already exist
            if hash_checker(trace_hash) is False:
                print_log(("such trace hash: %s already known, skipping" % trace_hash), reporterLog)
                sys.exit(0)
            else:
                print_log(("such trace hash: %s not found in our database" % trace_hash), reporterLog)

    if component.startswith("systemd") and os.path.exists(messages_file) and os.path.getsize(messages_file) > 0:
        print_log(("%s file exist in report, checking ..." % messages_file), reporterLog)
        msgf = open(messages_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        if messages_check.find(b'systemd') != -1 and messages_check.find(b'service watchdog timeout') != -1 \
                or ((messages_check.find(b'systemd-logind') != -1 \
                or messages_check.find(b'systemd-journald') != -1) \
                and (messages_check.find(b'ppoll') != -1) \
                or messages_check.find(b'watchdog') != -1):
            print_log("systemd* service killed by watchdog, skipping", reporterLog)
            sys.exit(0)

    if os.path.exists(corebt_file) and os.path.getsize(corebt_file) > 0:
        print_log("%s file exist in report, checking ..." % corebt_file, reporterLog)
        msgf = open(corebt_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        if messages_check.find(b'bash') != -1 and messages_check.find(b'kill') != -1:
            print_log("bash killed explicitly, skipping", reporterLog)
            sys.exit(0)
        if messages_check.find(b'golang') != -1: # not golang crash
            print_log("golang string in core_backtrace file found, skipping", reporterLog)
        else:
            with open(corebt_file, 'r') as corebt_json:
                data = json.load(corebt_json)
                format_data = format_abrt_stacktrace(data)
                trace_hash = hashlib.sha256(format_data.encode('utf-8')).hexdigest()

            # false here means that such HASH already exist
            if hash_checker(trace_hash) is False:
                print_log(("such trace hash: %s already known, skipping" % trace_hash), reporterLog)
                sys.exit(0)
            else:
                print_log(("such trace hash: %s not found in our database" % trace_hash), reporterLog)

    if os.path.exists(vmcore_file) and os.path.getsize(vmcore_file) > 0:
        print_log(("%s file exist in report, checking ..." % vmcore_file), reporterLog)
        hostname_file = dump_dir + "hostname"
        mhostinfo = dump_dir + "MoreHostInfo.xml"
        host_list = [hostname_file, mhostinfo]
        bug_hosts = ['msk-vpn.virtuozzo.com', 'mwa-fw01.virtuozzo.com', 'corp.acronis.com', 'vzqa.com', '83.223.195.66', 'vz-out.virtuozzo.com']
        msgf = open(vmcore_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        for item in host_list:
            if(os.path.exists(item)):
                host_item = open(item, 'r')
                # on non-qa nodes i see here just my hostname
                bug_host = str(host_item.readline().rstrip()).replace("Linux ", "").split(" ")[0]
                host_item.close()
                if (messages_check.find(b'hardware error') != -1 or messages_check.find(b'Hardware Error') != -1):
                    if bug_host not in bug_hosts or not service_active('mcelog.service'):
                        print_log("HW problem, skipping", reporterLog)
                        if not service_active('mcelog.service'):
                            try:
                                shutil.rmtree(os.environ.get('DUMP_DIR'))
                            except:
                                print_log("Attempted to remove original report folder completely but failed", reporterLog)
                        sys.exit(0)
                    return True

    # Save trace hash into report, if not empty
    if trace_hash:
        with open(dump_dir + "/backtrace_hash", "w") as f:
            f.write(trace_hash)

    return False

'''
Compute free space available for report
'''
def _check_free_space(path):
    f = os.statvfs(os.path.dirname(path))
    return f.f_bfree*f.f_frsize

'''
Get tail of log file
'''
def _get_log_tail(path):
    with open(path) as f:
        total_lines = sum(1 for line in f)
    res = []
    with open(path) as f:
        count = 0
        while True:
            line = f.readline()
            if not line:
                break
            if count >= total_lines - LOG_TAIL_LINES:
                res.append(line)
            count += 1
    return "".join(res)

'''
Add log file to report tarball in a 'safe' mode - by checking that there is
disk enough space for it and we won't exceed MaxCrashReportsSize report size
(but we are still allowed to add very small logs)
'''
def _add_file_safe(tar, src, path_add, file_name = None, tar_filter = None, tail_only = False):
    try:
        if os.path.isdir(src):
            log_size = estimate_compressed_folder_size(src)
        else:
            log_size = estimate_compressed_file_size(src)
        if not file_name:
            file_name = os.path.basename(src)
        size_before = os.path.getsize(reportFile)
        if log_size < _check_free_space(reportFile) and (MaxCrashReportsSize <= 0 or (log_size + size_before) < MaxCrashReportsSize):
            tar.add(src, path_add + file_name, filter = tar_filter)
            log_size_uncompressed = os.path.getsize(src)
            size_after = os.path.getsize(reportFile)
            if os.path.isfile(src) and log_size_uncompressed and size_after > size_before:
                real_ratio = log_size_uncompressed / float(size_after - size_before)
                if real_ratio < ESTIMATED_MIN_COMPRESS_RATIO:
                    print_log("Error: estimated compression ratio is too low for %s (estimated: %s, real: %s)" % (src, ESTIMATED_MIN_COMPRESS_RATIO, real_ratio), reporterLog)
        elif not tail_only and file_name.endswith(".log"):
            # Try to fit tail of the log if we cannot fit the entire log into the report tarball
            src_tail = tempfile.mkstemp(dir=dump_dir)
            with open(src_tail, "w") as f:
                f.write(_get_log_tail(src))
            _add_file_safe(tar, src_tail, path_add, file_name, tar_filter, tail_only = True)
            try:
                os.unlink(src_tail)
            except:
                print_log("Error: Can't remove temp file " + src_tail, reporterLog)
                pass
    except Exception as e:
        print_log("Warning: Can't add log file to archive! " + str(e), reporterLog)
        pass

'''
If hook is interrupted/killed, dangling symlinks can appear.
Just drop them.
'''
def _cleanup_old_runs():
    for f in glob.glob("/var/spool/abrt/*_*"):
        if os.path.islink(f) and not os.path.exists(f):
            os.unlink(f)

def _tar_set_permissions(tarinfo):
    tarinfo.mode = 0o644
    return tarinfo

def get_hostname():
    try:
        return socket.gethostname()
    except:
        return 'unknown'

if __name__ == "__main__":
    atexit.register(_cleanup)
    _cleanup_old_runs()

    # If we have a config file then try to get folder for the report from there
    if os.path.isfile(CONFIG):
        config = configparser.ConfigParser()
        config.read(CONFIG)
        try:
            # do not collect data if crashReportEnabled = false
            report_disabled = config['default']['crashReportEnabled']
            if report_disabled.lower() == 'false':
               sys.exit(0)
            reportFolder = config['default']['reportFolder']
            # DEFAULT means that we should guess path by ourselves, will do this below
            if reportFolder == 'DEFAULT':
                raise
            if not os.path.exists(reportFolder):
                os.makedirs(reportFolder)
            # We expect a folder
            if not os.path.isdir(reportFolder):
                raise
            # Just for beauty, drop doubled slashes from the path
            reportFile = (reportFolder + "/report.tar.gz").replace('//', '/')
        except Exception:
            reportFile = None
            pass

    if not reportFile:
        if os.path.isdir('/vz/tmp'):
            reportFile = '/vz/tmp/report.tar.gz'
        else:
            reportFile = '/var/tmp/report.tar.gz'

    # Use MaxCrashReportsSize from abrt.conf to limit size of added logs, as well
    try:
        # the value in abrt.conf is in MiB
        MaxCrashReportsSize = int(get_max_report_size()) * 1048576
    except:
        MaxCrashReportsSize = -1

    # We need some space to create tarball with report
    # Can't predict how many, but not more than MaxCrashReportsSize (if set)
    if MaxCrashReportsSize == -1:
        FreeSpaceNeeded = os.path.getsize(dump_dir)
    else:
        FreeSpaceNeeded = MaxCrashReportsSize

    if _check_free_space(reportFile) < FreeSpaceNeeded:
        print_log("Low free space, will not prepare and send report tarball", reporterLog)
        sys.exit(2)

    # Virtuozzo parser cannot handle dirs with ":" so rename them
    if ':' in dump_dir:
        tempDir = dump_dir.replace(':', "_")
        try:
            if os.path.islink(tempDir):
                os.unlink(tempDir)
            elif os.path.isdir(tempDir):
                shutil.rmtree(tempDir)
            elif os.path.isfile(tempDir):
                os.remove(tempDir)
            os.symlink(dump_dir, tempDir)
        except Exception as e:
            print_log("Error: Can't create report folder! " + str(e), reporterLog)
            sys.exit(1)
        dump_dir = tempDir
    path_dump = os.path.basename(dump_dir)
    component = ''
    try:
        component = open(dump_dir + '/component', 'r').read()
    except:
        print_log("Error: 'component' file should be present in log - can't send a report without it!", reporterLog)
        sys.exit(1)
    reason = ''
    try:
        reason = open(dump_dir + '/reason', 'r').read()
    except:
        print_log("Warning: There's no 'reason' file in log", reporterLog)

    pkg_name = ''
    try:
        pkg_name = open(dump_dir + '/pkg_name', 'r').read()
    except:
        pass

    try:
        generate_report_xml(dump_dir, pkg_name)
    except:
        # This atually means that we will not be able to parse report...
        print_log("Failed to dump Report.xml", reporterLog)

    try:
        cmdline = open(dump_dir + '/cmdline', 'r').read()
    except:
        cmdline = ''

    get_log('/proc/net/snmp', dump_dir)
    get_log('/proc/net/netstat', dump_dir)
    get_log('/sys/kernel/debug/cleancache/succ_gets', dump_dir)

    # VSTOR-47406: The coredump is truncated to MaxCrashReportsSize [MiB], no way to reuse it for debugging
    coredump = dump_dir + '/coredump'
    try:
        coredump_size = os.path.getsize(coredump)
        if coredump_size >= MaxCrashReportsSize:
            print_log("We are not going to collect coredump of size " + str(coredump_size/1048576) + " MiB", reporterLog)
            with open(coredump, "w") as f:
                f.write("This coredump is larger than limit MaxCrashReportsSize and therefore it has not been collected")
    except OSError:
        print_log("Coredump does not exist or is inaccessible.", reporterLog)

    hw_problem = known_issues(dump_dir, component)
    # Add some more files to standard abrt report
    try:
        tar = tarfile.open(reportFile, "w:gz", dereference=True)

        storage_related = ("storage" in component or "nfs" in component or "archive" in component)
        if storage_related and not hw_problem:
            collect_mds_log_tail(dump_dir)

        tar.add(dump_dir, path_dump)

        for log in COMMON_LOGS:
            if os.path.isfile(log):
                _add_file_safe(tar, log, path_dump + "/")

        # check that list not empty
        if get_vz_private_path():
           for dump_log in get_vz_private_path():
               # add dump with ct_id name
               # tmp/criu_dumps/24e8bf17-a7c6-42a3-9620-1de9e2154038
               _add_file_safe(tar, dump_log, path_dump + "/criu_dumps/", file_name = dump_log.split('/')[3])

        libvirt_logs = "/var/log/libvirt"
        if os.path.isdir(libvirt_logs) and not hw_problem:
            _add_file_safe(tar, libvirt_logs, path_dump + "/", file_name = "libvirt-logs")

        if storage_related and not hw_problem:
            for f in glob.glob("/var/log/ostor/*.log") + glob.glob("/var/log/ostor/*.log.zst") + glob.glob("/var/log/ostor/*.blog") + glob.glob("/var/log/ostor/*.log.gz"):
                _add_file_safe(tar, f, path_dump + "/ostor-logs/" + os.path.dirname(f).replace("/", "_")  + "_")
            for f in glob.glob("/vstorage/**/mds/logs/*.log") + glob.glob("/vstorage/**/mds/logs/*.log.zst") + glob.glob("/vstorage/**/mds/logs/*.blog") + glob.glob("/vstorage/mds/logs/*.log") + glob.glob("/vstorage/mds/logs/*.log.zst") + glob.glob("/vstorage/mds/logs/*.blog"):
                _add_file_safe(tar, f, path_dump + "/mds-logs/" + os.path.dirname(f).replace("/", "_")  + "_")
            for f in glob.glob("/vstorage/**/cs/logs/*.log") + glob.glob("/vstorage/**/cs/logs/*.log.zst") + glob.glob("/vstorage/**/cs/logs/*.blog"):
                _add_file_safe(tar, f, path_dump + "/cs-logs/" + os.path.dirname(f).replace("/", "_")  + "_")
            for f in glob.glob("/var/log/vstorage/**/vstorage-mount.log.zst") + glob.glob("/var/log/vstorage/**/vstorage-mount.blog"):
                _add_file_safe(tar, f, path_dump + "/vstorage-mount-logs/" + os.path.dirname(f).replace("/", "_")  + "_")
            for f in glob.glob("/var/log/abgw/abgw.log.zst") + glob.glob("/var/log/abgw/abgw.blog"):
                _add_file_safe(tar, f, path_dump + "/abgw-logs/")
            for f in glob.glob("/var/lib/pgsql/data/pg_log/*log"):
                _add_file_safe(tar, f, path_dump + "/pg-logs/")

            iscsi_logs = "/var/log/vstorage/iscsi"
            if os.path.isdir(iscsi_logs):
                _add_file_safe(tar, iscsi_logs, path_dump + "/", file_name = "iscsi-logs", tar_filter = _tar_set_permissions)

            if os.path.isdir("/var/log/vstorage-ui-agent"):
                for f in STORAGE_AGENT_LOGS:
                    full_path = "/var/log/vstorage-ui-agent/" + f
                    if os.path.isfile(full_path):
                        _add_file_safe(tar, full_path, path_dump + "/vstorage-ui-agent-logs/")

            if os.path.isdir("/var/log/vstorage-ui-backend"):
                for f in STORAGE_BACKEND_LOGS:
                    full_path = "/var/log/vstorage-ui-backend/" + f
                    if os.path.isfile(full_path):
                        _add_file_safe(tar, full_path, path_dump + "/vstorage-ui-backend-logs/")

        if "archive" in cmdline and not hw_problem:
            m = re.findall('-L (\S+)', cmdline)
            for journal in m:
                if os.path.isfile(journal):
                    _add_file_safe(tar, journal, path_dump + "/")

        tar.close()

    except Exception as e:
        print_log("Error: Can't create archive with logs! " + str(e), reporterLog)
        if os.path.exists(reportFile):
            os.remove(reportFile)
        sys.exit(1)

    if os.path.exists(btrace_file) and os.path.getsize(btrace_file) > 0:
        trace_hash = hashlib.sha256(btrace_file.encode('utf-8')).hexdigest()
    elif os.path.exists(corebt_file) and os.path.getsize(corebt_file) > 0:
        with open(corebt_file, 'r') as corebt_json:
            data = json.load(corebt_json)
            format_data = format_abrt_stacktrace(data)
            trace_hash = hashlib.sha256(format_data.encode('utf-8')).hexdigest()
    else:
       trace_hash = ''

    time.sleep(1) # too frequent requests are not allowed by the server

    try:
        os.environ['LD_LIBRARY_PATH'] = "/usr/libexec/openssl-fips/lib64"
        cmd_upload = ['curl', '-k']
        if trace_hash:
            cmd_upload.extend(['-F', 'backtrace_hash=' + trace_hash])
        cmd_upload.extend(['-D-', '-F', 'name=@' + reportFile, '-F', 'pkg="' + component + '"', '-F', 'desc="' + reason + '"', '-F', 'hostname="' + get_hostname() + '"', "https://report.virtuozzo.com/anacrash"])
        print_log("Upload command line: " + ' '.join(cmd_upload), reporterLog)
        p = subprocess.Popen(cmd_upload, env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
        curlReply = str(p.stdout.read())
        if 'status=OK' in curlReply:
            print_log("The problem report was successfully sent with id: " + re.sub('[^0-9]', '', curlReply.split(';')[-1]), reporterLog)
        else:
            print_log("Error: Can't upload the problem report: \n" + curlReply, reporterLog)
    except Exception as e:
        print_log("Error: Can't send report: " + str(e), reporterLog)
