#!/usr/bin/python
#
# Copyright (c) 2017-2019 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# either version 2.1 of the License, or (at your option) any
# later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with the product.  If not, see
# <http://www.gnu.org/licenses/>.
#
# Our contact details: Virtuozzo International GmbH, Vordergasse 59, 8200
# Schaffhausen, Switzerland.
#

import glob
import os
import subprocess
import sys
import shutil
import statvfs
from xml.dom import minidom

import tarfile
import mmap
import hashlib
import requests
import json
import time

def print_log(message, log):
    try:
        logFile = open(log, 'a')
        logFile.write(message + '\n')
        logFile.close()
    except:
        print("Can't write to log file: " + log)
    print(message)

# Logs smaller then this size are added even if total size of report
# will exceed the limit. Useful for config files.
MIN_SIZE = 20000
reporterLog = '/var/log/libreport-plugin-problem-report.log'

dump_dir = os.environ.get('DUMP_DIR')
if not dump_dir or not os.path.isdir(dump_dir):
    print_log("Error: Can't access folder with logs!", reporterLog)
    sys.exit(1)

# python traces
btrace_file = dump_dir + "/backtrace"
cmdline_file = dump_dir + "/cmdline"
messages_file = dump_dir + "/messages"
# binary traces
corebt_file = dump_dir + "/core_backtrace"
vmcore_file = dump_dir + "/vmcore-dmesg.txt"

def get_all_processes(path):
    try:
        p = subprocess.Popen(['ps', 'auxf'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        allProcesses = open(path + '/AllProcesses.txt', 'w')
        allProcesses.write('\n======= ps auxf =======\n')
        allProcesses.write(p.stdout.read())
        allProcesses.close()
        return True
    except:
        return False

def get_license(path):
    try:
        p = subprocess.Popen(['vzlicview'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        allProcesses = open(path + '/vzlicview.txt', 'w')
        allProcesses.write(p.stdout.read())
        allProcesses.close()
        return True
    except:
        return False

def get_storage_license(path):
    if not os.path.isdir("/etc/vstorage/clusters"):
        return False
    clusters = os.listdir("/etc/vstorage/clusters")
    if not clusters:
        return False
    try:
        for cluster_name in clusters:
            p = subprocess.Popen(['vstorage', '-c', cluster_name, 'view-license'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
            allProcesses = open(path + '/vzlic_storage.txt', 'a')
            allProcesses.write(p.stdout.read())
            allProcesses.close()
        return True
    except:
        return False


def get_fail_mds_logs(path):
    if not os.path.isdir("/etc/vstorage/clusters"):
        return False
    clusters = os.listdir("/etc/vstorage/clusters")
    if not clusters:
        return False
    # make sure that dir not exist
    mds_logdir = '/vstorage/mds/logs/'
    if os.path.exists(mds_logdir) and os.path.isdir(mds_logdir):
        return False
    try:
        for cluster_name in clusters:
            p = subprocess.Popen(['vstorage', '-c', cluster_name, 'mds-log-tail'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
            allProcesses = open(path + '/fail_mds_logs.txt', 'a')
            allProcesses.write(p.stdout.read())
            allProcesses.close()
        return True
    except:
        return False


def get_installed_software(path):
    try:
        p = subprocess.Popen(['rpm', '-qa'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        installedSoftware = open(path + '/InstalledSoftware.txt', 'w')
        installedSoftware.write(p.stdout.read())
        installedSoftware.close()
        return True
    except:
        return False

def get_meminfo(path):
    try:
        p = subprocess.Popen(['cat', '/proc/meminfo'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        cfg = open(path + '/meminfo.txt', 'w')
        cfg.write(p.stdout.read())
        cfg.close()
        return True
    except:
        return False

def get_network_cfg(path):
    try:
        p = subprocess.Popen(['ip', 'a'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        network_cfg = open(path + '/network_cfg.txt', 'w')
        network_cfg.write(p.stdout.read())
        network_cfg.close()
        return True
    except:
        return False

def get_log(log, path):
    try:
        shutil.copy(log, path)
        return True
    except:
        return False

def get_more_host_info(path):
    try:
        p = subprocess.Popen(['uname', '-a'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        moreHostInfo = open(path + '/MoreHostInfo.xml', 'w')
        moreHostInfo.write(p.stdout.read())
        moreHostInfo.close()
        return True
    except:
        return False

def get_mount_info():
    try:
        p = subprocess.Popen(['mount'], env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        mountInfo = p.stdout.read()
        return mountInfo
    except:
        return ''

def get_product_name():
    try:
        if (os.path.isfile('/etc/virtuozzo-release')):
            productName = open('/etc/virtuozzo-release', 'r').read().replace('\n', '')
        elif (os.path.isfile('/etc/hci-release')):
            productName = open('/etc/hci-release', 'r').read().replace('\n', '')
        else:
            productName = open('/etc/system-release', 'r').read().replace('\n', '')
        return productName
    except:
        return ''

def generate_report_xml(path):
    doc = minidom.Document()
    top = doc.createElement('ParallelsProblemReport')
    doc.appendChild(top)
    childType = doc.createElement('Type')
    childTypeText = doc.createTextNode('17')
    childType.appendChild(childTypeText)
    top.appendChild(childType)
    childProductName = doc.createElement('ProductName')
    childProductNameText = doc.createTextNode(get_product_name())
    childProductName.appendChild(childProductNameText)
    top.appendChild(childProductName)
    if get_network_cfg(path):
        childNetworkCfg = doc.createElement('NetworkConfiguration')
        childNetworkCfgText = doc.createCDATASection('network_cfg.txt')
        childNetworkCfg.appendChild(childNetworkCfgText)
        top.appendChild(childNetworkCfg)
    if get_meminfo(path):
        childCfg = doc.createElement('MemInfo')
        childCfgText = doc.createCDATASection('meminfo.txt')
        childCfg.appendChild(childCfgText)
        top.appendChild(childCfg)
    if get_installed_software(path):
        childInstalledSoftware = doc.createElement('InstalledSoftware')
        childInstalledSoftwareText = doc.createCDATASection('InstalledSoftware.txt')
        childInstalledSoftware.appendChild(childInstalledSoftwareText)
        top.appendChild(childInstalledSoftware)
    if get_license(path):
        childLic = doc.createElement('License')
        childLicText = doc.createCDATASection('vzlicview.txt')
        childLic.appendChild(childLicText)
        top.appendChild(childLic)
    if get_storage_license(path):
        childLicStor = doc.createElement('StorageLicense')
        childLicStorText = doc.createCDATASection('vzlic_storage.txt')
        childLicStor.appendChild(childLicStorText)
        top.appendChild(childLicStor)
    if get_more_host_info(path):
        childMoreHostInfo = doc.createElement('MoreHostInfo')
        childMoreHostInfoText = doc.createCDATASection('MoreHostInfo.xml')
        childMoreHostInfo.appendChild(childMoreHostInfoText)
        top.appendChild(childMoreHostInfo)
    if get_all_processes(path):
        childAllProcesses = doc.createElement('AllProcesses')
        childAllProcessesText = doc.createCDATASection('AllProcesses.txt')
        childAllProcesses.appendChild(childAllProcessesText)
        top.appendChild(childAllProcesses)
    childMountInfo = doc.createElement('MountInfo')
    childMountInfoText = doc.createCDATASection(get_mount_info())
    childMountInfo.appendChild(childMountInfoText)
    top.appendChild(childMountInfo)
    logsFound = 0
    logs = ['/var/log/shaman.log', '/var/log/prl-disp.log', '/var/log/prl.log', '/var/log/yum.log', '/var/log/cpufeatures.log', '/var/log/ploop.log', '/var/log/vcmmd.log', '/var/log/messages']
    for log in logs:
        if get_log(log, path):
            if logsFound == 0:
                childSystemLogs = doc.createElement('SystemLogs')
                childSystemLogs.setAttribute('id', '0')
                top.appendChild(childSystemLogs)
            childLog = doc.createElement('SystemLog')
            childLog.setAttribute('id', str(logsFound))
            childLog.setAttribute('dyn_lists', '')
            childSystemLogs.appendChild(childLog)
            childLogName = doc.createElement('Name')
            childLogNameText = doc.createTextNode(log)
            childLogName.appendChild(childLogNameText)
            childLog.appendChild(childLogName)
            childLogData = doc.createElement('Data')
            childLogDataText = doc.createCDATASection(os.path.basename(log))
            childLogData.appendChild(childLogDataText)
            childLog.appendChild(childLogData)
            logsFound = logsFound + 1;
            childSystemLogs.setAttribute('dyn_lists', 'SystemLog ' + str(logsFound))
    reportXML = open(path + '/Report.xml', 'w')
    reportXML.write(doc.toprettyxml(indent="  "))
    reportXML.close()

def get_folder_size(d):
    return sum( os.path.getsize(os.path.join(dirpath,filename)) for dirpath, dirnames, filenames in os.walk( d ) for filename in filenames )


def get_max_report_size():
    MaxCrashReportsSize  = -1
    if not os.path.isfile("/etc/abrt/abrt.conf"):
        return -1

    with open("/etc/abrt/abrt.conf") as myfile:
        for line in myfile:
            name, var = line.partition("=")[::2]
            if name.startswith('MaxCrashReportsSize'):
                MaxCrashReportsSize = var.strip()
                break
    return MaxCrashReportsSize


def hash_checker(trace_hash):
    headers = {'User-Agent': 'reporter-vz-prlrep'}
    url = "https://report.virtuozzo.com/check_hash.php?backtrace_hash={trace_hash}".format(trace_hash=trace_hash)
    try:
        resp = requests.get(url, headers=headers)
        if resp.status_code == 404:
            print_log('requested page [{}] not found'.format(url), reporterLog)
            return True
        if resp.status_code == 200:
            page = resp.content
            server_hash = page.strip()
            if server_hash != '0':
               print_log(("report number: %s" % server_hash), reporterLog)
               return False
            else:
               return True
    except:
        return True


def service_active(service):
    """Return True if service is running/waiting"""
    cmd = '/bin/systemctl is-active %s' % service
    proc = subprocess.Popen(cmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    proc.communicate()
    if proc.returncode == 0:
       return True
    return False


def format_abrt_stacktrace(data):
    def fmt_frame(arg):
        (n, fr) = arg
        fr = dict(fr)
        fr.setdefault('function_name', '?')
        fr.setdefault('file_name', '?')
        fr.setdefault('address', 0)
        return ('#%d ' % n + "0x%(address)016x %(function_name)s (%(file_name)s)" % fr)

    for thr in data['stacktrace']:
        if 'crash_thread' in thr:
            break
        else:
            return ''
    return '\n'.join(map(fmt_frame, enumerate(thr['frames'])))

def _cleanup(dump_dir):
    if dump_dir != os.environ.get('DUMP_DIR'):
        try:
            os.unlink(dump_dir)
        except:
            print_log("Error: Can't remove temp folder " + dump_dir, reporterLog)
            pass

'''
Check if we have caught a problem already known for report server.

If yes, then just exit.

If no - additionally check if this is hw-related problem and report it only for our own hosts.

If we have a hw problem with "MCE" in log but mcelog service is not running then just drop the report,
failed mcelog also triggers abrt crash handler but it is useless.

Return True if we found hw problem, False if not.
'''
def known_issues(dump_dir):
    trace_hash = ''
    # need to check fie size for mmap
    if os.path.exists(btrace_file) and os.path.getsize(btrace_file) and os.path.exists(cmdline_file) and os.path.getsize(cmdline_file) > 0:
        f = open(btrace_file)
        backtrace = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
        cmdf = open(cmdline_file)
        cmdline_check = mmap.mmap(cmdf.fileno(), 0, access=mmap.ACCESS_READ)
        if backtrace.find('/var/cache/yum') != -1 and cmdline_check.find("readykernel") != -1:
            print_log("/var/cache/yum detected in %s and readykernel in %s" % (btrace_file, cmdline_file), reporterLog)
            print_log("let's exit due we know what's wrong here", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)
        if backtrace.find('vzbuild') != -1 and backtrace.find('uncommitted changes') != -1 or backtrace.find('vzbuildError') != -1:
            print_log("vzbuild issue, ignoring ....", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)
        if backtrace.find('katool') != -1 and backtrace.find('credentials not specified') != -1:
            print_log("katool legitimate exception, ignoring ....", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)
        if backtrace.find('kolla') != -1 and backtrace.find('ConnectionError') != -1:
            print_log("kolla known issue, exiting ....", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)
        # need to make sure that trace with python there
        if "Python" in dump_dir:
            print_log("python string in backtrace file found, let's check hash", reporterLog)
            trace_hash = hashlib.sha256(btrace_file).hexdigest()
            # false here mean that such HASH already exist
            if hash_checker(trace_hash) is False:
                print_log(("such trace hash: %s already known, skipping" % trace_hash), reporterLog)
                _cleanup(dump_dir)
                sys.exit(0)
            else:
                print_log(("such trace hash: %s not found in our database" % trace_hash), reporterLog)

    component = open(dump_dir + '/component', 'r').read()
    if component.startswith("systemd") and os.path.exists(messages_file) and os.path.getsize(messages_file) > 0:
        print_log(("%s file exist in report, checking ..." % messages_file), reporterLog)
        msgf = open(messages_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        if messages_check.find('systemd') != -1 and messages_check.find('service watchdog timeout') != -1 \
                or ((messages_check.find('systemd-logind') != -1 \
                or messages_check.find('systemd-journald') != -1) \
                and (messages_check.find('ppoll') != -1) \
                or messages_check.find('watchdog') != -1):
            print_log("systemd* service killed by watchdog, skipping", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)

    if os.path.exists(corebt_file) and os.path.getsize(corebt_file) > 0:
        print("%s file exist in report, checking ..." % corebt_file)
        msgf = open(corebt_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        if messages_check.find('bash') != -1 and messages_check.find('kill') != -1:
            print_log("bash killed explicitly, skipping", reporterLog)
            _cleanup(dump_dir)
            sys.exit(0)
        if messages_check.find('golang') != -1: # not golang crash
            print_log("golang string in core_backtrace file found, skipping", reporterLog)
        else:
            with open(corebt_file, 'r') as corebt_json:
                data = json.load(corebt_json)
                format_data = format_abrt_stacktrace(data)
                trace_hash = hashlib.sha256(format_data).hexdigest()

            # false here means that such HASH already exist
            if hash_checker(trace_hash) is False:
                print_log(("such trace hash: %s already known, skipping" % trace_hash), reporterLog)
                _cleanup(dump_dir)
                sys.exit(0)
            else:
                print_log(("such trace hash: %s not found in our database" % trace_hash), reporterLog)

    if os.path.exists(vmcore_file) and os.path.getsize(vmcore_file) > 0:
        print_log(("%s file exist in report, checking ..." % vmcore_file), reporterLog)
        hostname_file = dump_dir + "hostname"
        mhostinfo = dump_dir + "MoreHostInfo.xml"
        host_list = [hostname_file, mhostinfo]
        bug_hosts = ['msk-vpn.virtuozzo.com', 'mwa-fw01.virtuozzo.com', 'corp.acronis.com', 'vzqa.com', '83.223.195.66', 'vz-out.virtuozzo.com']
        msgf = open(vmcore_file)
        messages_check = mmap.mmap(msgf.fileno(), 0, access=mmap.ACCESS_READ)
        for item in host_list:
            if(os.path.exists(item)):
                host_item = open(item, 'r')
                # on non-qa nodes i see here just my hostname
                bug_host = str(host_item.readline().rstrip()).replace("Linux ", "").split(" ")[0]
                host_item.close()
                if (messages_check.find('hardware error') != -1 or messages_check.find('Hardware Error') != -1):
                    if bug_host not in bug_hosts or not service_active('mcelog.service'):
                        print_log("HW problem, skipping", reporterLog)
                        _cleanup(dump_dir)
                        if not service_active('mcelog.service'):
                            try:
                                shutil.rmtree(os.environ.get('DUMP_DIR'))
                            except:
                                print_log("Attempted to remove original report folder completely but failed", reporterLog)
                        sys.exit(0)
                    return True

    # Save trace hash into report, if not empty
    if trace_hash:
        with open(dump_dir + "/backtrace_hash", "w") as f:
            f.write(trace_hash)
    
    return False

'''
Add log file to report tarball in a 'safe' mode - by checking that there is
disk enough space for it and we won't exceed MaxCrashReportsSize report size
(but we are still allowed to add very small logs)
'''
def _add_file_safe(tar, f, path_add, total_size):
    try:
        log_size = os.path.getsize(f)
        if (MaxCrashReportsSize > 0 and (log_size < MIN_SIZE or (log_size + total_size) < MaxCrashReportsSize)) \
                or ((log_size + total_size) < free_space):
                tar.add(f, path_add + os.path.basename(f))
                total_size += log_size
    except Exception as e:
        print_log("Warning: Can't add log file to archive! " + str(e), reporterLog)
        pass

    return total_size

'''
If hook is interrupted/killed, dangling symlinks can appear.
Just drop them.
'''
def _cleanup_old_runs():
    for f in glob.glob("/var/spool/abrt/*_*"):
        if os.path.islink(f) and not os.path.exists(f):
            os.unlink(f)


if __name__ == "__main__":
    _cleanup_old_runs()

    if os.path.isdir('/vz/tmp'):
        reportFile = '/vz/tmp/report.tar.gz'
    else:
        reportFile = '/var/tmp/report.tar.gz'

    # Use MaxCrashReportsSize from abrt.conf to limit size of added logs, as well
    try:
        # the value in abrt.conf is in MB
        MaxCrashReportsSize = int(get_max_report_size()) * 1000000
    except:
        MaxCrashReportsSize = -1

    # We need some space to create tarball with report
    # Can't predict how many, but not more than MaxCrashReportsSize (if set)
    if MaxCrashReportsSize == -1:
        FreeSpaceNeeded = os.path.getsize(dump_dir)
    else:
        FreeSpaceNeeded = MaxCrashReportsSize

    f = os.statvfs(os.path.dirname(reportFile))
    free_space = f[statvfs.F_BFREE]*f[statvfs.F_FRSIZE]
    if free_space < FreeSpaceNeeded :
        print_log("Low free space, will not prepare and send report tarball", reporterLog)
        sys.exit(2)

    # Virtuozzo parser cannot handle dirs with ":" so rename them
    if ':' in dump_dir:
        tempDir = dump_dir.replace(':', "_")
        try:
            if os.path.islink(tempDir):
                os.unlink(tempDir)
            elif os.path.isdir(tempDir):
                shutil.rmtree(tempDir)
            elif os.path.isfile(tempDir):
                os.remove(tempDir)
            os.symlink(dump_dir, tempDir)
        except Exception as e:
            print_log("Error: Can't create report folder! " + str(e), reporterLog)
            sys.exit(1)
        dump_dir = tempDir
    path_dump = os.path.basename(dump_dir)
    component = ''
    try:
        component = open(dump_dir + '/component', 'r').read()
    except:
        print_log("Error: 'component' file should be present in log - can't send a report without it!", reporterLog)
        _cleanup(dump_dir)
        sys.exit(1)
    reason = ''
    try:
        reason = open(dump_dir + '/reason', 'r').read()
    except:
        print_log("Warning: There's no 'reason' file in log", reporterLog)

    try:
        generate_report_xml(dump_dir)
    except:
        # This atually means that we will not be able to parse report...
        print_log("Failed to dump Report.xml", reporterLog)

    STORAGE_BACKEND_LOGS = ['backup.log', 'celery.log', 'celery-beat.log', 'celery-periodic.log',
        'celery-bouncer.log', 'messages', 'keystone-manage.log', 'keystone-wsgi-admin.log'
    ]
    STORAGE_AGENT_LOGS = ['messages.log', 'init-agent.log', 'oneshot-scripts.log', 'uwsgi.log'
    ]

    COMMON_LOGS = ['/etc/opt/pva/pva-release',
                   '/etc/vstorage/abgw.conf',
                   '/etc/vz/problem_contact.conf',
                   '/var/log/keystone/keystone.log',
                   '/var/log/openvswitch/ovs-vswitchd.log',
                   '/var/log/openvswitch/ovsdb-server.log',
                   '/var/log/vstorage/vstorage-iscsid.log',
                   '/var/log/vstorage/vstorage-tcm.log',
                   '/var/log/vstorage/vstorage-tcm-monitor.log',
                   '/var/log/vstorage/vstorage-target-monitor.log',
                   '/var/log/ganesha/ganesha.log',
                   '/var/log/vstorage/vstorage-target-manager.log',
                   '/var/log/shaman/shaman-monitor.log',
                   '/var/log/shaman/shaman.log',
                   '/sys/kernel/debug/cleancache/succ_gets',
                   '/proc/net/snmp',
                   '/proc/net/netstat'
    ]

    try:
        cmdline = open(dump_dir + '/cmdline', 'r').read()
    except:
        cmdline = ''


    hw_problem = known_issues(dump_dir)
    # Add some more files to standard abrt report
    try:
        tar = tarfile.open(reportFile, "w:gz", dereference=True)
        tar.add(dump_dir, path_dump)
        total_size = get_folder_size(dump_dir)

        for log in COMMON_LOGS:
            if os.path.isfile(log):
                total_size = _add_file_safe(tar, log, path_dump + "/", total_size)

        if os.path.isdir("/var/log/libvirt") and not hw_problem:
            log_size = get_folder_size("/var/log/libvirt")
            if (MaxCrashReportsSize > 0 and (log_size < MIN_SIZE or (log_size + total_size) < MaxCrashReportsSize)) \
                    or ((log_size + total_size) < free_space):
                tar.add("/var/log/libvirt", path_dump + "/libvirt-logs")
                total_size += log_size

        if ("storage" in component or "nfs" in component or "archive" in component) and not hw_problem:
            for f in glob.glob("/var/log/ostor/*.log") + glob.glob("/var/log/ostor/*.log.zst") + glob.glob("/var/log/ostor/*.blog") + glob.glob("/var/log/ostor/*.log.gz"):
                total_size = _add_file_safe(tar, f, path_dump + "/ostor-logs/" + os.path.dirname(f).replace("/", "_")  + "_", total_size)
            for f in glob.glob("/vstorage/**/mds/logs/*.log") + glob.glob("/vstorage/**/mds/logs/*.log.zst") + glob.glob("/vstorage/**/mds/logs/*.blog"):
                total_size = _add_file_safe(tar, f, path_dump + "/mds-logs/" + os.path.dirname(f).replace("/", "_")  + "_", total_size)
            for f in glob.glob("/vstorage/**/cs/logs/*.log") + glob.glob("/vstorage/**/cs/logs/*.log.zst") + glob.glob("/vstorage/**/cs/logs/*.blog"):
                total_size = _add_file_safe(tar, f, path_dump + "/cs-logs/" + os.path.dirname(f).replace("/", "_")  + "_", total_size)
            for f in glob.glob("/var/log/vstorage/**/vstorage-mount.log.zst") + glob.glob("/var/log/vstorage/**/vstorage-mount.blog"):
                total_size = _add_file_safe(tar, f, path_dump + "/vstorage-mount-logs/" + os.path.dirname(f).replace("/", "_")  + "_", total_size)
            for f in glob.glob("/var/log/vstorage/abgw.log.zst") + glob.glob("/var/log/vstorage/abgw.blog"):
                total_size = _add_file_safe(tar, f, path_dump + "/abgw-logs/", total_size)
            for f in glob.glob("/var/lib/pgsql/data/pg_log/*log"):
                total_size = _add_file_safe(tar, f, path_dump + "/pg-logs/", total_size)

            if os.path.isdir("/var/log/vstorage/iscsi"):
                log_size = get_folder_size("/var/log/vstorage/iscsi")
                if (MaxCrashReportsSize > 0 and (log_size < MIN_SIZE or (log_size + total_size) < MaxCrashReportsSize)) \
                        or ((log_size + total_size) < free_space):
                    tar.add("/var/log/vstorage/iscsi", path_dump + "/iscsi-logs")
                    total_size += log_size

            if os.path.isdir("/var/log/vstorage-ui-agent"):
                for f in STORAGE_AGENT_LOGS:
                    full_path = "/var/log/vstorage-ui-agent/" + f
                    if os.path.isfile(full_path):
                        total_size = _add_file_safe(tar, full_path, path_dump + "/vstorage-ui-agent-logs/", total_size)

            if os.path.isdir("/var/log/vstorage-ui-backend"):
                for f in STORAGE_BACKEND_LOGS:
                    full_path = "/var/log/vstorage-ui-backend/" + f
                    if os.path.isfile(full_path):
                        total_size = _add_file_safe(tar, full_path, path_dump + "/vstorage-ui-backend-logs/", total_size)
            if get_fail_mds_logs(dump_dir):
                total_size = _add_file_safe(tar, dump_dir + "/fail_mds_logs.txt", path_dump + "/", total_size)

        if "archive" in cmdline and not hw_problem:
            m = re.findall('-L (\S+)', cmdline)
            for journal in m:
                if os.path.isfile(journal):
                    total_size = _add_file_safe(tar, journal, path_dump + "/", total_size)

        tar.close()

    except Exception as e:
        print_log("Error: Can't create archive with logs! " + str(e), reporterLog)
        if os.path.exists(reportFile):
            os.remove(reportFile)
        _cleanup(dump_dir)
        sys.exit(1)

    if os.path.exists(btrace_file) and os.path.getsize(btrace_file) > 0:
        trace_hash = hashlib.sha256(btrace_file).hexdigest()
    elif os.path.exists(corebt_file) and os.path.getsize(corebt_file) > 0:
        with open(corebt_file, 'r') as corebt_json:
            data = json.load(corebt_json)
            format_data = format_abrt_stacktrace(data)
            trace_hash = hashlib.sha256(format_data).hexdigest()
    else:
       trace_hash = ''

    time.sleep(1) # too frequent requests are not allowed by the server

    if os.path.exists("/usr/share/disp-helper/ovz-cert.crt"):
        p = subprocess.Popen(['curl', '--cacert', '/usr/share/disp-helper/ovz-cert.crt', '-D-', '-F', 'name=@' + reportFile, '-F', 'backtrace_hash=' + trace_hash, '-F', 'pkg="' + component + '"', '-F', 'desc="' + reason + '"', "https://report.virtuozzo.com/anacrash"],
                               env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    elif trace_hash:
        p = subprocess.Popen(['curl', '-k', '-D-', '-F', 'name=@' + reportFile, '-F', 'backtrace_hash=' + trace_hash, '-F', 'pkg="' + component + '"', '-F', 'desc="' + reason + '"', "https://report.virtuozzo.com/anacrash"],
                               env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    else:
        p = subprocess.Popen(['curl', '-k', '-D-', '-F', 'name=@' + reportFile, '-F', 'pkg="' + component + '"', '-F', 'desc="' + reason + '"', "https://report.virtuozzo.com/anacrash"],
                               env=os.environ, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    curlReply = p.stdout.read()
    if 'status=OK' in curlReply:
        print_log("The problem report was successfully sent with id: " + curlReply.split(';')[-1], reporterLog)
    else:
        print_log("Error: Can't upload the problem report: \n" + curlReply, reporterLog)
    # Delete temp directory if we created it
    if dump_dir != os.environ.get('DUMP_DIR'):
        try:
            os.unlink(dump_dir)
        except:
            print_log("Error: Can't remove temp folder " + dump_dir, reporterLog)
            pass

    try:
        os.remove(reportFile)
    except:
        print_log("Error: Can't remove temp file " + reportFile, reporterLog)
        pass

