#!/usr/bin/python3

# appstream-data-generator is an application for importing appdata and metainfo
# from repository along with icons and converting it into appstream-data format
#
# Copyright (C) 2018-2019 Aleksei Nikiforov <darktemplar@basealt.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

# requirements:
# - bsdtar
# - convert from ImageMagick
# - python3(rpm)
# - python3(lxml)
# - python3(PIL.Image)
# - python3(xdg.DesktopEntry)

# This script does appoximately following actions:
# 1) takes path(s) to locally available repositores
# 2) reads contents files from repositores and saves information
#    about *.appdata.xml and *.metainfo.xml files
# 2.1) at the same time mapping between those files and original packages is created,
#      which is used later in step 4.2.2
# 3) for each package containing *.appdata.xml file(s):
# 3.1) finds containing RPM package
# 3.2) extracts every *.appdata.xml, icon and desktop files
# 3.3) reads every extracted *.appdata.xml file and for each of it:
# 3.3.1) tries to find corresponding .desktop file mentioned in *.appdata.xml file,
#        if no such file is found, current *.appdata.xml is effectively skipped
# 3.3.2) tries to find icon files, resizes icons if it's needed
# 3.3.3) adds some missing data to *.appdata.xml, including pkgname, releases,
#        name, summary, keywords, categories
# 3.3.4) processed xml data is saved
# 4) for each package containing *.metainfo.xml file(s):
# 4.1) finds containing RPM package
# 4.2) extracts every *.metainfo.xml file, and processes it
# 5) composes single file out of all processed *.appdata.xml and *.metainfo.xml files

import sys
import os
import glob
import tempfile
import atexit
import shutil
import subprocess
import argparse
import re
import rpm
# use lxml instead of standard xml.etree due to namespaces
import lxml.etree as ET
import configparser
from PIL import Image
import xdg.DesktopEntry

import gettext

gettext.bindtextdomain('appstream-data')
gettext.textdomain('appstream-data')

_ = gettext.gettext

def convert_package_name(originalname, convertion_list):
    """ if original name is found in convertion list, converted name is returned, otherwise original name is returned """
    newname = convertion_list.get(originalname)
    if newname is not None:
        return newname
    else:
        return originalname

def ensure_directory(path):
    """ check that path is directory or doesn't exist, exit if it exists and not a directory """
    if os.path.isdir(path):
        pass
    elif not os.path.exists(path):
        os.mkdir(path)
    else:
        print(_("ERROR: Path exists and is not a directory: {}").format(path), file=sys.stderr)
        sys.exit(-1)

def ensure_directory_recursive(path):
    """ create directory and necessary parent directories """
    if os.path.isdir(path):
        pass
    elif not os.path.exists(path):
        ensure_directory_recursive(os.path.dirname(path))
        os.mkdir(path)
    else:
        print(_("ERROR: Path exists and is not a directory: {}").format(path), file=sys.stderr)
        sys.exit(-1)

def get_icon_size(filename):
    """ get icon size """
    width = 0
    height = 0

    try:
        im = Image.open(filename)
        width, height = im.size
    except:
        print(_("ERROR: failed to determine size of icon: {}").format(filename), file=sys.stderr)
        pass

    return width, height

def remove_known_image_ext(filename):
    """ remove known extension of image file name """
    if filename.endswith(".png") or filename.endswith(".xpm") or filename.endswith(".svg") or filename.endswith(".svgz"):
        filename = os.path.splitext(filename)[0]

    return filename

def remove_nonints(array):
    result = []
    for item in array:
        if isinstance(item, int):
            result.append(item)
    return result

def find_icon_theme_dirs(iconthemefilename):
    """ find directories in icon themes """
    result = []

    try:
        themefile = configparser.ConfigParser(strict=False) # Ignore duplicated sections
        themefile.read([iconthemefilename])
        directories = themefile["Icon Theme"]["Directories"]
        if directories:
            directories_list = directories.split(',')
            for directory in directories_list:
                try:
                    if themefile[directory]["Context"] == "Applications":
                        result.append(directory)
                except:
                    pass
    except:
        pass

    return result

def find_icons(globsarray):
    """ find icons in directories """
    result = {}
    for globitem in globsarray:
        globresult = glob.glob(globitem)
        if not globresult:
            continue
        for filename in sorted(globresult):
            if not os.path.isfile(filename):
                continue
            if filename.endswith(".png"):
                width, height = get_icon_size(filename)
                if (width != 0) and (height != 0):
                    if width >= height:
                        result[width] = filename
                    else:
                        result[height] = filename
            if filename.endswith(".xpm"):
                result["xpm"] = filename
            if filename.endswith(".svg"):
                result["scalable"] = filename

    return result

def copy_icons(icons, destinationdir, iconname, iconsizes):
    """ copy icons, resize if necessary """
    remaining_sizes = iconsizes.copy()
    remaining_sizes_copy = remaining_sizes.copy()
    for iconsize in remaining_sizes_copy:
        if iconsize in icons:
            originalfile = icons[iconsize]
            iconsdestdir = os.path.join(destinationdir, "{}x{}".format(iconsize, iconsize))
            destinationname = os.path.join(iconsdestdir, iconname)
            ensure_directory_recursive(iconsdestdir)
            if verbose:
                print(_("Copying image file {} to {}").format(originalfile, iconsdestdir))
            shutil.copyfile(originalfile, destinationname)
            remaining_sizes.remove(iconsize)

    if remaining_sizes:
        remaining_sizes_copy = remaining_sizes.copy()
        for iconsize in remaining_sizes_copy:
            if "scalable" in icons:
                originalfile = icons["scalable"]
            else:
                found_size = 0
                for size in sorted(remove_nonints(icons.keys())):
                    found_size = size
                    if size >= iconsize:
                        break

                if found_size != 0:
                    originalfile = icons[found_size]
                else:
                    if "xpm" in icons:
                        originalfile = icons["xpm"]
                    else:
                        continue

            iconsdestdir = os.path.join(destinationdir, "{}x{}".format(iconsize, iconsize))
            destinationname = os.path.join(iconsdestdir, iconname)
            ensure_directory_recursive(iconsdestdir)
            if verbose:
                print(_("Copying image file {} to {}, resizing to {}x{}").format(originalfile, destinationname, iconsize, iconsize))
            # Option '-strip' removes additional metadata headers, including ones containing timestamps.
            # It's needed to remove timestamps to make converted images identical if their bitmap data is identical
            subprocess.call(["magick", "-background", "none", originalfile, "-resize", "{}x{}".format(iconsize, iconsize), "-strip", destinationname])
            remaining_sizes.remove(iconsize)

    return True if not remaining_sizes else False

def prefix_array_with_elements(prefix_value, original_array):
    """ take icons array and convert it to cmdline arguments """
    result = []
    for item in original_array:
        result.append(prefix_value)
        result.append(item)
    return result

def unpack_package(packagename, repositorypaths, xmlnamefilterlist, outputdir, listfilesdir, unpackdirs):
    """ unpack package if it's name matches specified and return list of unpacked xml files to process """
    xmlfileslist = set()
    rpmhdr = None

    for path in repositorypaths:
        for rpmfile in glob.glob(os.path.join(path, "RPMS.classic", packagename + "-*.rpm")):
            rpmts = rpm.ts()
            fdno = os.open(rpmfile, os.O_RDONLY)
            rpmhdr = rpmts.hdrFromFdno(fdno)
            os.close(fdno)
            if rpmhdr[rpm.RPMTAG_NAME].decode() == packagename:
                if verbose:
                    print(_("Processing rpm file {}").format(rpmfile))

                listfilesdirname = os.path.join(listfilesdir, packagename)

                with open(listfilesdirname, 'w') as listfile:
                    for line in subprocess.check_output(["bsdtar", "tf", rpmfile]).decode().splitlines(True):
                        if xmlnamefilterlist:
                            if (line.startswith("./usr/share/appdata/") or line.startswith("./usr/share/metainfo/")) and any(line.rstrip().endswith(ext) for ext in xmlnamefilterlist):
                                listfile.write(line)
                                xmlfileslist.add(line.rstrip())
                        for dirname in unpackdirs:
                            if line.startswith("." + dirname):
                                listfile.write(line)

                subprocess.call(["bsdtar", "xf", rpmfile, "-T", listfilesdirname, "-C", outputdir])
                return rpmhdr, xmlfileslist

    return rpmhdr, xmlfileslist

def read_lines_from_file(filename):
    result = set()
    if filename is not None:
        with open(filename) as opened_file:
            for line in opened_file:
                line = line.rstrip()
                if line:
                    result.add(line)
    return result

def read_lines_from_file_and_split_two(filename):
    result = {}
    if filename is not None:
        with open(filename) as opened_file:
            for line in opened_file:
                value1, value2 = line.rstrip().split(' ',1)
                if value1 and value2:
                    result[value1] = value2
                else:
                    print(_("Warning: skipping line from file {}: {}").format(filename, line), file=sys.stderr)
    return result

def read_lines_from_file_and_split_multiple(filename):
    result = {}
    if filename is not None:
        with open(filename) as opened_file:
            for line in opened_file:
                values = line.rstrip().split()
                if len(values) >= 2:
                    value = values.pop(0)
                    if value not in result:
                        result[value] = values
                    else:
                        result[value] += values
                else:
                    print(_("Warning: skipping line from file {}: {}").format(filename, line), file=sys.stderr)
    return result

def should_skip(packagename, skiplist):
    """ checks skiplist and returns true if package should be skipped """
    for skippedpackage in skiplist:
        skipregex = re.compile(skippedpackage)
        if skipregex.match(packagename):
            return True

    return False

parser = argparse.ArgumentParser(description=_('Convert appdata and metainfo files from repositories into appinfo file'))
parser.add_argument("output", type=str, help=_("output directory name"))
parser.add_argument("repository", type=str, nargs='+', help=_("path to repository"))
parser.add_argument("--skiplist", "-s", type=str, help=_("path to file containing names of packages to be skipped. Format is: regex, one item per line"))
parser.add_argument("--exclusivelist", "-e", type=str, help=_("path to file containing names of packages to be exclusively used. Format is: exact package name, one item per line"))
parser.add_argument("--nameconvert", "-n", type=str, help=_("path to file containing package name convertion list. Format is: '$original_name $converted_name', one item per line"))
parser.add_argument("--additionaldesktopnames", "-c", type=str, help=_("path to file containing list of desktop file names to be additionally checked. Format is: '$package_name $desktop_name [$desktop_name ...]'. One package per line, multiple desktop files may be specified."))
parser.add_argument("--additionalpackages", "-g", type=str, help=_("path to file containing list of packages to be unpacked in addition to main package. Format is: '$package_name $additional_package_name [$additional_package_name ...]. One main package name per line, multiple additional packages per line."))
parser.add_argument("--skiplanguageslist", "-l", type=str, help=_("path to file containing list of packages and languages to be skipped for specified packages. Format is: '$package_name $skipped_language [$skipped_language ...]'. One package per line, multiple languages for specified package per line."))
parser.add_argument("--origin", "-o", type=str, default="altlinux", help=_("origin of metadata, by default 'altlinux'"))
parser.add_argument("--iconsizes", "-i", type=int, action='append', help=_("sizes of icons to copy, may be specified multiple times, default is 64 and 128"))
parser.add_argument("--applicationsdir", "-a", type=str, action='append',
    default=[
        "/usr/share/applications/",
        "/usr/share/applications/kde/",
        "/usr/share/applications/kde4/",
        "/usr/share/applications/kf5/",
        "/usr/share/kde4/applications/",
        "/usr/share/kde4/applications/kde4/",
        "/usr/share/kde/applications/",
        "/usr/share/kde/applications/kde/",
        "/usr/share/kf5/applications/",
        "/usr/share/kf5/applications/kf5/",
        "/usr/share/mate/applications/",
        "/usr/share/tde3/applications/"
    ],
    help=_("additional directories to look for .desktop files. Must end with '/' character."))
parser.add_argument("--iconsdir", "-d", type=str, action='append',
    default=[
        "/usr/share/icons/",
        "/usr/share/kde4/icons/",
        "/usr/share/kde/icons/",
        "/usr/share/kf5/icons/",
        "/usr/share/icons/mate/",
        "/usr/share/tde3/icons/"
    ],
    help=_("additional directories to look for icon files. Must end with '/' character."))
parser.add_argument("--icontheme", "-t", type=str, action='append',
    default=[
        "hicolor",
        "breeze",
        "oxygen",
        "Adwaita",
        "gnome"
    ],
    help=_("additional icon themes to check icons in"))
parser.add_argument("--pixmapsdir", "-p", type=str, action='append',
    default=[
        "/usr/share/pixmaps/"
    ],
    help=_("additional directories to look for pixmap files. Must end with '/' character."))
parser.add_argument("--usedesktopfiles", "-u", action='store_true', help=_("use desktop files to generate appdata if package doesn't contain appdata"))
parser.add_argument("--userpmlicense", "-r", action='store_true', help=_("add license from rpm tag to generated appdata. Only works for appdata generated from desktop file"))
parser.add_argument("--with-progress", "-P", action='store_true', help=_("print processing progress in format: `0/0: Processing file of package`. --verbose includes this flag"))
parser.add_argument("--desktopfileskiplist", type=str, help=_("Path to file containing names of desktop files to be skipped. Only useful when appdata is generated from desktop files. Format is: full filename, one item per line"))
parser.add_argument("--desktopfileexclusivelist", type=str, help=_("Path to file containing names of desktop files to be exclusively used. Only useful when appdata is generated from desktop files. Format is: full filename, one item per line"))
parser.add_argument("--allownoicons", action='store_true', help=_("Convert errors about missing icons to warnings"))
parser.add_argument("--verbose", "-v", action='store_true', help=_("Increase output verbosity"))
parser.add_argument("--nocleanup", "-k", action='store_true', help=_("Don't remove temporary data"))
parser.add_argument("--generated_metadata_license", "-m", type=str, default="CC0-1.0", help=_("Generated metadata license. If empty value is specified, generated metadata license tag is omitted. Only used for metadata generated from desktop files. Default is 'CC0-1.0'."))
args = parser.parse_args()

outputdir = args.output
repositorypaths = args.repository
skiplistfilename = args.skiplist
exclusivelistfilename = args.exclusivelist
nameconvertfilename = args.nameconvert
additionaldesktopnamesfilename = args.additionaldesktopnames
additionalpackagesfilename = args.additionalpackages
skiplanguageslistfilename = args.skiplanguageslist
originname = args.origin
appdirs = args.applicationsdir
iconsdirs = args.iconsdir
iconthemes = args.icontheme
pixmapdirs = args.pixmapsdir
usedesktopfiles = args.usedesktopfiles
allownoicons = args.allownoicons
verbose = args.verbose
nocleanup = args.nocleanup
generated_metadata_license = args.generated_metadata_license
userpmlicense = args.userpmlicense
with_progress = args.with_progress or verbose
desktopfileskiplistfilename = args.desktopfileskiplist
desktopfileexclusivelistfilename = args.desktopfileexclusivelist

iconsizes = set()
if args.iconsizes is None:
    iconsizes.add(64)
    iconsizes.add(128)
else:
    for size in args.iconsizes:
        iconsizes.add(size)

# check that repository contains base/contents_index file
for path in repositorypaths:
    if (not os.path.isdir(path)) or (not os.path.isfile(os.path.join(path, 'base', 'contents_index'))):
        print(_("ERROR: Path is not a valid repository: {}").format(path), file=sys.stderr)
        sys.exit(-1)

ensure_directory(outputdir)
ensure_directory(os.path.join(outputdir, "xmls"))
ensure_directory(os.path.join(outputdir, "icons"))

skiplist = read_lines_from_file(skiplistfilename)
exclusivelist = read_lines_from_file(exclusivelistfilename)
exclusivelist_encountered = set()
nameconvert = read_lines_from_file_and_split_two(nameconvertfilename)
additionaldesktopnames = read_lines_from_file_and_split_multiple(additionaldesktopnamesfilename)
additionalpackages = read_lines_from_file_and_split_multiple(additionalpackagesfilename)
skiplanguageslist = read_lines_from_file_and_split_multiple(skiplanguageslistfilename)
desktopfileskiplist = read_lines_from_file(desktopfileskiplistfilename)
desktopfileexclusive = read_lines_from_file(desktopfileexclusivelistfilename)
desktopfileexclusive_encountered = set()

appstream_packages = []
addon_packages = []
desktop_packages = {}
processed_appstream_packages = set()
processed_addon_packages = set()
entries_appstream_set = []
entries_addon_set = []
appstream_regex = re.compile(r"^.*/([^/]+)(?:\.appdata\.xml|\.metainfo\.xml)$")
found_desktop_files = {}
found_appstream_packages = []

# find all interesting files in repositories and save package names
for path in repositorypaths:
    if verbose:
        print(_("Reading file {}").format(os.path.join(path, 'base', 'contents_index')))
    with open(os.path.join(path, 'base', 'contents_index'), encoding="latin-1") as contents_file:
        for line in contents_file:
            if line.startswith("/usr/share/appdata/") or line.startswith("/usr/share/metainfo/"):
                filename, package = line.rstrip().rsplit('\t', 1)
                if verbose:
                    print(_("Parse %s from package %s") % (filename,package))
                if filename.endswith(".appdata.xml") or filename.endswith(".metainfo.xml"):
                    if not should_skip(package, skiplist):
                        appdata_match = appstream_regex.match(filename)
                        if not appdata_match:
                            break

                        found_appstream_packages.append(package)
                        appstream_packages.append((path, package, filename))

            if usedesktopfiles:
                found_appdir_entry = False
                for appdir_entry in appdirs:
                    if line.startswith(appdir_entry):
                        found_appdir_entry = True
                        break
                if found_appdir_entry:
                    filename, package = line.rstrip().rsplit('\t', 1)
                    if filename.endswith(".desktop") and (filename not in desktopfileskiplist):
                        if package not in found_desktop_files:
                            found_desktop_files[package] = {}
                            found_desktop_files[package]["repo"] = path
                            found_desktop_files[package]["files"] = []
                        found_desktop_files[package]["files"].append(filename)

    if usedesktopfiles:
        for desktop_pkg_name in found_desktop_files:
            if (desktop_pkg_name not in found_appstream_packages) and (not should_skip(desktop_pkg_name, skiplist)):
                desktop_packages[desktop_pkg_name] = found_desktop_files[desktop_pkg_name]

    for pkg_name in found_appstream_packages:
        if pkg_name.endswith(".metainfo.xml") and pkg_name not in found_desktop_files.keys() and not should_skip(pkg_name, skiplist):
            addon_packages.append(pkg_name)

    found_appstream_packages = list(filter(lambda found_pkg_name: found_pkg_name not in addon_packages, found_appstream_packages))
    appstream_packages = list(filter(lambda package_item: package_item[1] not in addon_packages, appstream_packages))

processed_count:int = 0
total_count:int = len(exclusivelist) if exclusivelist else len(appstream_packages) + (len(desktop_packages) if usedesktopfiles else 0) + len(addon_packages) - len(skiplist)
total_align:int = len(str(total_count))

tempdirname = tempfile.mkdtemp()
if nocleanup:
    if verbose:
        print("Temp dir: {}".format(tempdirname))
else:
    atexit.register(shutil.rmtree, tempdirname)

listfilesdir = os.path.join(tempdirname, "listfiles")
packagefilesdir = os.path.join(tempdirname, "packagefiles")
listaddonsfilesdir = os.path.join(tempdirname, "addonlistfiles")
packageaddonsfilesdir = os.path.join(tempdirname, "packageaddonfiles")
packagedesktopfilesdir = os.path.join(tempdirname, "packagedesktopfiles")
convertedxmldir = os.path.join(tempdirname, "convertedxml")
output_icons_dir = os.path.join(outputdir, "icons", originname)

os.mkdir(listfilesdir)
os.mkdir(packagefilesdir)
os.mkdir(listaddonsfilesdir)
os.mkdir(packageaddonsfilesdir)
os.mkdir(packagedesktopfilesdir)
os.mkdir(convertedxmldir)

# process appstream xml files
for packageitem in appstream_packages:
    processed_count += 1

    package_name = packageitem[1]
    package_repo_path = packageitem[0]
    if exclusivelist:
        if package_name not in exclusivelist:
            continue
        exclusivelist_encountered.add(package_name)

    if package_name not in processed_appstream_packages:
        # don't process same package twice
        # useful in case of package containing multiple appdata files
        processed_appstream_packages.add(package_name)

        packagefilesdirname = os.path.join(packagefilesdir, package_name)
        convertedxmldirname = os.path.join(convertedxmldir, package_name)
        ensure_directory(packagefilesdirname)
        ensure_directory(convertedxmldirname)

        rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], [".appdata.xml", ".metainfo.xml"], packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

        if package_name in additionalpackages:
            for package in additionalpackages[package_name]:
                if package:
                    unpack_package(package, repositorypaths, [".appdata.xml", ".metainfo.xml"], packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

        for xmlfilename in xmlfileslist:
            if with_progress:
                print(_("{}/{}: Processing xml {} of package {}").format(str(processed_count).zfill(total_align), total_count, xmlfilename, package_name), flush=True)

            # if parsing file fails, report it, but continue processing remaining files
            try:
                xmlfile_name = os.path.join(packagefilesdirname, xmlfilename.strip(os.sep))
                xmlinfo_root = ET.parse(xmlfile_name)
                xmlinfo_processed = xmlinfo_root.getroot()

                if xmlinfo_processed.find("pkgname") is not None:
                    if verbose:
                        print(_("Skipping {} as it already contains <pkgname> element").format(xmlfile_name))
                    continue

                appended_pkgname = convert_package_name(package_name, nameconvert)

                desktopfilenames = set()
                for id_element in xmlinfo_processed.iterfind('id', xmlinfo_processed.nsmap):
                    desktopfilenames.add(id_element.text)
                for launchable_element in xmlinfo_processed.iterfind('launchable', xmlinfo_processed.nsmap):
                    desktopfilenames.add(launchable_element.text)
                if package_name in additionaldesktopnames:
                    for desktopname in additionaldesktopnames[package_name]:
                        if desktopname:
                            desktopfilenames.add(desktopname)

                if not desktopfilenames:
                    print(_("ERROR: Failed to parse file {} from package {}: couldn't find desktop file name").format(xmlfilename, package_name), file=sys.stderr)
                    continue

                appended_release = str(rpmhdr[rpm.RPMTAG_BUILDTIME]) + ";" + str(rpmhdr[rpm.RPMTAG_VERSION].decode())

                iconname = None

                # Search cached iconname from appdata file
                for icon in xmlinfo_processed.iterfind('icon', xmlinfo_processed.nsmap):
                    iconname = icon.text

                for desktopfilename in desktopfilenames:
                    desktopfilename = os.path.basename(desktopfilename)
                    desktopfilenameprocessed = desktopfilename
                    if not desktopfilenameprocessed.endswith(".desktop"):
                        desktopfilenameprocessed = desktopfilenameprocessed + ".desktop"

                    for dirname in appdirs:
                        desktopfilepath = os.path.join(packagefilesdirname, dirname.strip(os.sep), desktopfilenameprocessed)

                        if os.path.isfile(desktopfilepath):
                            try:
                                config = xdg.DesktopEntry.DesktopEntry(desktopfilepath)
                                iconname = config.get("Icon")
                            except:
                                print(_("ERROR: Failed to parse file {} from package {}").format(desktopfilepath, package_name), file=sys.stderr)
                                pass
                            if iconname is not None:
                                break

                    if iconname is not None and not allownoicons:
                        break

                # Process icon
                if iconname is not None:
                    basic_iconname = remove_known_image_ext(os.path.basename(iconname))
                    basic_icon_extensions = [ ".png", ".svg", ".xpm" ]

                    iconsdirsglob = [ os.path.join(packagefilesdirname, iconname.strip(os.sep)) ]

                    for iconsdir in iconsdirs:
                        for icon_theme in iconthemes:
                            theme_dirs = [ "*/apps" ] # fallback value in case no index.theme is present
                            themefilename = os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, "index.theme")
                            if os.path.isfile(themefilename):
                                theme_dirs = find_icon_theme_dirs(themefilename)
                            for theme_dir in theme_dirs:
                                for extension in basic_icon_extensions:
                                    iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, theme_dir, basic_iconname + extension))

                    for iconsdir in iconsdirs:
                        for icon_theme in iconthemes:
                            for extension in basic_icon_extensions:
                                iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, basic_iconname + extension))

                    for iconsdir in iconsdirs:
                        for extension in basic_icon_extensions:
                            iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), basic_iconname + extension))

                    for pixmapdir in pixmapdirs:
                        for extension in basic_icon_extensions:
                            iconsdirsglob.append(os.path.join(packagefilesdirname, pixmapdir.strip(os.sep), basic_iconname + extension))

                    icons = find_icons(iconsdirsglob)
                    if not icons:
                        if not allownoicons:
                            print(_("ERROR: Failed to find icons {}.* for file {} from package {}").format(basic_iconname, desktopfilename, package_name), file=sys.stderr)
                            continue
                        else:
                            print(_("Warning: Failed to find icons {}.* for file {} from package {}").format(basic_iconname, desktopfilename, package_name), file=sys.stderr)

                    if icons:
                        result = copy_icons(icons, output_icons_dir, basic_iconname + ".png", iconsizes)
                        if not result:
                            if not allownoicons:
                                print(_("ERROR: Failed to copy all icons for file {} from package {}").format(desktopfilename, package_name), file=sys.stderr)
                                continue
                            else:
                                print(_("Warning: Failed to copy all icons for file {} from package {}").format(desktopfilename, package_name), file=sys.stderr)

                    copied_icons_data = []
                    if icons:
                        for iconsize in iconsizes:
                            copied_icons_data.append("cached;" + str(iconsize) + ";" + str(iconsize) + ";" + basic_iconname + ".png")
                        copied_icons_data = prefix_array_with_elements("--icon", copied_icons_data)

                # Process XML contents
                if iconname is None:
                    if not allownoicons:
                        print(_("ERROR: Failed to parse files {} from package {}: couldn't find icon file name").format(desktopfilenames, package_name), file=sys.stderr)
                        continue
                    else:
                        copied_icons_data = []

                skipped_languages = []
                if package_name in skiplanguageslist:
                    skipped_languages = prefix_array_with_elements("--exclude_language", skiplanguageslist[package_name])

                outputxmlname = os.path.join(convertedxmldirname, os.path.basename(xmlfilename))
                res = subprocess.call(["appstream-data-appdata-converter", xmlfile_name, outputxmlname, desktopfilepath, appended_pkgname, "--release", appended_release] + skipped_languages + copied_icons_data)
                if res == 0:
                    entries_appstream_set.append(outputxmlname)
                else:
                    print(_("ERROR: Failed to process file {} from package {}").format(xmlfilename, package_name), file=sys.stderr)

            except ET.ParseError as err:
                print(_("ERROR: Failed to parse XML file {} from package {}: {}").format(xmlfilename, package_name, str(err)), file=sys.stderr)

# process addon xml files
for packageitem in addon_packages:
    processed_count += 1

    package_name = packageitem[1]
    package_repo_path = packageitem[0]
    if exclusivelist:
        if package_name not in exclusivelist:
            continue
        exclusivelist_encountered.add(package_name)

    if package_name not in processed_addon_packages:
        # don't process same package twice
        # useful in case of package containing multiple appdata files
        processed_addon_packages.add(package_name)

        packagefilesdirname = os.path.join(packageaddonsfilesdir, package_name)
        convertedxmldirname = os.path.join(convertedxmldir, package_name)
        ensure_directory(packagefilesdirname)
        ensure_directory(convertedxmldirname)

        rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], ".metainfo.xml", packagefilesdirname, listaddonsfilesdir, [])

        for xmlfilename in xmlfileslist:
            if with_progress:
                print(_("{}/{}: Processing xml {} of package {}").format(str(processed_count).zfill(total_align), total_count, xmlfilename, package_name), flush=True)

            # if parsing file fails, report it, but continue processing remaining files
            try:
                xmlfile_name = os.path.join(packagefilesdirname, xmlfilename.strip(os.sep))

                xmlinfo_root = ET.parse(xmlfile_name)
                xmlinfo_processed = xmlinfo_root.getroot()

                outputxmlname = os.path.join(convertedxmldirname, os.path.basename(xmlfilename))
                res = subprocess.call(["appstream-data-metainfo-converter", xmlfile_name, outputxmlname, convert_package_name(package_name, nameconvert)])
                if res == 0:
                    entries_addon_set.append(outputxmlname)
                else:
                    print(_("ERROR: Failed to process file {} from package {}").format(xmlfilename, package_name), file=sys.stderr)

            except ET.ParseError as err:
                print(_("ERROR: Failed to parse XML file {} from package {}: {}").format(xmlfilename, package_name, str(err)), file=sys.stderr)


# process .desktop files
if usedesktopfiles:
    for package_name in desktop_packages:
        processed_count += 1

        package_repo_path = desktop_packages[package_name]["repo"]
        if exclusivelist:
            if package_name not in exclusivelist:
                continue
            exclusivelist_encountered.add(package_name)

        packagefilesdirname = os.path.join(packagedesktopfilesdir, package_name)
        convertedxmldirname = os.path.join(convertedxmldir, package_name)
        ensure_directory(packagefilesdirname)
        ensure_directory(convertedxmldirname)

        rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], None, packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

        if package_name in additionalpackages:
            for package in additionalpackages[package_name]:
                if package:
                    unpack_package(package, repositorypaths, None, packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

        for found_desktop_name in desktop_packages[package_name]["files"]:
            if desktopfileexclusive:
                if found_desktop_name not in desktopfileexclusive:
                    continue
            desktopfileexclusive_encountered.add(found_desktop_name)

            if with_progress:
                print(_("{}/{}: Processing desktop {} of package {}").format(str(processed_count).zfill(total_align), total_count, found_desktop_name, package_name), flush=True)

            appended_pkgname = convert_package_name(package_name, nameconvert)
            appended_release = str(rpmhdr[rpm.RPMTAG_BUILDTIME]) + ";" + str(rpmhdr[rpm.RPMTAG_VERSION].decode())

            appended_url = None
            if rpm.RPMTAG_URL in rpmhdr:
                appended_url = str(rpmhdr[rpm.RPMTAG_URL].decode())

            appended_description = None
            if rpm.RPMTAG_DESCRIPTION in rpmhdr:
                appended_description = str(rpmhdr[rpm.RPMTAG_DESCRIPTION].decode())

            appended_project_license = None
            if userpmlicense and (rpm.RPMTAG_LICENSE in rpmhdr):
                appended_project_license = str(rpmhdr[rpm.RPMTAG_LICENSE].decode())

            desktopfilepath = os.path.join(packagefilesdirname, found_desktop_name.strip(os.sep))
            if not os.path.isfile(desktopfilepath):
                print(_("ERROR: Failed to find file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)
                continue

            iconname = None

            try:
                config = xdg.DesktopEntry.DesktopEntry(desktopfilepath)
                iconname = config.get("Icon")
            except:
                print(_("ERROR: Failed to parse file {} from package {}").format(desktopfilepath, package_name), file=sys.stderr)
                pass

            if iconname is None:
                print(_("ERROR: Failed to parse file {} from package {}: couldn't find icon file name").format(found_desktop_name, package_name), file=sys.stderr)
                continue

            basic_iconname = remove_known_image_ext(os.path.basename(iconname))
            basic_icon_extensions = [ ".png", ".svg", ".xpm" ]

            iconsdirsglob = [ os.path.join(packagefilesdirname, iconname.strip(os.sep)) ]

            for iconsdir in iconsdirs:
                for icon_theme in iconthemes:
                    theme_dirs = [ "*/apps" ] # fallback value in case no index.theme is present
                    themefilename = os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, "index.theme")
                    if os.path.isfile(themefilename):
                        theme_dirs = find_icon_theme_dirs(themefilename)
                    for theme_dir in theme_dirs:
                        for extension in basic_icon_extensions:
                            iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, theme_dir, basic_iconname + extension))

            for iconsdir in iconsdirs:
                for icon_theme in iconthemes:
                    for extension in basic_icon_extensions:
                        iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, basic_iconname + extension))

            for iconsdir in iconsdirs:
                for extension in basic_icon_extensions:
                    iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), basic_iconname + extension))

            for pixmapdir in pixmapdirs:
                for extension in basic_icon_extensions:
                    iconsdirsglob.append(os.path.join(packagefilesdirname, pixmapdir.strip(os.sep), basic_iconname + extension))

            icons = find_icons(iconsdirsglob)
            if not icons:
                if not allownoicons:
                    print(_("ERROR: Failed to find icons for file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)
                    continue
                else:
                    print(_("Warning: Failed to find icons for file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)

            if icons:
                result = copy_icons(icons, output_icons_dir, basic_iconname + ".png", iconsizes)
                if not result:
                    if not allownoicons:
                        print(_("ERROR: Failed to copy all icons for file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)
                        continue
                    else:
                        print(_("Warning: Failed to copy all icons for file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)

            copied_icons_data = []
            if icons:
                for iconsize in iconsizes:
                    copied_icons_data.append("cached;" + str(iconsize) + ";" + str(iconsize) + ";" + basic_iconname + ".png")
                copied_icons_data = prefix_array_with_elements("--icon", copied_icons_data)

            skipped_languages = []
            if package_name in skiplanguageslist:
                skipped_languages = prefix_array_with_elements("--exclude_language", skiplanguageslist[package_name])

            cmdline_url = []
            if appended_url:
                cmdline_url.append("--url")
                cmdline_url.append(appended_url)

            cmdline_descr = []
            if appended_url:
                cmdline_descr.append("--description")
                cmdline_descr.append(appended_description)

            cmdline_metadata_license = []
            if generated_metadata_license:
                cmdline_metadata_license.append("--metadata_license")
                cmdline_metadata_license.append(generated_metadata_license)

            cmdline_project_license = []
            if appended_project_license:
                cmdline_project_license.append("--project_license")
                cmdline_project_license.append(appended_project_license)

            outputxmlname = os.path.join(convertedxmldirname, os.path.basename(found_desktop_name) + ".appdata.xml")
            res = subprocess.call(["appstream-data-desktop-converter", desktopfilepath, outputxmlname, appended_pkgname, "--release", appended_release] + cmdline_url + cmdline_descr + cmdline_metadata_license + cmdline_project_license + skipped_languages + copied_icons_data)
            if res == 0:
                entries_appstream_set.append(outputxmlname)
            else:
                print(_("ERROR: Failed to process file {} from package {}").format(found_desktop_name, package_name), file=sys.stderr)

if exclusivelist:
    for item in exclusivelist:
        if item not in exclusivelist_encountered:
            print(_("ERROR: Package {} from exclusive list not found").format(item))

if desktopfileexclusive:
    for item in desktopfileexclusive:
        if item not in desktopfileexclusive_encountered:
            print(_("ERROR: Desktop file {} from desktop files exclusive list not found").format(item))

if (not entries_appstream_set) and (not entries_addon_set):
    print(_("ERROR: no appdata files processed"), file=sys.stderr)
    sys.exit(-1)

subprocess.call(["appstream-data-composer", os.path.join(outputdir, "xmls", originname + ".xml"), "--extra_tag", "origin;" + originname, "--extra_tag", "version;0.8"] + sorted(entries_appstream_set) + sorted(entries_addon_set))
