#!/usr/bin/python3

# appstream-data-generator is an application for importing appdata and metainfo
# from repository along with icons and converting it into appstream-data format
#
# Copyright (C) 2018-2019 Aleksei Nikiforov <darktemplar@basealt.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

# requirements:
# - bsdtar
# - convert from ImageMagick
# - python3(rpm)
# - python3(lxml)
# - python3(PIL.Image)
# - python3(xdg.DesktopEntry)

# This script does appoximately following actions:
# 1) takes path(s) to locally available repositores
# 2) reads contents files from repositores and saves information
#    about *.appdata.xml and *.metainfo.xml files
# 2.1) at the same time mapping between those files and original packages is created,
#      which is used later in step 4.2.2
# 3) for each package containing *.appdata.xml file(s):
# 3.1) finds containing RPM package
# 3.2) extracts every *.appdata.xml, icon and desktop files
# 3.3) reads every extracted *.appdata.xml file and for each of it:
# 3.3.1) tries to find corresponding .desktop file mentioned in *.appdata.xml file,
#        if no such file is found, current *.appdata.xml is effectively skipped
# 3.3.2) tries to find icon files, resizes icons if it's needed
# 3.3.3) adds some missing data to *.appdata.xml, including pkgname, releases,
#        name, summary, keywords, categories
# 3.3.4) processed xml data is saved
# 4) for each package containing *.metainfo.xml file(s):
# 4.1) finds containing RPM package
# 4.2) extracts every *.metainfo.xml file, and processes it
# 5) composes single file out of all processed *.appdata.xml and *.metainfo.xml files

import sys
import os
import glob
import tempfile
import atexit
import shutil
import subprocess
import argparse
import re
import rpm
# use lxml instead of standard xml.etree due to namespaces
import lxml.etree as ET
import configparser
from PIL import Image
import xdg.DesktopEntry

def convert_package_name(originalname, convertion_list):
	""" if original name is found in convertion list, converted name is returned, otherwise original name is returned """
	newname = convertion_list.get(originalname)
	if newname is not None:
		return newname
	else:
		return originalname

def ensure_directory(path):
	""" check that path is directory or doesn't exist, exit if it exists and not a directory """
	if os.path.isdir(path):
		pass
	elif not os.path.exists(path):
		os.mkdir(path)
	else:
		print("ERROR: Path exists and is not a directory: {}".format(path), file=sys.stderr)
		sys.exit(-1)

def ensure_directory_recursive(path):
	""" create directory and necessary parent directories """
	if os.path.isdir(path):
		pass
	elif not os.path.exists(path):
		ensure_directory_recursive(os.path.dirname(path))
		os.mkdir(path)
	else:
		print("ERROR: Path exists and is not a directory: {}".format(path), file=sys.stderr)
		sys.exit(-1)

def get_icon_size(filename):
	""" get icon size """
	width = 0
	height = 0

	try:
		im = Image.open(filename)
		width, height = im.size
	except:
		print("ERROR: failed to determine size of icon: {}".format(filename), file=sys.stderr)
		pass

	return width, height

def remove_known_image_ext(filename):
	""" remove known extension of image file name """
	if filename.endswith(".png") or filename.endswith(".xpm") or filename.endswith(".svg") or filename.endswith(".svgz"):
		filename = os.path.splitext(filename)[0]

	return filename

def remove_nonints(array):
	result = []
	for item in array:
		if isinstance(item, int):
			result.append(item)
	return result

def find_icon_theme_dirs(iconthemefilename):
	""" find directories in icon themes """
	result = []

	try:
		themefile = configparser.ConfigParser(strict=False) # Ignore duplicated sections
		themefile.read([iconthemefilename])
		directories = themefile["Icon Theme"]["Directories"]
		if directories:
			directories_list = directories.split(',')
			for directory in directories_list:
				try:
					if themefile[directory]["Context"] == "Applications":
						result.append(directory)
				except:
					pass
	except:
		pass

	return result

def find_icons(globsarray):
	""" find icons in directories """
	result = {}
	for globitem in globsarray:
		globresult = glob.glob(globitem)
		if not globresult:
			continue
		for filename in sorted(globresult):
			if not os.path.isfile(filename):
				continue
			if filename.endswith(".png"):
				width, height = get_icon_size(filename)
				if (width != 0) and (height != 0):
					if width >= height:
						result[width] = filename
					else:
						result[height] = filename
			if filename.endswith(".xpm"):
				result["xpm"] = filename
			if filename.endswith(".svg"):
				result["scalable"] = filename

	return result

def copy_icons(icons, destinationdir, iconname, iconsizes):
	""" copy icons, resize if necessary """
	remaining_sizes = iconsizes.copy()
	remaining_sizes_copy = remaining_sizes.copy()
	for iconsize in remaining_sizes_copy:
		if iconsize in icons:
			originalfile = icons[iconsize]
			iconsdestdir = os.path.join(destinationdir, "{}x{}".format(iconsize, iconsize))
			destinationname = os.path.join(iconsdestdir, iconname)
			ensure_directory_recursive(iconsdestdir)
			if verbose:
				print("Copying image file {} to {}".format(originalfile, iconsdestdir))
			shutil.copyfile(originalfile, destinationname)
			remaining_sizes.remove(iconsize)

	if remaining_sizes:
		remaining_sizes_copy = remaining_sizes.copy()
		for iconsize in remaining_sizes_copy:
			if "scalable" in icons:
				originalfile = icons["scalable"]
			else:
				found_size = 0
				for size in sorted(remove_nonints(icons.keys())):
					found_size = size
					if size >= iconsize:
						break

				if found_size != 0:
					originalfile = icons[found_size]
				else:
					if "xpm" in icons:
						originalfile = icons["xpm"]
					else:
						continue

			iconsdestdir = os.path.join(destinationdir, "{}x{}".format(iconsize, iconsize))
			destinationname = os.path.join(iconsdestdir, iconname)
			ensure_directory_recursive(iconsdestdir)
			if verbose:
				print("Copying image file {} to {}, resizing to {}x{}".format(originalfile, destinationname, iconsize, iconsize))
			# Option '-strip' removes additional metadata headers, including ones containing timestamps.
			# It's needed to remove timestamps to make converted images identical if their bitmap data is identical
			subprocess.call(["convert", "-background", "none", originalfile, "-resize", "{}x{}".format(iconsize, iconsize), "-strip", destinationname])
			remaining_sizes.remove(iconsize)

	return True if not remaining_sizes else False

def prefix_array_with_elements(prefix_value, original_array):
	""" take icons array and convert it to cmdline arguments """
	result = []
	for item in original_array:
		result.append(prefix_value)
		result.append(item)
	return result

def unpack_package(packagename, repositorypaths, xmlnamefilter, outputdir, listfilesdir, unpackdirs):
	""" unpack package if it's name matches specified and return list of unpacked xml files to process """
	xmlfileslist = set()
	rpmhdr = None

	for path in repositorypaths:
		for rpmfile in glob.glob(os.path.join(path, "RPMS.classic", packagename + "-*.rpm")):
			rpmts = rpm.ts()
			fdno = os.open(rpmfile, os.O_RDONLY)
			rpmhdr = rpmts.hdrFromFdno(fdno)
			os.close(fdno)
			if rpmhdr[rpm.RPMTAG_NAME].decode() == packagename:
				if verbose:
					print("Processing rpm file {}".format(rpmfile))

				listfilesdirname = os.path.join(listfilesdir, packagename)

				with open(listfilesdirname, 'w') as listfile:
					for line in subprocess.check_output(["bsdtar", "tf", rpmfile]).decode().splitlines(True):
						if xmlnamefilter:
							if (line.startswith("./usr/share/appdata/") or line.startswith("./usr/share/metainfo/")) and line.rstrip().endswith(xmlnamefilter):
								listfile.write(line)
								xmlfileslist.add(line.rstrip())
						for dirname in unpackdirs:
							if line.startswith("." + dirname):
								listfile.write(line)

				subprocess.call(["bsdtar", "xf", rpmfile, "-T", listfilesdirname, "-C", outputdir])
				return rpmhdr, xmlfileslist

	return rpmhdr, xmlfileslist

def read_lines_from_file(filename):
	result = set()
	if filename is not None:
		with open(filename) as opened_file:
			for line in opened_file:
				line = line.rstrip()
				if line:
					result.add(line)
	return result

def read_lines_from_file_and_split_two(filename):
	result = {}
	if filename is not None:
		with open(filename) as opened_file:
			for line in opened_file:
				value1, value2 = line.rstrip().split(' ',1)
				if value1 and value2:
					result[value1] = value2
				else:
					print("Warning: skipping line from file {}: {}".format(filename, line), file=sys.stderr)
	return result

def read_lines_from_file_and_split_multiple(filename):
	result = {}
	if filename is not None:
		with open(filename) as opened_file:
			for line in opened_file:
				values = line.rstrip().split()
				if len(values) >= 2:
					value = values.pop(0)
					if value not in result:
						result[value] = values
					else:
						result[value] += values
				else:
					print("Warning: skipping line from file {}: {}".format(filename, line), file=sys.stderr)
	return result

def should_skip(packagename, skiplist):
	""" checks skiplist and returns true if package should be skipped """
	for skippedpackage in skiplist:
		skipregex = re.compile(skippedpackage)
		if skipregex.match(packagename):
			return True

	return False

parser = argparse.ArgumentParser(description='Convert appdata and metainfo files from repositories into appinfo file')
parser.add_argument("output", type=str, help="output directory name")
parser.add_argument("repository", type=str, nargs='+', help="path to repository")
parser.add_argument("--skiplist", "-s", type=str, help="path to file containing names of packages to be skipped. Format is: regex, one item per line")
parser.add_argument("--exclusivelist", "-e", type=str, help="path to file containing names of packages to be exclusively used. Format is: exact package name, one item per line")
parser.add_argument("--nameconvert", "-n", type=str, help="path to file containing package name convertion list. Format is: '$original_name $converted_name', one item per line")
parser.add_argument("--additionaldesktopnames", "-c", type=str, help="path to file containing list of desktop file names to be additionally checked. Format is: '$package_name $desktop_name [$desktop_name ...]'. One package per line, multiple desktop files may be specified.")
parser.add_argument("--additionalpackages", "-g", type=str, help="path to file containing list of packages to be unpacked in addition to main package. Format is: '$package_name $additional_package_name [$additional_package_name ...]. One main package name per line, multiple additional packages per line.")
parser.add_argument("--skiplanguageslist", "-l", type=str, help="path to file containing list of packages and languages to be skipped for specified packages. Format is: '$package_name $skipped_language [$skipped_language ...]'. One package per line, multiple languages for specified package per line.")
parser.add_argument("--origin", "-o", type=str, default="altlinux", help="origin of metadata, by default 'altlinux'")
parser.add_argument("--iconsizes", "-i", type=int, action='append', help="sizes of icons to copy, may be specified multiple times, default is 64 and 128")
parser.add_argument("--applicationsdir", "-a", type=str, action='append',
	default=[
		"/usr/share/applications/",
		"/usr/share/applications/kde/",
		"/usr/share/applications/kde4/",
		"/usr/share/applications/kf5/",
		"/usr/share/kde4/applications/",
		"/usr/share/kde4/applications/kde4/",
		"/usr/share/kde/applications/",
		"/usr/share/kde/applications/kde/",
		"/usr/share/kf5/applications/",
		"/usr/share/kf5/applications/kf5/",
		"/usr/share/mate/applications/",
		"/usr/share/tde3/applications/"
	],
	help="additional directories to look for .desktop files. Must end with '/' character.")
parser.add_argument("--iconsdir", "-d", type=str, action='append',
	default=[
		"/usr/share/icons/",
		"/usr/share/kde4/icons/",
		"/usr/share/kde/icons/",
		"/usr/share/kf5/icons/",
		"/usr/share/icons/mate/",
		"/usr/share/tde3/icons/"
	],
	help="additional directories to look for icon files. Must end with '/' character.")
parser.add_argument("--icontheme", "-t", type=str, action='append',
	default=[
		"hicolor",
		"breeze",
		"oxygen",
		"Adwaita",
		"gnome"
	],
	help="additional icon themes to check icons in")
parser.add_argument("--pixmapsdir", "-p", type=str, action='append',
	default=[
		"/usr/share/pixmaps/"
	],
	help="additional directories to look for pixmap files. Must end with '/' character.")
parser.add_argument("--usedesktopfiles", "-u", action='store_true', help="use desktop files to generate appdata if package doesn't contain appdata")
parser.add_argument("--userpmlicense", "-r", action='store_true', help="add license from rpm tag to generated appdata. Only works for appdata generated from desktop file")
parser.add_argument("--desktopfileskiplist", type=str, help="path to file containing names of desktop files to be skipped. Only useful when appdata is generated from desktop files. Format is: full filename, one item per line")
parser.add_argument("--desktopfileexclusivelist", type=str, help="path to file containing names of desktop files to be exclusively used. Only useful when appdata is generated from desktop files. Format is: full filename, one item per line")
parser.add_argument("--allownoicons", action='store_true', help="convert errors about missing icons to warnings")
parser.add_argument("--verbose", "-v", action='store_true', help="increase output verbosity")
parser.add_argument("--nocleanup", "-k", action='store_true', help="don't remove temporary data")
parser.add_argument("--generated_metadata_license", "-m", type=str, default="CC0-1.0", help="Generated metadata license. If empty value is specified, generated metadata license tag is omitted. Only used for metadata generated from desktop files. Default is 'CC0-1.0'.")
args = parser.parse_args()

outputdir = args.output
repositorypaths = args.repository
skiplistfilename = args.skiplist
exclusivelistfilename = args.exclusivelist
nameconvertfilename = args.nameconvert
additionaldesktopnamesfilename = args.additionaldesktopnames
additionalpackagesfilename = args.additionalpackages
skiplanguageslistfilename = args.skiplanguageslist
originname = args.origin
appdirs = args.applicationsdir
iconsdirs = args.iconsdir
iconthemes = args.icontheme
pixmapdirs = args.pixmapsdir
usedesktopfiles = args.usedesktopfiles
allownoicons = args.allownoicons
verbose = args.verbose
nocleanup = args.nocleanup
generated_metadata_license = args.generated_metadata_license
userpmlicense = args.userpmlicense
desktopfileskiplistfilename = args.desktopfileskiplist
desktopfileexclusivelistfilename = args.desktopfileexclusivelist

iconsizes = set()
if args.iconsizes is None:
	iconsizes.add(64)
	iconsizes.add(128)
else:
	for size in args.iconsizes:
		iconsizes.add(size)

# check that repository contains base/contents_index file
for path in repositorypaths:
	if (not os.path.isdir(path)) or (not os.path.isfile(os.path.join(path, 'base', 'contents_index'))):
		print("ERROR: Path is not a valid repository: {}".format(path), file=sys.stderr)
		sys.exit(-1)

ensure_directory(outputdir)
ensure_directory(os.path.join(outputdir, "xmls"))
ensure_directory(os.path.join(outputdir, "icons"))

skiplist = read_lines_from_file(skiplistfilename)
exclusivelist = read_lines_from_file(exclusivelistfilename)
exclusivelist_encountered = set()
nameconvert = read_lines_from_file_and_split_two(nameconvertfilename)
additionaldesktopnames = read_lines_from_file_and_split_multiple(additionaldesktopnamesfilename)
additionalpackages = read_lines_from_file_and_split_multiple(additionalpackagesfilename)
skiplanguageslist = read_lines_from_file_and_split_multiple(skiplanguageslistfilename)
desktopfileskiplist = read_lines_from_file(desktopfileskiplistfilename)
desktopfileexclusive = read_lines_from_file(desktopfileexclusivelistfilename)
desktopfileexclusive_encountered = set()

packages = []
addonpackages = []
desktoppackages = {}
processedpackages = set()
processedaddonpackages = set()
entries_appdata_set = []
entries_metainfo_set = []
appdata_regex = re.compile("^.*/([^/]+)(?:\.appdata\.xml|\.metainfo\.xml)$")
found_desktop_files = {}
found_appdata_packages = []

# find all interesting files in repositories and save package names
for path in repositorypaths:
	if verbose:
		print("Reading file {}".format(os.path.join(path, 'base', 'contents_index')))
	with open(os.path.join(path, 'base', 'contents_index'), encoding="latin-1") as contents_file:
		for line in contents_file:
			if line.startswith("/usr/share/appdata/") or line.startswith("/usr/share/metainfo/"):
				filename, package = line.rstrip().rsplit('\t', 1)
				if filename.endswith(".appdata.xml") or filename.endswith(".metainfo.xml"):
					if not should_skip(package, skiplist):
						appdata_match = appdata_regex.match(filename)
						if not appdata_match:
							break

						found_appdata_packages.append(package)
						if filename.endswith(".appdata.xml"):
							packages.append((path, package, filename))
						else:
							addonpackages.append((path, package, filename))

			if usedesktopfiles:
				found_appdir_entry = False
				for appdir_entry in appdirs:
					if line.startswith(appdir_entry):
						found_appdir_entry = True
						break
				if found_appdir_entry:
					filename, package = line.rstrip().rsplit('\t', 1)
					if filename.endswith(".desktop") and (filename not in desktopfileskiplist):
						if package not in found_desktop_files:
							found_desktop_files[package] = {}
							found_desktop_files[package]["repo"] = path
							found_desktop_files[package]["files"] = []
						found_desktop_files[package]["files"].append(filename)

	if usedesktopfiles:
		for desktop_pkg_name in found_desktop_files:
			if (desktop_pkg_name not in found_appdata_packages) and (not should_skip(desktop_pkg_name, skiplist)):
				desktoppackages[desktop_pkg_name] = found_desktop_files[desktop_pkg_name]

tempdirname = tempfile.mkdtemp()
if nocleanup:
	if verbose:
		print("Temp dir: {}".format(tempdirname))
else:
	atexit.register(shutil.rmtree, tempdirname)

listfilesdir = os.path.join(tempdirname, "listfiles")
packagefilesdir = os.path.join(tempdirname, "packagefiles")
listaddonsfilesdir = os.path.join(tempdirname, "addonlistfiles")
packageaddonsfilesdir = os.path.join(tempdirname, "packageaddonfiles")
packagedesktopfilesdir = os.path.join(tempdirname, "packagedesktopfiles")
convertedxmldir = os.path.join(tempdirname, "convertedxml")
output_icons_dir = os.path.join(outputdir, "icons", originname)

os.mkdir(listfilesdir)
os.mkdir(packagefilesdir)
os.mkdir(listaddonsfilesdir)
os.mkdir(packageaddonsfilesdir)
os.mkdir(packagedesktopfilesdir)
os.mkdir(convertedxmldir)

# process .appdata.xml files
for packageitem in packages:
	package_name = packageitem[1]
	package_repo_path = packageitem[0]
	if exclusivelist:
		if package_name not in exclusivelist:
			continue
		exclusivelist_encountered.add(package_name)

	if package_name not in processedpackages:
		# don't process same package twice
		# useful in case of package containing multiple appdata files
		processedpackages.add(package_name)

		packagefilesdirname = os.path.join(packagefilesdir, package_name)
		convertedxmldirname = os.path.join(convertedxmldir, package_name)
		ensure_directory(packagefilesdirname)
		ensure_directory(convertedxmldirname)

		rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], ".appdata.xml", packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

		if package_name in additionalpackages:
			for package in additionalpackages[package_name]:
				if package:
					unpack_package(package, repositorypaths, ".appdata.xml", packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

		for xmlfilename in xmlfileslist:
			# if parsing file fails, report it, but continue processing remaining files
			if verbose:
				print("Processing xml file {}".format(xmlfilename))
			try:
				xmlfile_name = os.path.join(packagefilesdirname, xmlfilename.strip(os.sep))
				xmlinfo_root = ET.parse(xmlfile_name)
				xmlinfo_processed = xmlinfo_root.getroot()

				appended_pkgname = convert_package_name(package_name, nameconvert)

				desktopfilenames = set()
				for id_element in xmlinfo_processed.iterfind('id', xmlinfo_processed.nsmap):
					desktopfilenames.add(id_element.text)
				for launchable_element in xmlinfo_processed.iterfind('launchable', xmlinfo_processed.nsmap):
					desktopfilenames.add(launchable_element.text)
				if package_name in additionaldesktopnames:
					for desktopname in additionaldesktopnames[package_name]:
						if desktopname:
							desktopfilenames.add(desktopname)

				if not desktopfilenames:
					print("ERROR: Failed to parse file {} from package {}: couldn't find desktop file name".format(xmlfilename, package_name), file=sys.stderr)
					continue

				appended_release = str(rpmhdr[rpm.RPMTAG_BUILDTIME]) + ";" + str(rpmhdr[rpm.RPMTAG_VERSION].decode())

				iconname = None

				for desktopfilename in desktopfilenames:
					desktopfilename = os.path.basename(desktopfilename)
					desktopfilenameprocessed = desktopfilename
					if not desktopfilenameprocessed.endswith(".desktop"):
						desktopfilenameprocessed = desktopfilenameprocessed + ".desktop"

					for dirname in appdirs:
						desktopfilepath = os.path.join(packagefilesdirname, dirname.strip(os.sep), desktopfilenameprocessed)

						if os.path.isfile(desktopfilepath):
							try:
								config = xdg.DesktopEntry.DesktopEntry(desktopfilepath)
								iconname = config.get("Icon")
							except:
								print("ERROR: Failed to parse file {} from package {}".format(desktopfilepath, package_name), file=sys.stderr)
								pass
							if iconname is not None:
								break

					if iconname is not None:
						break

				if iconname is None:
					print("ERROR: Failed to parse files {} from package {}: couldn't find icon file name".format(desktopfilenames, package_name), file=sys.stderr)
					continue

				basic_iconname = remove_known_image_ext(os.path.basename(iconname))
				basic_icon_extensions = [ ".png", ".svg", ".xpm" ]

				iconsdirsglob = [ os.path.join(packagefilesdirname, iconname.strip(os.sep)) ]

				for iconsdir in iconsdirs:
					for icon_theme in iconthemes:
						theme_dirs = [ "*/apps" ] # fallback value in case no index.theme is present
						themefilename = os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, "index.theme")
						if os.path.isfile(themefilename):
							theme_dirs = find_icon_theme_dirs(themefilename)
						for theme_dir in theme_dirs:
							for extension in basic_icon_extensions:
								iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, theme_dir, basic_iconname + extension))

				for iconsdir in iconsdirs:
					for icon_theme in iconthemes:
						for extension in basic_icon_extensions:
							iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, basic_iconname + extension))

				for iconsdir in iconsdirs:
					for extension in basic_icon_extensions:
						iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), basic_iconname + extension))

				for pixmapdir in pixmapdirs:
					for extension in basic_icon_extensions:
						iconsdirsglob.append(os.path.join(packagefilesdirname, pixmapdir.strip(os.sep), basic_iconname + extension))

				icons = find_icons(iconsdirsglob)
				if not icons:
					if not allownoicons:
						print("ERROR: Failed to find icons for file {} from package {}".format(desktopfilename, package_name), file=sys.stderr)
						continue
					else:
						print("Warning: Failed to find icons for file {} from package {}".format(desktopfilename, package_name), file=sys.stderr)

				if icons:
					result = copy_icons(icons, output_icons_dir, basic_iconname + ".png", iconsizes)
					if not result:
						if not allownoicons:
							print("ERROR: Failed to copy all icons for file {} from package {}".format(desktopfilename, package_name), file=sys.stderr)
							continue
						else:
							print("Warning: Failed to copy all icons for file {} from package {}".format(desktopfilename, package_name), file=sys.stderr)

				copied_icons_data = []
				if icons:
					for iconsize in iconsizes:
						copied_icons_data.append("cached;" + str(iconsize) + ";" + str(iconsize) + ";" + basic_iconname + ".png")
					copied_icons_data = prefix_array_with_elements("--icon", copied_icons_data)

				skipped_languages = []
				if package_name in skiplanguageslist:
					skipped_languages = prefix_array_with_elements("--exclude_language", skiplanguageslist[package_name])

				outputxmlname = os.path.join(convertedxmldirname, os.path.basename(xmlfilename))
				res = subprocess.call(["appstream-data-appdata-converter", xmlfile_name, outputxmlname, desktopfilepath, appended_pkgname, "--release", appended_release] + skipped_languages + copied_icons_data)
				if res == 0:
					entries_appdata_set.append(outputxmlname)
				else:
					print("ERROR: Failed to process file {} from package {}".format(xmlfilename, package_name), file=sys.stderr)

			except ET.ParseError:
				print("ERROR: Failed to parse file {} from package {}".format(xmlfilename, package_name), file=sys.stderr)

# process .metainfo.xml files
for packageitem in addonpackages:
	package_name = packageitem[1]
	package_repo_path = packageitem[0]
	if exclusivelist:
		if package_name not in exclusivelist:
			continue
		exclusivelist_encountered.add(package_name)

	if package_name not in processedaddonpackages:
		# don't process same package twice
		# useful in case of package containing multiple appdata files
		processedaddonpackages.add(package_name)

		packagefilesdirname = os.path.join(packageaddonsfilesdir, package_name)
		convertedxmldirname = os.path.join(convertedxmldir, package_name)
		ensure_directory(packagefilesdirname)
		ensure_directory(convertedxmldirname)

		rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], ".metainfo.xml", packagefilesdirname, listaddonsfilesdir, [])

		for xmlfilename in xmlfileslist:
			# if parsing file fails, report it, but continue processing remaining files
			if verbose:
				print("Processing xml file {}".format(xmlfilename))
			try:
				xmlfile_name = os.path.join(packagefilesdirname, xmlfilename.strip(os.sep))

				xmlinfo_root = ET.parse(xmlfile_name)
				xmlinfo_processed = xmlinfo_root.getroot()

				# "font" or other types are currently not supported
				if xmlinfo_processed.attrib.get("type") != "addon":
					print("Warning: skipping file {} from package {}: unsupported type {}".format(xmlfilename, package_name, xmlinfo_processed.attrib.get("type")), file=sys.stderr)
					continue

				outputxmlname = os.path.join(convertedxmldirname, os.path.basename(xmlfilename))
				res = subprocess.call(["appstream-data-metainfo-converter", xmlfile_name, outputxmlname, convert_package_name(package_name, nameconvert)])
				if res == 0:
					entries_metainfo_set.append(outputxmlname)
				else:
					print("ERROR: Failed to process file {} from package {}".format(xmlfilename, package_name), file=sys.stderr)

			except ET.ParseError:
				print("ERROR: Failed to parse file {} from package {}".format(xmlfilename, package_name), file=sys.stderr)

# process .desktop files
if usedesktopfiles:
	for package_name in desktoppackages:
		package_repo_path = desktoppackages[package_name]["repo"]
		if exclusivelist:
			if package_name not in exclusivelist:
				continue
			exclusivelist_encountered.add(package_name)

		packagefilesdirname = os.path.join(packagedesktopfilesdir, package_name)
		convertedxmldirname = os.path.join(convertedxmldir, package_name)
		ensure_directory(packagefilesdirname)
		ensure_directory(convertedxmldirname)

		rpmhdr, xmlfileslist = unpack_package(package_name, [ package_repo_path ], None, packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

		if package_name in additionalpackages:
			for package in additionalpackages[package_name]:
				if package:
					unpack_package(package, repositorypaths, None, packagefilesdirname, listfilesdir, appdirs + iconsdirs + pixmapdirs)

		for found_desktop_name in desktoppackages[package_name]["files"]:
			if desktopfileexclusive:
				if found_desktop_name not in desktopfileexclusive:
					continue
			desktopfileexclusive_encountered.add(found_desktop_name)

			if verbose:
				print("Processing desktop file {}".format(found_desktop_name))

			appended_pkgname = convert_package_name(package_name, nameconvert)
			appended_release = str(rpmhdr[rpm.RPMTAG_BUILDTIME]) + ";" + str(rpmhdr[rpm.RPMTAG_VERSION].decode())

			appended_url = None
			if rpm.RPMTAG_URL in rpmhdr:
				appended_url = str(rpmhdr[rpm.RPMTAG_URL].decode())

			appended_description = None
			if rpm.RPMTAG_DESCRIPTION in rpmhdr:
				appended_description = str(rpmhdr[rpm.RPMTAG_DESCRIPTION].decode())

			appended_project_license = None
			if userpmlicense and (rpm.RPMTAG_LICENSE in rpmhdr):
				appended_project_license = str(rpmhdr[rpm.RPMTAG_LICENSE].decode())

			desktopfilepath = os.path.join(packagefilesdirname, found_desktop_name.strip(os.sep))
			if not os.path.isfile(desktopfilepath):
				print("ERROR: Failed to find file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)
				continue

			iconname = None

			try:
				config = xdg.DesktopEntry.DesktopEntry(desktopfilepath)
				iconname = config.get("Icon")
			except:
				print("ERROR: Failed to parse file {} from package {}".format(desktopfilepath, package_name), file=sys.stderr)
				pass

			if iconname is None:
				print("ERROR: Failed to parse file {} from package {}: couldn't find icon file name".format(found_desktop_name, package_name), file=sys.stderr)
				continue

			basic_iconname = remove_known_image_ext(os.path.basename(iconname))
			basic_icon_extensions = [ ".png", ".svg", ".xpm" ]

			iconsdirsglob = [ os.path.join(packagefilesdirname, iconname.strip(os.sep)) ]

			for iconsdir in iconsdirs:
				for icon_theme in iconthemes:
					theme_dirs = [ "*/apps" ] # fallback value in case no index.theme is present
					themefilename = os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, "index.theme")
					if os.path.isfile(themefilename):
						theme_dirs = find_icon_theme_dirs(themefilename)
					for theme_dir in theme_dirs:
						for extension in basic_icon_extensions:
							iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, theme_dir, basic_iconname + extension))

			for iconsdir in iconsdirs:
				for icon_theme in iconthemes:
					for extension in basic_icon_extensions:
						iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), icon_theme, basic_iconname + extension))

			for iconsdir in iconsdirs:
				for extension in basic_icon_extensions:
					iconsdirsglob.append(os.path.join(packagefilesdirname, iconsdir.strip(os.sep), basic_iconname + extension))

			for pixmapdir in pixmapdirs:
				for extension in basic_icon_extensions:
					iconsdirsglob.append(os.path.join(packagefilesdirname, pixmapdir.strip(os.sep), basic_iconname + extension))

			icons = find_icons(iconsdirsglob)
			if not icons:
				if not allownoicons:
					print("ERROR: Failed to find icons for file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)
					continue
				else:
					print("Warning: Failed to find icons for file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)

			if icons:
				result = copy_icons(icons, output_icons_dir, basic_iconname + ".png", iconsizes)
				if not result:
					if not allownoicons:
						print("ERROR: Failed to copy all icons for file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)
						continue
					else:
						print("Warning: Failed to copy all icons for file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)

			copied_icons_data = []
			if icons:
				for iconsize in iconsizes:
					copied_icons_data.append("cached;" + str(iconsize) + ";" + str(iconsize) + ";" + basic_iconname + ".png")
				copied_icons_data = prefix_array_with_elements("--icon", copied_icons_data)

			skipped_languages = []
			if package_name in skiplanguageslist:
				skipped_languages = prefix_array_with_elements("--exclude_language", skiplanguageslist[package_name])

			cmdline_url = []
			if appended_url:
				cmdline_url.append("--url")
				cmdline_url.append(appended_url)

			cmdline_descr = []
			if appended_url:
				cmdline_descr.append("--description")
				cmdline_descr.append(appended_description)

			cmdline_metadata_license = []
			if generated_metadata_license:
				cmdline_metadata_license.append("--metadata_license")
				cmdline_metadata_license.append(generated_metadata_license)

			cmdline_project_license = []
			if appended_project_license:
				cmdline_project_license.append("--project_license")
				cmdline_project_license.append(appended_project_license)

			outputxmlname = os.path.join(convertedxmldirname, os.path.basename(found_desktop_name) + ".appdata.xml")
			res = subprocess.call(["appstream-data-desktop-converter", desktopfilepath, outputxmlname, appended_pkgname, "--release", appended_release] + cmdline_url + cmdline_descr + cmdline_metadata_license + cmdline_project_license + skipped_languages + copied_icons_data)
			if res == 0:
				entries_appdata_set.append(outputxmlname)
			else:
				print("ERROR: Failed to process file {} from package {}".format(found_desktop_name, package_name), file=sys.stderr)

if exclusivelist:
	for item in exclusivelist:
		if item not in exclusivelist_encountered:
			print("ERROR: Package {} from exclusive list not found".format(item))

if desktopfileexclusive:
	for item in desktopfileexclusive:
		if item not in desktopfileexclusive_encountered:
			print("ERROR: Desktop file {} from desktop files exclusive list not found".format(item))

if (not entries_appdata_set) and (not entries_metainfo_set):
	print("ERROR: no appdata files processed", file=sys.stderr)
	sys.exit(-1)

subprocess.call(["appstream-data-composer", os.path.join(outputdir, "xmls", originname + ".xml"), "--extra_tag", "origin;" + originname, "--extra_tag", "version;0.8"] + sorted(entries_appdata_set) + sorted(entries_metainfo_set))
