On some servers attached to a landscape instance, I encountered this stacktrace when trying to run sudo landscape-package-reporter:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/landscape/client/package/reporter.py", line 92, in <lambda>
result.addCallback(lambda x: self.request_unknown_hashes())
File "/usr/lib/python3/dist-packages/landscape/client/package/reporter.py", line 485, in request_unknown_hashes
self._facade.ensure_channels_reloaded()
File "/usr/lib/python3/dist-packages/landscape/lib/apt/package/facade.py", line 265, in ensure_channels_reloaded
self.reload_channels()
File "/usr/lib/python3/dist-packages/landscape/lib/apt/package/facade.py", line 253, in reload_channels
version, with_info=False).get_hash()
File "/usr/lib/python3/dist-packages/landscape/lib/apt/package/facade.py", line 402, in get_package_skeleton
return build_skeleton_apt(pkg, with_info=with_info, with_unicode=True)
File "/usr/lib/python3/dist-packages/landscape/lib/apt/package/skeleton.py", line 131, in build_skeleton_apt
version.record, "Provides", DEB_PROVIDES))
File "/usr/lib/python3/dist-packages/apt/package.py", line 690, in record
return Record(self._records.record)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x96 in position 724: invalid start byteTracing down the issue, it was related with a misplaced set of Unicode bytes (EB BF BD) in an old veeam version in version 1.0.0.944 of the veeamsnap package in /var/lib/apt/lists/repository.veeam.com_backup_linux_agent_dpkg_debian_public_dists_stable_veeam_binary-amd64_Packages: The Description field contains this text:
[...] Linux � simple [...]
The strange character is the U+FFFD � REPLACEMENT CHARACTER.
You can fix it by deleting this character. It’s just at the end of /var/lib/apt/lists/repository.veeam.com_backup_linux_agent_dpkg_debian_public_dists_stable_veeam_binary-amd64_Packages. However, if there’s an update for that repository, your change will be overwritten.
In order to fix it (my fix is for landscape-client version 18.01-0ubuntu3.5), I added a try: ... except: ... clause to skeleton.py, which will ignore some properties of the package where the issue occurs:
try:
relations.update(parse_record_field(
version.record, "Provides", DEB_PROVIDES))
relations.add((
DEB_NAME_PROVIDES,
"%s = %s" % (version.package.name, version.version)))
relations.update(parse_record_field(
version.record, "Pre-Depends", DEB_REQUIRES, DEB_OR_REQUIRES))
relations.update(parse_record_field(
version.record, "Depends", DEB_REQUIRES, DEB_OR_REQUIRES))
relations.add((
DEB_UPGRADES, "%s < %s" % (version.package.name, version.version)))
relations.update(parse_record_field(
version.record, "Conflicts", DEB_CONFLICTS))
relations.update(parse_record_field(
version.record, "Breaks", DEB_CONFLICTS))
skeleton.relations = sorted(relations)
if with_info:
skeleton.section = version.section
skeleton.summary = version.summary
skeleton.description = version.description
skeleton.size = version.size
if version.installed_size > 0:
skeleton.installed_size = version.installed_size
if with_unicode and not _PY3:
skeleton.section = skeleton.section.decode("utf-8")
skeleton.summary = skeleton.summary.decode("utf-8")
# Avoid double-decoding package descriptions in build_skeleton_apt,
# which causes an error with newer python-apt (Xenial onwards)
if not isinstance(skeleton.description, unicode):
skeleton.description = skeleton.description.decode("utf-8")
return skeleton
except UnicodeError:
return skeletonReplace /usr/lib/python3/dist-packages/landscape/lib/apt/package/skeleton.py by this:
from landscape.lib.hashlib import sha1
import apt_pkg
from twisted.python.compat import unicode, _PY3
PACKAGE = 1 << 0
PROVIDES = 1 << 1
REQUIRES = 1 << 2
UPGRADES = 1 << 3
CONFLICTS = 1 << 4
DEB_PACKAGE = 1 << 16 | PACKAGE
DEB_PROVIDES = 2 << 16 | PROVIDES
DEB_NAME_PROVIDES = 3 << 16 | PROVIDES
DEB_REQUIRES = 4 << 16 | REQUIRES
DEB_OR_REQUIRES = 5 << 16 | REQUIRES
DEB_UPGRADES = 6 << 16 | UPGRADES
DEB_CONFLICTS = 7 << 16 | CONFLICTS
class PackageTypeError(Exception):
"""Raised when an unsupported package type is passed to build_skeleton."""
class PackageSkeleton(object):
section = None
summary = None
description = None
size = None
installed_size = None
_hash = None
def __init__(self, type, name, version):
self.type = type
self.name = name
self.version = version
self.relations = []
def add_relation(self, type, info):
self.relations.append((type, info))
def get_hash(self):
"""Calculate the package hash.
If C{set_hash} has been used, that hash will be returned and the
hash won't be the calculated value.
"""
if self._hash is not None:
return self._hash
# We use ascii here as encoding for backwards compatibility as it was
# default encoding for conversion from unicode to bytes in Python 2.7.
package_info = ("[%d %s %s]" % (self.type, self.name, self.version)
).encode("ascii")
digest = sha1(package_info)
self.relations.sort()
for pair in self.relations:
digest.update(("[%d %s]" % (pair[0], pair[1])
).encode("ascii"))
return digest.digest()
def set_hash(self, package_hash):
"""Set the hash to an explicit value.
This should be used when the hash is previously known and can't
be calculated from the relations anymore.
The only use case for this is package resurrection. We're
planning on getting rid of package resurrection, and this code
can be removed when that is done.
"""
self._hash = package_hash
def relation_to_string(relation_tuple):
"""Convert an apt relation to a string representation.
@param relation_tuple: A tuple, (name, version, relation). version
and relation can be the empty string, if the relation is on a
name only.
Returns something like "name > 1.0"
"""
name, version, relation_type = relation_tuple
relation_string = name
if relation_type:
relation_string += " %s %s" % (relation_type, version)
return relation_string
def parse_record_field(record, record_field, relation_type,
or_relation_type=None):
"""Parse an apt C{Record} field and return skeleton relations
@param record: An C{apt.package.Record} instance with package information.
@param record_field: The name of the record field to parse.
@param relation_type: The deb relation that can be passed to
C{skeleton.add_relation()}
@param or_relation_type: The deb relation that should be used if
there is more than one value in a relation.
"""
relations = set()
values = apt_pkg.parse_depends(record.get(record_field, ""))
for value in values:
value_strings = [relation_to_string(relation) for relation in value]
value_relation_type = relation_type
if len(value_strings) > 1:
value_relation_type = or_relation_type
relation_string = " | ".join(value_strings)
relations.add((value_relation_type, relation_string))
return relations
def build_skeleton_apt(version, with_info=False, with_unicode=False):
"""Build a package skeleton from an apt package.
@param version: An instance of C{apt.package.Version}
@param with_info: Whether to extract extra information about the
package, like description, summary, size.
@param with_unicode: Whether the C{name} and C{version} of the
skeleton should be unicode strings.
"""
name, version_string = version.package.name, version.version
if with_unicode:
name, version_string = unicode(name), unicode(version_string)
skeleton = PackageSkeleton(DEB_PACKAGE, name, version_string)
relations = set()
try:
relations.update(parse_record_field(
version.record, "Provides", DEB_PROVIDES))
relations.add((
DEB_NAME_PROVIDES,
"%s = %s" % (version.package.name, version.version)))
relations.update(parse_record_field(
version.record, "Pre-Depends", DEB_REQUIRES, DEB_OR_REQUIRES))
relations.update(parse_record_field(
version.record, "Depends", DEB_REQUIRES, DEB_OR_REQUIRES))
relations.add((
DEB_UPGRADES, "%s < %s" % (version.package.name, version.version)))
relations.update(parse_record_field(
version.record, "Conflicts", DEB_CONFLICTS))
relations.update(parse_record_field(
version.record, "Breaks", DEB_CONFLICTS))
skeleton.relations = sorted(relations)
if with_info:
skeleton.section = version.section
skeleton.summary = version.summary
skeleton.description = version.description
skeleton.size = version.size
if version.installed_size > 0:
skeleton.installed_size = version.installed_size
if with_unicode and not _PY3:
skeleton.section = skeleton.section.decode("utf-8")
skeleton.summary = skeleton.summary.decode("utf-8")
# Avoid double-decoding package descriptions in build_skeleton_apt,
# which causes an error with newer python-apt (Xenial onwards)
if not isinstance(skeleton.description, unicode):
skeleton.description = skeleton.description.decode("utf-8")
return skeleton
except UnicodeError:
return skeleton
After that, you can run sudo landscape-package-reporter again.
