From fb9b56c453f1c93b617b2b2d98e8bb4db005c8f7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 20 Oct 2025 20:47:26 -0400 Subject: [PATCH 01/21] fix tag handling Signed-off-by: Marcus Furlong --- hosts/models.py | 2 +- hosts/utils.py | 19 +++++++++++++++++-- sbin/patchman | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index a6c451b5a..5b7b3979f 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -85,7 +85,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {self.tags}\n' + text += f'Tags : {" ".join(self.tags.slugs())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' diff --git a/hosts/utils.py b/hosts/utils.py index b328129f2..f07d5d1e5 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -18,8 +18,9 @@ from socket import gethostbyaddr, gaierror, herror from django.db import transaction, IntegrityError +from taggit.models import Tag -from patchman.signals import error_message +from patchman.signals import error_message, info_message def update_rdns(host): @@ -62,7 +63,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.osvariant = osvariant host.domain = domain host.lastreport = report.created - host.tags = report.tags + host.tags.set(report.tags.split(','), clear=True) if report.reboot == 'True': host.reboot_required = True else: @@ -73,3 +74,17 @@ def get_or_create_host(report, arch, osvariant, domain): if host: host.check_rdns() return host + + +def clean_tags(): + """ Delete Tags that have no Host + """ + tags = Tag.objects.filter( + host__isnull=True, + ) + tlen = tags.count() + if tlen == 0: + info_message.send(sender=None, text='No orphaned Tags found.') + else: + info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + tags.delete() diff --git a/sbin/patchman b/sbin/patchman index 9cc6048e5..c09114340 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -34,6 +34,7 @@ from errata.utils import mark_errata_security_updates, enrich_errata, \ scan_package_updates_for_affected_packages from errata.tasks import update_errata from hosts.models import Host +from hosts.utils import clean_tags from modules.utils import clean_modules from packages.utils import clean_packages, clean_packageupdates, clean_packagenames from repos.models import Repository @@ -362,6 +363,7 @@ def dbcheck(remove_duplicates=False): clean_repos() clean_modules() clean_packageupdates() + clean_tags() def collect_args(): From 2bcd4dac0bd7dd81f4ebe4aa483532aec04a8650 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:45 -0400 Subject: [PATCH 02/21] fix package filter list for errata --- packages/views.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/views.py b/packages/views.py index cd53fa6e0..c55a6c72c 100644 --- a/packages/views.py +++ b/packages/views.py @@ -62,9 +62,16 @@ def package_list(request): if 'affected_by_errata' in request.GET: affected_by_errata = request.GET['affected_by_errata'] == 'true' if affected_by_errata: - packages = packages.filter(erratum__isnull=False) + packages = packages.filter(affected_by_erratum__isnull=False) else: - packages = packages.filter(erratum__isnull=True) + packages = packages.filter(affected_by_erratum__isnull=True) + + if 'provides_fix_in_erratum' in request.GET: + provides_fix_in_erratum = request.GET['provides_fix_in_erratum'] == 'true' + if provides_fix_in_erratum: + packages = packages.filter(provides_fix_in_erratum__isnull=False) + else: + packages = packages.filter(provides_fix_in_erratum__isnull=True) if 'installed_on_hosts' in request.GET: installed_on_hosts = request.GET['installed_on_hosts'] == 'true' @@ -102,6 +109,8 @@ def package_list(request): filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) + filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', + {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Installed on Hosts', 'installed_on_hosts', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Available in Repos', 'available_in_repos', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) From 856f41f9f7cdc8fbebb0c6c3685a014526efdc36 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:29 -0400 Subject: [PATCH 03/21] add support for zstd compression in deb and rpm repos fixes: #698 Signed-off-by: Marcus Furlong --- debian/control | 2 +- repos/repo_types/deb.py | 7 ++++++- repos/repo_types/rpm.py | 2 ++ requirements.txt | 1 + setup.cfg | 1 + util/__init__.py | 17 +++++++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 67026269f..7bfe320eb 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit + python3-redis, python3-git, python3-django-taggit, python3-zstandard Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 25d8eba75..1d3607c55 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -71,7 +71,12 @@ def refresh_deb_repo(repo): are and then fetches and extracts packages from those files. """ - formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages'] + formats = [ + 'Packages.xz', + 'Packages.bz2', + 'Packages.gz', + 'Packages', + ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index d9501cde9..93d470076 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -57,10 +57,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): which type of repo it is, then refreshes the mirrors """ formats = [ + 'repodata/repomd.xml.zst', 'repodata/repomd.xml.xz', 'repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', + 'suse/repodata/repomd.xml.zst', 'suse/repodata/repomd.xml.xz', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', diff --git a/requirements.txt b/requirements.txt index 2f264c9bd..788f42413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 +zstandard==0.25.0 diff --git a/setup.cfg b/setup.cfg index 7af9ccb0c..b1d5ee4e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ requires = /usr/bin/python3 python3-importlib-metadata python3-cvss python3-redis + python3-zstandard redis celery python3-django-celery-beat diff --git a/util/__init__.py b/util/__init__.py index ac6f8f1bd..4a3f9caaf 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -21,6 +21,11 @@ import zlib import lzma import os +try: + # python 3.14+ - can also remove the dependency at that stage + from compression import zstd +except ImportError: + import zstandard as zstd from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -202,6 +207,16 @@ def unxz(contents): error_message.send(sender=None, text='lzma: ' + e) +def unzstd(contents): + """ unzstd contents in memory and return the data + """ + try: + zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() + return zstddata + except zstd.ZstdError as e: + error_message.send(sender=None, text='zstd: ' + e) + + def extract(data, fmt): """ Extract the contents based on mimetype or file ending. Return the unmodified data if neither mimetype nor file ending matches, otherwise @@ -214,6 +229,8 @@ def extract(data, fmt): m = magic.open(magic.MAGIC_MIME) m.load() mime = m.buffer(data).split(';')[0] + if mime == 'application/zstd' or fmt.endswith('zst'): + return unzstd(data) if mime == 'application/x-xz' or fmt.endswith('xz'): return unxz(data) elif mime == 'application/x-bzip2' or fmt.endswith('bz2'): From 8d9da89ebccd367de221b891ad02f99b205d9c19 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:41 -0400 Subject: [PATCH 04/21] simplify logging --- arch/utils.py | 10 ++--- errata/models.py | 12 ++--- errata/sources/distros/arch.py | 5 ++- errata/sources/distros/centos.py | 5 ++- errata/sources/distros/debian.py | 7 +-- errata/sources/distros/rocky.py | 11 ++--- errata/sources/distros/ubuntu.py | 7 +-- errata/sources/repos/yum.py | 5 ++- errata/tasks.py | 2 +- errata/utils.py | 9 ++-- hosts/models.py | 14 +++--- hosts/tasks.py | 4 +- hosts/utils.py | 8 ++-- modules/utils.py | 8 ++-- packages/utils.py | 26 +++++------ patchman/receivers.py | 21 ++++----- patchman/signals.py | 8 ++-- reports/models.py | 10 ++--- reports/tasks.py | 4 +- reports/utils.py | 5 ++- repos/models.py | 18 ++++---- repos/repo_types/arch.py | 11 ++--- repos/repo_types/deb.py | 11 ++--- repos/repo_types/gentoo.py | 15 ++++--- repos/repo_types/rpm.py | 8 ++-- repos/repo_types/yast.py | 5 ++- repos/repo_types/yum.py | 23 +++++----- repos/utils.py | 37 ++++++++-------- sbin/patchman | 76 ++++++++++++++++---------------- security/models.py | 8 ++-- util/__init__.py | 24 +++++----- util/logging.py | 42 ++++++++++++++++++ 32 files changed, 258 insertions(+), 201 deletions(-) create mode 100644 util/logging.py diff --git a/arch/utils.py b/arch/utils.py index 1498fdec9..04d0b3506 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from arch.models import PackageArchitecture, MachineArchitecture -from patchman.signals import info_message +from util.logging import info_message def clean_package_architectures(): @@ -24,9 +24,9 @@ def clean_package_architectures(): parches = PackageArchitecture.objects.filter(package__isnull=True) plen = parches.count() if plen == 0: - info_message.send(sender=None, text='No orphaned PackageArchitectures found.') + info_message(text='No orphaned PackageArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures') + info_message(text=f'Removing {plen} orphaned PackageArchitectures') parches.delete() @@ -39,9 +39,9 @@ def clean_machine_architectures(): ) mlen = marches.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned MachineArchitectures found.') + info_message(text='No orphaned MachineArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures') + info_message(text=f'Removing {mlen} orphaned MachineArchitectures') marches.delete() diff --git a/errata/models.py b/errata/models.py index b10daf4dd..cfc9bd0df 100644 --- a/errata/models.py +++ b/errata/models.py @@ -25,7 +25,7 @@ from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from patchman.signals import error_message +from util.logging import error_message from util import get_url @@ -70,7 +70,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -84,7 +84,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -93,7 +93,7 @@ def fetch_osv_dev_data(self): osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}' res = get_url(osv_dev_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}') + error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}') return data = res.content osv_dev_json = json.loads(data) @@ -102,7 +102,7 @@ def fetch_osv_dev_data(self): def parse_osv_dev_data(self, osv_dev_json): name = osv_dev_json.get('id') if name != self.name: - error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}') + error_message(text=f'Erratum name mismatch - {self.name} != {name}') return related = osv_dev_json.get('related') if related: @@ -155,7 +155,7 @@ def add_cve(self, cve_id): """ Add a CVE to an Erratum object """ if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit(): - error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}') + error_message(text=f'Not a CVE ID: {cve_id}') return self.cves.add(get_or_create_cve(cve_id)) diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 40d0dadad..87c6c47a6 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,7 +20,8 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from packages.models import Package from packages.utils import find_evr, get_matching_packages, get_or_create_package from util import get_url, fetch_content @@ -99,7 +100,7 @@ def process_arch_erratum(advisory, osrelease): add_arch_erratum_references(e, advisory) add_arch_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_arch_linux_osrelease(): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index eefb2b887..d2722a6be 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -20,7 +20,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import parse_package_string, get_or_create_package -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type @@ -34,7 +35,7 @@ def update_centos_errata(): if actual_checksum != expected_checksum: e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) else: if data: parse_centos_errata(bunzip2(data)) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 93ae2bd54..1ae919e47 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -27,7 +27,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, find_evr -from patchman.signals import error_message, pbar_start, pbar_update, warning_message +from util.logging import error_message, warning_message +from patchman.signals import pbar_start, pbar_update from util import get_url, fetch_content, get_setting_of_type, extract DSCs = {} @@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames): for package in packages: process_debian_erratum_fixed_packages(e, package) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def parse_debian_erratum_package(line, accepted_codenames): @@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version): """ Fetch the package list from a DSC file for a given source package/version """ if not DSCs.get(package) or not DSCs[package].get(version): - warning_message.send(sender=None, text=f'No DSC found for {package} {version}') + warning_message(text=f'No DSC found for {package} {version}') return source_url = DSCs[package][version]['url'] res = get_url(source_url) diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 693d7b0cb..16d4d12c0 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -25,7 +25,8 @@ from packages.models import Package from packages.utils import parse_package_string, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, info_message, error_message +from util import get_url, fetch_content +from util.logging import info_message, error_message def update_rocky_errata(concurrent_processing=True): @@ -50,16 +51,16 @@ def check_rocky_errata_endpoint_health(rocky_errata_api_host): health = json.loads(data) if health.get('status') == 'ok': s = f'Rocky Errata API healthcheck OK: {rocky_errata_healthcheck_url}' - info_message.send(sender=None, text=s) + info_message(text=s) return True else: s = f'Rocky Errata API healthcheck FAILED: {rocky_errata_healthcheck_url}' - error_message.send(sender=None, text=s) + error_message(text=s) return False except Exception as e: s = f'Rocky Errata API healthcheck exception occured: {rocky_errata_healthcheck_url}\n' s += str(e) - error_message.send(sender=None, text=s) + error_message(text=s) return False @@ -194,7 +195,7 @@ def process_rocky_erratum(advisory): add_rocky_erratum_oses(e, advisory) add_rocky_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_rocky_erratum_references(e, advisory): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 7f50962ce..d1ce7cc58 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -28,7 +28,8 @@ from packages.models import Package from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update def update_ubuntu_errata(concurrent_processing=False): @@ -45,7 +46,7 @@ def update_ubuntu_errata(concurrent_processing=False): else: e = 'Ubuntu USN DB checksum mismatch, skipping Ubuntu errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) def fetch_ubuntu_usn_db(): @@ -126,7 +127,7 @@ def process_usn(usn_id, advisory, accepted_releases): add_ubuntu_erratum_references(e, usn_id, advisory) add_ubuntu_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_ubuntu_erratum_osreleases(e, affected_releases, accepted_releases): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index dfeed879a..f361d10e6 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -23,7 +23,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from patchman.signals import pbar_start, pbar_update, error_message +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url @@ -38,7 +39,7 @@ def extract_updateinfo(data, url, concurrent_processing=True): elen = root.__len__() updates = root.findall('update') except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing updateinfo file from {url} : {e}') + error_message(text=f'Error parsing updateinfo file from {url} : {e}') if concurrent_processing: extract_updateinfo_concurrently(updates, elen) else: diff --git a/errata/tasks.py b/errata/tasks.py index fe53b4151..f1d6eeee3 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -22,7 +22,7 @@ from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from patchman.signals import error_message +from util.logging import error_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type diff --git a/errata/utils.py b/errata/utils.py index d8099db4c..a8d8d4245 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -21,7 +21,8 @@ from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from patchman.signals import pbar_start, pbar_update, warning_message +from util.logging import warning_message +from patchman.signals import pbar_start, pbar_update def get_or_create_erratum(name, e_type, issue_date, synopsis): @@ -36,16 +37,16 @@ def get_or_create_erratum(name, e_type, issue_date, synopsis): days_delta = abs(e.issue_date.date() - issue_date_tz.date()).days updated = False if e.e_type != e_type: - warning_message.send(sender=None, text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') + warning_message(text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') e.e_type = e_type updated = True if days_delta > 1: text = f'Updating {name} issue date `{e.issue_date.date()}` -> `{issue_date_tz.date()}`' - warning_message.send(sender=None, text=text) + warning_message(text=text) e.issue_date = issue_date_tz updated = True if e.synopsis != synopsis: - warning_message.send(sender=None, text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') + warning_message(text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') e.synopsis = synopsis updated = True if updated: diff --git a/hosts/models.py b/hosts/models.py index 5b7b3979f..650544dca 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -34,7 +34,7 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from patchman.signals import info_message +from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo @@ -90,7 +90,7 @@ def show(self): text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) @@ -114,13 +114,13 @@ def check_rdns(self): if self.check_dns: update_rdns(self) if self.hostname.lower() == self.reversedns.lower(): - info_message.send(sender=None, text='Reverse DNS matches') + info_message(text='Reverse DNS matches') else: text = 'Reverse DNS mismatch found: ' text += f'{self.hostname} != {self.reversedns}' - info_message.send(sender=None, text=text) + info_message(text=text) else: - info_message.send(sender=None, text='Reverse DNS check disabled') + info_message(text='Reverse DNS check disabled') def clean_reports(self): """ Remove all but the last 3 reports for a host @@ -131,7 +131,7 @@ def clean_reports(self): for report in Report.objects.filter(host=self).order_by('-created')[3:]: report.delete() if rlen > 0: - info_message.send(sender=None, text=f'{self.hostname}: removed {rlen} old reports') + info_message(text=f'{self.hostname}: removed {rlen} old reports') def get_host_repo_packages(self): if self.host_repos_only: @@ -163,7 +163,7 @@ def process_update(self, package, highest_package): security = True update = get_or_create_package_update(oldpackage=package, newpackage=highest_package, security=security) self.updates.add(update) - info_message.send(sender=None, text=f'{update}') + info_message(text=f'{update}') return update.id def find_updates(self): diff --git a/hosts/tasks.py b/hosts/tasks.py index 2fdce96fb..1643901d1 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -20,7 +20,7 @@ from hosts.models import Host from util import get_datetime_now -from patchman.signals import info_message +from util.logging import info_message @shared_task @@ -78,4 +78,4 @@ def find_all_host_updates_homogenous(): phost.updated_at = ts phost.save() updated_hosts.append(phost) - info_message.send(sender=None, text=f'Added the same updates to {phost}') + info_message(text=f'Added the same updates to {phost}') diff --git a/hosts/utils.py b/hosts/utils.py index f07d5d1e5..d6e663cf8 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -20,7 +20,7 @@ from django.db import transaction, IntegrityError from taggit.models import Tag -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message def update_rdns(host): @@ -70,7 +70,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = False host.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) if host: host.check_rdns() return host @@ -84,7 +84,7 @@ def clean_tags(): ) tlen = tags.count() if tlen == 0: - info_message.send(sender=None, text='No orphaned Tags found.') + info_message(text='No orphaned Tags found.') else: - info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + info_message(text=f'{tlen} orphaned Tags found.') tags.delete() diff --git a/modules/utils.py b/modules/utils.py index f56a0f627..05c57c809 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from django.db import IntegrityError -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message from modules.models import Module from arch.models import PackageArchitecture @@ -37,7 +37,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): repo=repo, ) except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) module = Module.objects.get( name=name, stream=stream, @@ -73,7 +73,7 @@ def clean_modules(): ) mlen = modules.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned Modules found.') + info_message(text='No orphaned Modules found.') else: - info_message.send(sender=None, text=f'{mlen} orphaned Modules found.') + info_message(text=f'{mlen} orphaned Modules found.') modules.delete() diff --git a/packages/utils.py b/packages/utils.py index 9b0982253..f00f6710f 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -22,7 +22,7 @@ from arch.models import PackageArchitecture from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString -from patchman.signals import error_message, info_message, warning_message +from util.logging import error_message, info_message, warning_message def convert_package_to_packagestring(package): @@ -141,7 +141,7 @@ def parse_redhat_package_string(pkg_str): name, epoch, ver, rel, dist, arch = m.groups() else: e = f'Error parsing package string: "{pkg_str}"' - error_message.send(sender=None, text=e) + error_message(text=e) return if dist: rel = f'{rel}.{dist}' @@ -195,7 +195,7 @@ def get_or_create_package(name, epoch, version, release, arch, p_type): package = packages.first() # TODO this should handle gentoo package categories too, otherwise we may be deleting packages # that should be kept - warning_message.send(sender=None, text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') + warning_message(text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') packages.exclude(id=package.id).delete() return package @@ -218,10 +218,10 @@ def get_or_create_package_update(oldpackage, newpackage, security): except MultipleObjectsReturned: e = 'Error: MultipleObjectsReturned when attempting to add package \n' e += f'update with oldpackage={oldpackage} | newpackage={newpackage}:' - error_message.send(sender=None, text=e) + error_message(text=e) updates = PackageUpdate.objects.filter(oldpackage=oldpackage, newpackage=newpackage) for update in updates: - error_message.send(sender=None, text=str(update)) + error_message(text=str(update)) return try: if update: @@ -281,13 +281,13 @@ def clean_packageupdates(): for update in package_updates: if update.host_set.count() == 0: text = f'Removing unused PackageUpdate {update}' - info_message.send(sender=None, text=text) + info_message(text=text) update.delete() for duplicate in package_updates: if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ update.security == duplicate.security and update.id != duplicate.id: text = f'Removing duplicate PackageUpdate: {update}' - info_message.send(sender=None, text=text) + info_message(text=text) for host in duplicate.host_set.all(): host.updates.remove(duplicate) host.updates.add(update) @@ -307,12 +307,12 @@ def clean_packages(remove_duplicates=False): ) plen = packages.count() if plen == 0: - info_message.send(sender=None, text='No orphaned Packages found.') + info_message(text='No orphaned Packages found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned Packages') + info_message(text=f'Removing {plen} orphaned Packages') packages.delete() if remove_duplicates: - info_message.send(sender=None, text='Checking for duplicate Packages...') + info_message(text='Checking for duplicate Packages...') for package in Package.objects.all(): potential_duplicates = Package.objects.filter( name=package.name, @@ -326,7 +326,7 @@ def clean_packages(remove_duplicates=False): if potential_duplicates.count() > 1: for dupe in potential_duplicates: if dupe.id != package.id: - info_message.send(sender=None, text=f'Removing duplicate Package {dupe}') + info_message(text=f'Removing duplicate Package {dupe}') dupe.delete() @@ -336,7 +336,7 @@ def clean_packagenames(): names = PackageName.objects.filter(package__isnull=True) nlen = names.count() if nlen == 0: - info_message.send(sender=None, text='No orphaned PackageNames found.') + info_message(text='No orphaned PackageNames found.') else: - info_message.send(sender=None, text=f'Removing {nlen} orphaned PackageNames') + info_message(text=f'Removing {nlen} orphaned PackageNames') names.delete() diff --git a/patchman/receivers.py b/patchman/receivers.py index 5ec32cddb..8d8893cab 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -21,7 +21,8 @@ from django.dispatch import receiver from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update, \ + info_message_s, warning_message_s, error_message_s, debug_message_s from django.conf import settings @@ -47,36 +48,36 @@ def pbar_update_receiver(**kwargs): update_pbar(index) -@receiver(info_message) -def print_info_message(sender=None, **kwargs): - """ Receiver to print an info message, no color +@receiver(info_message_s) +def print_info_message(**kwargs): + """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.RESET_ALL + Fore.RESET + text) -@receiver(warning_message) +@receiver(warning_message_s) def print_warning_message(**kwargs): - """ Receiver to print a warning message in yellow text + """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.BRIGHT + Fore.YELLOW + text) -@receiver(error_message) +@receiver(error_message_s) def print_error_message(**kwargs): - """ Receiver to print an error message in red text + """ Receiver to handle an error message, red text """ text = str(kwargs.get('text')) if text: tqdm.write(Style.BRIGHT + Fore.RED + text) -@receiver(debug_message) +@receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to print a debug message in blue, if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if verbose and DEBUG are set """ text = str(kwargs.get('text')) if get_verbosity() and settings.DEBUG and text: diff --git a/patchman/signals.py b/patchman/signals.py index 917a48e47..799b9c98e 100644 --- a/patchman/signals.py +++ b/patchman/signals.py @@ -19,7 +19,7 @@ pbar_start = Signal() pbar_update = Signal() -info_message = Signal() -warning_message = Signal() -error_message = Signal() -debug_message = Signal() +info_message_s = Signal() +warning_message_s = Signal() +error_message_s = Signal() +debug_message_s = Signal() diff --git a/reports/models.py b/reports/models.py index 6818ea23f..d529804b9 100644 --- a/reports/models.py +++ b/reports/models.py @@ -19,7 +19,7 @@ from django.urls import reverse from hosts.utils import get_or_create_host -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message class Report(models.Model): @@ -97,11 +97,11 @@ def process(self, find_updates=True, verbose=False): """ Process a report and extract os, arch, domain, packages, repos etc """ if not self.os or not self.kernel or not self.arch: - error_message.send(sender=None, text=f'Error: OS, kernel or arch not sent with report {self.id}') + error_message(text=f'Error: OS, kernel or arch not sent with report {self.id}') return if self.processed: - info_message.send(sender=None, text=f'Report {self.id} has already been processed') + info_message(text=f'Report {self.id} has already been processed') return from reports.utils import get_arch, get_os, get_domain @@ -111,7 +111,7 @@ def process(self, find_updates=True, verbose=False): host = get_or_create_host(self, arch, osvariant, domain) if verbose: - info_message.send(sender=None, text=f'Processing report {self.id} - {self.host}') + info_message(text=f'Processing report {self.id} - {self.host}') from reports.utils import process_packages, process_repos, process_updates, process_modules process_repos(report=self, host=host) @@ -124,5 +124,5 @@ def process(self, find_updates=True, verbose=False): if find_updates: if verbose: - info_message.send(sender=None, text=f'Finding updates for report {self.id} - {self.host}') + info_message(text=f'Finding updates for report {self.id} - {self.host}') host.find_updates() diff --git a/reports/tasks.py b/reports/tasks.py index db9e41032..fe294e8d1 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -21,7 +21,7 @@ from hosts.models import Host from reports.models import Report -from util import info_message +from util.logging import info_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -48,5 +48,5 @@ def clean_reports_with_no_hosts(): for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): text = f'Deleting report {report.id} for Host `{report.host}` as the host no longer exists' - info_message.send(sender=None, text=text) + info_message(text=text) report.delete() diff --git a/reports/utils.py b/reports/utils.py index 641f90df2..76b6e09cb 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -26,7 +26,8 @@ from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant from packages.models import Package, PackageCategory from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.models import Repository, Mirror, MirrorPackage from repos.utils import get_or_create_repo @@ -93,7 +94,7 @@ def process_packages(report, host): host.packages.add(package) else: if pkg_str[0].lower() != 'gpg-pubkey': - info_message.send(sender=None, text=f'No package returned for {pkg_str}') + info_message(text=f'No package returned for {pkg_str}') pbar_update.send(sender=None, index=i + 1) for package in host.packages.all(): diff --git a/repos/models.py b/repos/models.py index 181a103d8..a1db2a934 100644 --- a/repos/models.py +++ b/repos/models.py @@ -26,7 +26,7 @@ from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from patchman.signals import info_message, warning_message, error_message +from util.logging import info_message, warning_message, error_message class Repository(models.Model): @@ -72,7 +72,7 @@ def show(self): text += f'arch: {self.arch}\n' text += 'Mirrors:' - info_message.send(sender=None, text=text) + info_message(text=text) for mirror in self.mirror_set.all(): mirror.show() @@ -99,10 +99,10 @@ def refresh(self, force=False): refresh_gentoo_repo(self) else: text = f'Error: unknown repo type for repo {self.id}: {self.repotype}' - error_message.send(sender=None, text=text) + error_message(text=text) else: text = 'Repo requires authentication, not updating' - warning_message.send(sender=None, text=text) + warning_message(text=text) def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors @@ -168,7 +168,7 @@ def show(self): text = f' {self.id} : {self.url}\n' text += ' last updated: ' text += f'{self.timestamp} checksum: {self.packages_checksum}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def fail(self): """ Records that the mirror has failed @@ -178,10 +178,10 @@ def fail(self): """ if self.repo.auth_required: text = f'Mirror requires authentication, not updating - {self.url}' - warning_message.send(sender=None, text=text) + warning_message(text=text) return text = f'No usable mirror found at {self.url}' - error_message.send(sender=None, text=text) + error_message(text=text) default_max_mirror_failures = 28 max_mirror_failures = get_setting_of_type( setting_name='MAX_MIRROR_FAILURES', @@ -191,11 +191,11 @@ def fail(self): self.fail_count = self.fail_count + 1 if max_mirror_failures == -1: text = f'Mirror has failed {self.fail_count} times, but MAX_MIRROR_FAILURES=-1, not disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) elif self.fail_count > max_mirror_failures: self.refresh = False text = f'Mirror has failed {self.fail_count} times (max={max_mirror_failures}), disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) self.last_access_ok = False self.save() diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 6e85b153c..09719428c 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,7 +18,8 @@ from io import BytesIO from packages.models import PackageString -from patchman.signals import info_message, warning_message, pbar_start, pbar_update +from util.logging import info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages from util import get_datetime_now, get_checksum, Checksum @@ -34,7 +35,7 @@ def refresh_arch_repo(repo): for i, mirror in enumerate(enabled_mirrors): if i >= max_mirrors: text = f'{max_mirrors} Mirrors already refreshed (max={max_mirrors}), skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) break res = find_mirror_url(mirror.url, [fname]) @@ -42,7 +43,7 @@ def refresh_arch_repo(repo): continue mirror_url = res.url text = f'Found Arch Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -54,7 +55,7 @@ def refresh_arch_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum @@ -111,5 +112,5 @@ def extract_arch_packages(data): packagetype='A') packages.add(package) else: - info_message.send(sender=None, text='No Packages found in Repo') + info_message(text='No Packages found in Repo') return packages diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 1d3607c55..c6c26d787 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -19,7 +19,8 @@ from debian.debian_support import Version from packages.models import PackageString -from patchman.signals import error_message, pbar_start, pbar_update, info_message, warning_message +from util.logging import error_message, info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url from util import get_datetime_now, get_checksum, Checksum, extract @@ -30,7 +31,7 @@ def extract_deb_packages(data, url): try: extracted = extract(data, url).decode('utf-8') except UnicodeDecodeError as e: - error_message.send(sender=None, text=f'Skipping {url} : {e}') + error_message(text=f'Skipping {url} : {e}') return package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) @@ -61,7 +62,7 @@ def extract_deb_packages(data, url): packagetype='D') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages @@ -86,7 +87,7 @@ def refresh_deb_repo(repo): continue mirror_url = res.url text = f'Found deb Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -98,7 +99,7 @@ def refresh_deb_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 8e4198d9c..e440f0d5f 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -26,7 +26,8 @@ from packages.models import PackageString from packages.utils import find_evr -from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid @@ -56,7 +57,7 @@ def refresh_gentoo_main_repo(repo): if mirror.packages_checksum == checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue res = get_url(mirror.url) @@ -70,7 +71,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue extracted = extract(data, mirror.url) - info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}') + info_message(text=f'Found Gentoo Repo - {mirror.url}') computed_checksum = get_checksum(data, Checksum.md5) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'): @@ -165,7 +166,7 @@ def get_gentoo_overlay_mirrors(repo_name): if element.text.startswith('http'): mirrors.append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_overlays_url}: {e}') + error_message(text=f'Error parsing {gentoo_overlays_url}: {e}') return mirrors @@ -199,7 +200,7 @@ def get_gentoo_mirror_urls(): if element.get('protocol') == 'http': mirrors[name]['urls'].append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_distfiles_url}: {e}') + error_message(text=f'Error parsing {gentoo_distfiles_url}: {e}') mirror_urls = [] # for now, ignore region data and choose MAX_MIRRORS mirrors at random for _, v in mirrors.items(): @@ -274,7 +275,7 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds): ) packages.add(package) plen = len(packages) - info_message.send(sender=None, text=f'Extracted {plen} Packages', plen=plen) + info_message(text=f'Extracted {plen} Packages', plen=plen) return packages @@ -282,7 +283,7 @@ def extract_gentoo_overlay_packages(mirror): """ Extract packages from gentoo overlay repo """ t = tempfile.mkdtemp() - info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}') + info_message(text=f'Extracting Gentoo packages from {mirror.url}') git.Repo.clone_from(mirror.url, t, depth=1) packages = set() extracted_ebuilds = extract_gentoo_overlay_ebuilds(t) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 93d470076..516618096 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,7 +16,7 @@ from django.db.models import Q -from patchman.signals import info_message, warning_message +from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data @@ -47,7 +47,7 @@ def max_mirrors_refreshed(repo, checksum, ts): have_checksum_and_ts = repo.mirror_set.filter(mirrors_q).count() if have_checksum_and_ts >= max_mirrors: text = f'{max_mirrors} Mirrors already have this checksum and timestamp, skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) return True return False @@ -87,11 +87,11 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): if mirror_url.endswith('content'): text = f'Found yast rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yast_repo(mirror, repo_data) else: text = f'Found yum rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yum_repo(mirror, repo_data, mirror_url, errata_only) if mirror.last_access_ok: mirror.timestamp = ts diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index 0ef543582..bf5940403 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,7 +17,8 @@ import re from packages.models import PackageString -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -65,5 +66,5 @@ def extract_yast_packages(data): packagetype='R') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index 7ac858162..bc0fbc4bb 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -22,7 +22,8 @@ from errata.sources.repos.yum import extract_updateinfo from packages.models import Package, PackageString from packages.utils import get_or_create_package, parse_package_string -from patchman.signals import warning_message, error_message, pbar_start, pbar_update +from util.logging import warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -50,7 +51,7 @@ def get_repomd_url(mirror_url, data, url_type='primary'): checksum = grandchild.text checksum_type = grandchild.attrib.get('type') except ElementTree.ParseError as e: - error_message.send(sender=None, text=(f'Error parsing repomd from {mirror_url}: {e}')) + error_message(text=(f'Error parsing repomd from {mirror_url}: {e}')) if not location: return None, None, None url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location @@ -65,7 +66,7 @@ def extract_module_metadata(data, url, repo): try: modules_yaml = yaml.safe_load_all(extracted) except yaml.YAMLError as e: - error_message.send(sender=None, text=f'Error parsing modules.yaml: {e}') + error_message(text=f'Error parsing modules.yaml: {e}') mlen = len(re.findall(r'---', yaml.dump(extracted.decode()))) pbar_start.send(sender=None, ptext=f'Extracting {mlen} Modules ', plen=mlen) @@ -150,10 +151,10 @@ def extract_yum_packages(data, url): i += 1 else: text = f'Error parsing Package: {name} {epoch} {version} {release} {arch}' - error_message.send(sender=None, text=text) + error_message(text=text) elem.clear() except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing yum primary.xml from {url}: {e}') + error_message(text=f'Error parsing yum primary.xml from {url}: {e}') return packages @@ -162,7 +163,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='updateinfo') if not url: - warning_message.send(sender=None, text=f'No Errata metadata found in {mirror_url}') + warning_message(text=f'No Errata metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -177,7 +178,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): if mirror.errata_checksum and mirror.errata_checksum == checksum: text = 'Mirror Errata checksum has not changed, skipping Erratum refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.errata_checksum = checksum @@ -191,7 +192,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='modules') if not url: - warning_message.send(sender=None, text=f'No Module metadata found in {mirror_url}') + warning_message(text=f'No Module metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -206,7 +207,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): if mirror.modules_checksum and mirror.modules_checksum == checksum: text = 'Mirror Modules checksum has not changed, skipping Module refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.modules_checksum = checksum @@ -220,7 +221,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='primary') if not url: - warning_message.send(sender=None, text=f'No Package metadata found in {mirror_url}') + warning_message(text=f'No Package metadata found in {mirror_url}') data = fetch_mirror_data( mirror=mirror, url=url, @@ -234,7 +235,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): if mirror.packages_checksum and mirror.packages_checksum == checksum: text = 'Mirror Packages checksum has not changed, skipping Package refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.packages_checksum = checksum diff --git a/repos/utils.py b/repos/utils.py index 49b5d07fb..c11c41ada 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -26,7 +26,8 @@ from packages.models import Package from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from patchman.signals import info_message, warning_message, error_message, debug_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -77,7 +78,7 @@ def update_mirror_packages(mirror, packages): package = convert_packagestring_to_package(strpackage) mirror_package, c = MirrorPackage.objects.get_or_create(mirror=mirror, package=package) except Package.MultipleObjectsReturned: - error_message.send(sender=None, text=f'Duplicate Package found in {mirror}: {strpackage}') + error_message(text=f'Duplicate Package found in {mirror}: {strpackage}') def find_mirror_url(stored_mirror_url, formats): @@ -89,7 +90,7 @@ def find_mirror_url(stored_mirror_url, formats): if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = f"{mirror_url.rstrip('/')}/{fmt}" - debug_message.send(sender=None, text=f'Checking for Mirror at {mirror_url}') + debug_message(text=f'Checking for Mirror at {mirror_url}') try: res = get_url(mirror_url) except RetryError: @@ -133,7 +134,7 @@ def get_metalink_urls(url): if greatgreatgrandchild.attrib.get('protocol') in ['https', 'http']: metalink_urls.append(greatgreatgrandchild.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing metalink {url}: {e}') + error_message(text=f'Error parsing metalink {url}: {e}') return metalink_urls @@ -152,12 +153,12 @@ def get_mirrorlist_urls(url): return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) if mirror_urls: - debug_message.send(sender=None, text=f'Found mirrorlist: {url}') + debug_message(text=f'Found mirrorlist: {url}') return mirror_urls else: - debug_message.send(sender=None, text=f'Not a mirrorlist: {url}') + debug_message(text=f'Not a mirrorlist: {url}') except Exception as e: - error_message.send(sender=None, text=f'Error attempting to parse a mirrorlist: {e} {url}') + error_message(text=f'Error attempting to parse a mirrorlist: {e} {url}') def add_mirrors_from_urls(repo, mirror_urls): @@ -172,7 +173,7 @@ def add_mirrors_from_urls(repo, mirror_urls): existing = repo.mirror_set.filter(q).count() if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' - warning_message.send(sender=None, text=text) + warning_message(text=text) break from repos.models import Mirror # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? @@ -180,7 +181,7 @@ def add_mirrors_from_urls(repo, mirror_urls): m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) if c: text = f'Added Mirror - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) def check_for_mirrorlists(repo): @@ -193,7 +194,7 @@ def check_for_mirrorlists(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found mirrorlist - {mirror.url}') + info_message(text=f'Found mirrorlist - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -210,7 +211,7 @@ def check_for_metalinks(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found metalink - {mirror.url}') + info_message(text=f'Found metalink - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -249,9 +250,9 @@ def mirror_checksum_is_valid(computed, provided, mirror, metadata_type): """ if not computed or computed != provided: text = f'Checksum failed for mirror {mirror.id}, not refreshing {metadata_type} metadata' - error_message.send(sender=None, text=text) + error_message(text=text) text = f'Found checksum: {computed}\nExpected checksum: {provided}' - error_message.send(sender=None, text=text) + error_message(text=text) mirror.last_access_ok = False mirror.fail() return False @@ -296,9 +297,9 @@ def clean_repos(): repos = Repository.objects.filter(mirror__isnull=True) rlen = repos.count() if rlen == 0: - info_message.send(sender=None, text='No Repositories with zero Mirrors found.') + info_message(text='No Repositories with zero Mirrors found.') else: - info_message.send(sender=None, text=f'Removing {rlen} empty Repositories.') + info_message(text=f'Removing {rlen} empty Repositories.') repos.delete() @@ -309,13 +310,13 @@ def remove_mirror_trailing_slashes(): mirrors = Mirror.objects.filter(url__endswith='/') mlen = mirrors.count() if mlen == 0: - info_message.send(sender=None, text='No Mirrors with trailing slashes found.') + info_message(text='No Mirrors with trailing slashes found.') else: - info_message.send(sender=None, text=f'Removing trailing slashes from {mlen} Mirrors.') + info_message(text=f'Removing trailing slashes from {mlen} Mirrors.') for mirror in mirrors: mirror.url = mirror.url.rstrip('/') try: mirror.save() except IntegrityError: - warning_message.send(sender=None, text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') + warning_message(text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') mirror.delete() diff --git a/sbin/patchman b/sbin/patchman index c09114340..47d89b06b 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from reports.models import Report from reports.tasks import clean_reports_with_no_hosts from security.utils import update_cves, update_cwes from util import set_verbosity, get_datetime_now -from patchman.signals import info_message +from util.logging import info_message def get_host(host=None, action='Performing action'): @@ -64,7 +64,7 @@ def get_host(host=None, action='Performing action'): matches = Host.objects.filter(hostname__startswith=host).count() text = f'{matches} Hosts match hostname "{host}"' - info_message.send(sender=None, text=text) + info_message(text=text) return host_obj @@ -84,7 +84,7 @@ def get_hosts(hosts=None, action='Performing action'): host_objs.append(host_obj) else: text = f'{action} for all Hosts\n' - info_message.send(sender=None, text=text) + info_message(text=text) host_objs = Host.objects.all() return host_objs @@ -107,7 +107,7 @@ def get_repos(repo=None, action='Performing action', only_enabled=False): else: repos = Repository.objects.all() - info_message.send(sender=None, text=text) + info_message(text=text) return repos @@ -118,9 +118,9 @@ def refresh_repos(repo=None, force=False): repos = get_repos(repo, 'Refreshing metadata', True) for repo in repos: text = f'Repository {repo.id} : {repo}' - info_message.send(sender=None, text=text) + info_message(text=text) repo.refresh(force) - info_message.send(sender=None, text='') + info_message(text='') def list_repos(repos=None): @@ -161,10 +161,10 @@ def host_updates_alt(host=None): hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) if host not in updated_hosts: host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') host.updated_at = ts host.save() @@ -200,10 +200,10 @@ def host_updates_alt(host=None): phost.save() updated_hosts.append(phost) text = f'Added the same updates to {phost}' - info_message.send(sender=None, text=text) + info_message(text=text) else: text = 'Updates already added in this run' - info_message.send(sender=None, text=text) + info_message(text=text) def host_updates(host=None): @@ -211,9 +211,9 @@ def host_updates(host=None): """ hosts = get_hosts(host, 'Finding updates') for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') def diff_hosts(hosts): @@ -236,47 +236,47 @@ def diff_hosts(hosts): repo_diff_AB = reposA.difference(reposB) repo_diff_BA = reposB.difference(reposA) - info_message.send(sender=None, text=f'+ {hostA.hostname}') - info_message.send(sender=None, text=f'- {hostB.hostname}') + info_message(text=f'+ {hostA.hostname}') + info_message(text=f'- {hostB.hostname}') if hostA.os != hostB.os: - info_message.send(sender=None, text='\nOperating Systems') - info_message.send(sender=None, text=f'+ {hostA.os}') - info_message.send(sender=None, text=f'- {hostB.os}') + info_message(text='\nOperating Systems') + info_message(text=f'+ {hostA.os}') + info_message(text=f'- {hostB.os}') else: - info_message.send(sender=None, text='\nNo OS differences') + info_message(text='\nNo OS differences') if hostA.arch != hostB.arch: - info_message.send(sender=None, text='\nArchitecture') - info_message.send(sender=None, text=f'+ {hostA.arch}') - info_message.send(sender=None, text=f'- {hostB.arch}') + info_message(text='\nArchitecture') + info_message(text=f'+ {hostA.arch}') + info_message(text=f'- {hostB.arch}') else: - info_message.send(sender=None, text='\nNo Architecture differences') + info_message(text='\nNo Architecture differences') if hostA.kernel != hostB.kernel: - info_message.send(sender=None, text='\nKernels') - info_message.send(sender=None, text=f'+ {hostA.kernel}') - info_message.send(sender=None, text=f'- {hostB.kernel}') + info_message(text='\nKernels') + info_message(text=f'+ {hostA.kernel}') + info_message(text=f'- {hostB.kernel}') else: - info_message.send(sender=None, text='\nNo Kernel differences') + info_message(text='\nNo Kernel differences') if len(package_diff_AB) != 0 or len(package_diff_BA) != 0: - info_message.send(sender=None, text='\nPackages') + info_message(text='\nPackages') for package in package_diff_AB: - info_message.send(sender=None, text=f'+ {package}') + info_message(text=f'+ {package}') for package in package_diff_BA: - info_message.send(sender=None, text=f'- {package}') + info_message(text=f'- {package}') else: - info_message.send(sender=None, text='\nNo Package differences') + info_message(text='\nNo Package differences') if len(repo_diff_AB) != 0 or len(repo_diff_BA) != 0: - info_message.send(sender=None, text='\nRepositories') + info_message(text='\nRepositories') for repo in repo_diff_AB: - info_message.send(sender=None, text=f'+ {repo}') + info_message(text=f'+ {repo}') for repo in repo_diff_BA: - info_message.send(sender=None, text=f'- {repo}') + info_message(text=f'- {repo}') else: - info_message.send(sender=None, text='\nNo Repo differences') + info_message(text='\nNo Repo differences') def delete_hosts(hosts=None): @@ -286,7 +286,7 @@ def delete_hosts(hosts=None): matching_hosts = get_hosts(hosts) for host in matching_hosts: text = f'Deleting host: {host.hostname}:' - info_message.send(sender=None, text=text) + info_message(text=text) host.delete() @@ -300,7 +300,7 @@ def toggle_host_hro(hosts=None, host_repos_only=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} host_repos_only') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.host_repos_only = host_repos_only host.save() @@ -315,7 +315,7 @@ def toggle_host_check_dns(hosts=None, check_dns=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} check_dns') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.check_dns = check_dns host.save() @@ -347,7 +347,7 @@ def process_reports(host=None, force=False): text = 'Processing Reports for all Hosts' reports = Report.objects.filter(processed=force).order_by('created') - info_message.send(sender=None, text=text) + info_message(text=text) for report in reports: report.process(find_updates=False) diff --git a/security/models.py b/security/models.py index 9c097eed9..7f674a0b7 100644 --- a/security/models.py +++ b/security/models.py @@ -152,7 +152,7 @@ def fetch_mitre_cve_data(self): mitre_cve_url = f'https://cveawg.mitre.org/api/cve/{self.cve_id}' res = get_url(mitre_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} MITRE data') cve_json = json.loads(data) @@ -162,7 +162,7 @@ def fetch_osv_dev_cve_data(self): osv_dev_cve_url = f'https://api.osv.dev/v1/vulns/{self.cve_id}' res = get_url(osv_dev_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} OSV data') cve_json = json.loads(data) @@ -186,7 +186,7 @@ def fetch_nist_cve_data(self): res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') cve_json = json.loads(data) self.parse_nist_cve_data(cve_json) @@ -197,7 +197,7 @@ def parse_nist_cve_data(self, cve_json): cve = vulnerability.get('cve') cve_id = cve.get('id') if cve_id != self.cve_id: - error_message.send(sender=None, text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') + error_message(text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') return metrics = cve.get('metrics') for metric, score_data in metrics.items(): diff --git a/util/__init__.py b/util/__init__.py index 4a3f9caaf..c3dfd6bd5 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -38,7 +38,7 @@ from django.utils.dateparse import parse_datetime from django.conf import settings -from patchman.signals import error_message, info_message, debug_message +from util.logging import error_message, info_message, debug_message pbar = None verbose = None @@ -109,7 +109,7 @@ def fetch_content(response, text='', ljust=35): data += chunk return data else: - info_message.send(sender=None, text=text) + info_message(text=text) return response.content @@ -128,16 +128,16 @@ def get_url(url, headers=None, params=None): if not params: params = {} try: - debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') + debug_message(text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) - debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') + debug_message(text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response response.raise_for_status() except requests.exceptions.TooManyRedirects: - error_message.send(sender=None, text=f'Too many redirects - {url}') + error_message(text=f'Too many redirects - {url}') except ConnectionError: - error_message.send(sender=None, text=f'Connection error - {url}') + error_message(text=f'Connection error - {url}') return response @@ -180,7 +180,7 @@ def gunzip(contents): wbits = zlib.MAX_WBITS | 32 return zlib.decompress(contents, wbits) except zlib.error as e: - error_message.send(sender=None, text='gunzip: ' + str(e)) + error_message(text='gunzip: ' + str(e)) def bunzip2(contents): @@ -191,10 +191,10 @@ def bunzip2(contents): return bzip2data except IOError as e: if e == 'invalid data stream': - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) except ValueError as e: if e == "couldn't find end of stream": - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) def unxz(contents): @@ -204,7 +204,7 @@ def unxz(contents): xzdata = lzma.decompress(contents) return xzdata except lzma.LZMAError as e: - error_message.send(sender=None, text='lzma: ' + e) + error_message(text='lzma: ' + e) def unzstd(contents): @@ -214,7 +214,7 @@ def unzstd(contents): zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() return zstddata except zstd.ZstdError as e: - error_message.send(sender=None, text='zstd: ' + e) + error_message(text=f'zstd: {e}') def extract(data, fmt): @@ -253,7 +253,7 @@ def get_checksum(data, checksum_type): checksum = get_md5(data) else: text = f'Unknown checksum type: {checksum_type}' - error_message.send(sender=None, text=text) + error_message(text=text) return checksum diff --git a/util/logging.py b/util/logging.py new file mode 100644 index 000000000..dd79d296c --- /dev/null +++ b/util/logging.py @@ -0,0 +1,42 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from datetime import datetime + +from patchman.signals import info_message_s +from patchman.signals import warning_message_s +from patchman.signals import error_message_s +from patchman.signals import debug_message_s + + +def info_message(text): + ts = datetime.now() + info_message_s.send(sender=None, text=text, ts=ts) + + +def warning_message(text): + ts = datetime.now() + warning_message_s.send(sender=None, text=text, ts=ts) + + +def debug_message(text): + ts = datetime.now() + debug_message_s.send(sender=None, text=text, ts=ts) + + +def error_message(text): + ts = datetime.now() + error_message_s.send(sender=None, text=text, ts=ts) From 00fbd6e42581ffedf93cdfedd47154cb6ffdeea2 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:50 -0400 Subject: [PATCH 05/21] use redis for caching and use locks for tasks --- errata/tasks.py | 68 ++++++++++++++++++++-------------- etc/patchman/local_settings.py | 16 +++----- reports/tasks.py | 15 +++++++- repos/tasks.py | 17 ++++++++- security/tasks.py | 33 ++++++++++++++--- 5 files changed, 101 insertions(+), 48 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index f1d6eeee3..d9aaf3d42 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -16,13 +16,15 @@ from celery import shared_task +from django.core.cache import cache + from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message +from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type @@ -44,34 +46,44 @@ def update_yum_repo_errata(repo_id=None, force=False): def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ - errata_os_updates = [] - erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] - erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] - if erratum_type: - if erratum_type not in erratum_types: - error_message.send(sender=None, text=f'Erratum type `{erratum_type}` not in {erratum_types}') - else: - errata_os_updates = erratum_type + lock_key = 'update_errata_lock' + # lock will expire after 48 hours + lock_expire = 60 * 60 * 48 + + if cache.add(lock_key, 'true', lock_expire): + try: + errata_os_updates = [] + erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] + erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + if erratum_type: + if erratum_type not in erratum_types: + error_message(text=f'Erratum type `{erratum_type}` not in {erratum_types}') + else: + errata_os_updates = erratum_type + else: + errata_os_updates = get_setting_of_type( + setting_name='ERRATA_OS_UPDATES', + setting_type=list, + default=erratum_type_defaults, + ) + if 'yum' in errata_os_updates: + update_yum_repo_errata(repo_id=repo, force=force) + if 'arch' in errata_os_updates: + update_arch_errata() + if 'alma' in errata_os_updates: + update_alma_errata() + if 'rocky' in errata_os_updates: + update_rocky_errata() + if 'debian' in errata_os_updates: + update_debian_errata() + if 'ubuntu' in errata_os_updates: + update_ubuntu_errata() + if 'centos' in errata_os_updates: + update_centos_errata() + finally: + cache.delete(lock_key) else: - errata_os_updates = get_setting_of_type( - setting_name='ERRATA_OS_UPDATES', - setting_type=list, - default=erratum_type_defaults, - ) - if 'yum' in errata_os_updates: - update_yum_repo_errata(repo_id=repo, force=force) - if 'arch' in errata_os_updates: - update_arch_errata() - if 'alma' in errata_os_updates: - update_alma_errata() - if 'rocky' in errata_os_updates: - update_rocky_errata() - if 'debian' in errata_os_updates: - update_debian_errata() - if 'ubuntu' in errata_os_updates: - update_ubuntu_errata() - if 'centos' in errata_os_updates: - update_centos_errata() + warning_message('Already updating Errata, skipping task.') @shared_task diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d2..9e7ca21b9 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -44,22 +44,16 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', + 'TIMEOUT': 0, } } -# Uncomment to enable redis caching for e.g. 30 seconds -# Note that the UI results may be out of date for this amount of time -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.redis.RedisCache', -# 'LOCATION': 'redis://127.0.0.1:6379', -# 'TIMEOUT': 30, -# } -# } - from datetime import timedelta # noqa from celery.schedules import crontab # noqa CELERY_BEAT_SCHEDULE = { diff --git a/reports/tasks.py b/reports/tasks.py index fe294e8d1..d2a47e8fd 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -17,11 +17,12 @@ from celery import shared_task +from django.core.cache import cache from django.db.utils import OperationalError from hosts.models import Host from reports.models import Report -from util.logging import info_message +from util.logging import info_message, warning_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -29,7 +30,17 @@ def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - report.process() + lock_key = f'process_report_lock_{report_id}' + # lock will expire after 1 hour + lock_expire = 60 * 60 + + if cache.add(lock_key, 'true', lock_expire): + try: + report.process() + finally: + cache.delete(lock_key) + else: + warning_message(f'Already processing report {report_id}, skipping task.') @shared_task diff --git a/repos/tasks.py b/repos/tasks.py index 39098fa8d..436da82a0 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from repos.models import Repository +from util.logging import warning_message @shared_task @@ -32,5 +35,15 @@ def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ repos = Repository.objects.filter(enabled=True) - for repo in repos: - refresh_repo.delay(repo.id, force) + lock_key = 'refresh_repos_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(lock_key, 'true', lock_expire): + try: + for repo in repos: + refresh_repo.delay(repo.id, force) + finally: + cache.delete(lock_key) + else: + warning_message('Already refreshing repos, skipping task.') diff --git a/security/tasks.py b/security/tasks.py index a04bb1c84..ce60df83c 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from security.models import CVE, CWE +from util.logging import warning_message @shared_task @@ -31,8 +34,18 @@ def update_cve(cve_id): def update_cves(): """ Task to update all CVEs """ - for cve in CVE.objects.all(): - update_cve.delay(cve.id) + lock_key = 'update_cves_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cve in CVE.objects.all(): + update_cve.delay(cve.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CVEs, skipping task.') @shared_task @@ -45,7 +58,17 @@ def update_cwe(cwe_id): @shared_task def update_cwes(): - """ Task to update all CWEa + """ Task to update all CWEs """ - for cwe in CWE.objects.all(): - update_cwe.delay(cwe.id) + lock_key = 'update_cwes_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cwe in CWE.objects.all(): + update_cwe.delay(cwe.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CWEs, skipping task.') From fd6f9aadf43f8e988967bbdc2efe0b3dc1665f94 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 18:38:04 -0400 Subject: [PATCH 06/21] add errata source options to config file --- errata/sources/distros/debian.py | 2 +- errata/sources/distros/ubuntu.py | 2 +- etc/patchman/local_settings.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 1ae919e47..ece3754dd 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -264,7 +264,7 @@ def get_accepted_debian_codenames(): """ Get acceptable Debian OS codenames Can be overridden by specifying DEBIAN_CODENAMES in settings """ - default_codenames = ['bookworm', 'bullseye'] + default_codenames = ['bookworm', 'trixie'] accepted_codenames = get_setting_of_type( setting_name='DEBIAN_CODENAMES', setting_type=list, diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index d1ce7cc58..6fafb40ac 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -203,7 +203,7 @@ def get_accepted_ubuntu_codenames(): """ Get acceptable Ubuntu OS codenames Can be overridden by specifying UBUNTU_CODENAMES in settings """ - default_codenames = ['focal', 'jammy', 'noble'] + default_codenames = ['jammy', 'noble'] accepted_codenames = get_setting_of_type( setting_name='UBUNTU_CODENAMES', setting_type=list, diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d2..ab93c89d0 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -41,6 +41,18 @@ # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14 +# list of errata sources to update, remove unwanted ones to improve performance +ERRATA_OS_UPDATES = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + +# list of Alma Linux releases to update +ALMA_RELEASES = [8, 9, 10] + +# list of Debian Linux releases to update +DEBIAN_CODENAMES = ['bookworm', 'trixie'] + +# list of Ubuntu Linux releases to update +UBUNTU_CODENAMES = ['jammy', 'noble'] + # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False From f06729e89aafa10910a421a21761d60d54345896 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 14 May 2025 22:39:29 -0400 Subject: [PATCH 07/21] remove daily cronjob in favour of patchman-celery --- debian/python3-patchman.cron.daily | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 debian/python3-patchman.cron.daily diff --git a/debian/python3-patchman.cron.daily b/debian/python3-patchman.cron.daily deleted file mode 100644 index d4752f75d..000000000 --- a/debian/python3-patchman.cron.daily +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/usr/bin/patchman -a -q From a59a23b5e26b3e2c6436c140055bf55a1b47c712 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 30 Oct 2025 22:02:44 -0400 Subject: [PATCH 08/21] add isort check --- .github/workflows/lint-and-test.yml | 4 +++ arch/admin.py | 3 ++- arch/serializers.py | 2 +- arch/utils.py | 2 +- arch/views.py | 7 ++--- domains/admin.py | 1 + errata/admin.py | 1 + errata/models.py | 7 +++-- errata/sources/distros/alma.py | 2 +- errata/sources/distros/arch.py | 10 ++++--- errata/sources/distros/centos.py | 7 ++--- errata/sources/distros/debian.py | 8 +++--- errata/sources/distros/rocky.py | 12 +++++---- errata/sources/distros/ubuntu.py | 13 ++++++--- errata/sources/repos/yum.py | 4 +-- errata/tasks.py | 7 +++-- errata/utils.py | 4 +-- errata/views.py | 7 +++-- etc/patchman/local_settings.py | 4 ++- hooks/yum/patchman.py | 1 + hooks/zypper/patchman.py | 3 ++- hosts/admin.py | 1 + hosts/migrations/0001_initial.py | 3 ++- hosts/migrations/0002_initial.py | 2 +- .../0004_remove_host_tags_host_tags.py | 3 ++- hosts/migrations/0006_migrate_to_tz_aware.py | 1 + hosts/migrations/0007_alter_host_tags.py | 2 +- hosts/models.py | 3 ++- hosts/tasks.py | 1 - hosts/templatetags/report_alert.py | 2 +- hosts/utils.py | 4 +-- hosts/views.py | 23 ++++++++-------- modules/admin.py | 1 + modules/migrations/0001_initial.py | 2 +- modules/utils.py | 4 +-- modules/views.py | 7 +++-- operatingsystems/admin.py | 3 ++- operatingsystems/forms.py | 4 +-- operatingsystems/migrations/0002_initial.py | 2 +- operatingsystems/migrations/0003_os_arch.py | 2 +- operatingsystems/serializers.py | 2 +- operatingsystems/views.py | 17 +++++++----- packages/admin.py | 1 + packages/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20250207_1319.py | 2 +- packages/serializers.py | 2 +- packages/utils.py | 4 ++- packages/views.py | 13 ++++----- patchman/__init__.py | 3 +-- patchman/celery.py | 3 ++- patchman/receivers.py | 16 +++++------ patchman/urls.py | 3 +-- patchman/wsgi.py | 3 +-- reports/admin.py | 1 + .../migrations/0004_migrate_to_tz_aware.py | 1 + reports/models.py | 6 +++-- reports/tasks.py | 1 - reports/utils.py | 13 ++++++--- reports/views.py | 19 ++++++------- repos/admin.py | 3 ++- repos/forms.py | 7 +++-- repos/migrations/0001_initial.py | 2 +- repos/migrations/0003_migrate_to_tz_aware.py | 1 + repos/models.py | 9 +++---- repos/repo_types/arch.py | 9 ++++--- repos/repo_types/deb.py | 9 ++++--- repos/repo_types/gentoo.py | 16 +++++++---- repos/repo_types/rpm.py | 7 +++-- repos/repo_types/yast.py | 2 +- repos/repo_types/yum.py | 5 ++-- repos/serializers.py | 2 +- repos/tasks.py | 1 - repos/utils.py | 18 +++++++++---- repos/views.py | 27 ++++++++++--------- sbin/patchman | 21 +++++++++------ security/admin.py | 2 +- security/models.py | 4 +-- security/tasks.py | 1 - security/views.py | 11 ++++---- setup.py | 3 ++- util/__init__.py | 23 +++++++++------- util/filterspecs.py | 5 ++-- util/logging.py | 8 +++--- util/tasks.py | 4 ++- util/templatetags/common.py | 7 +++-- util/views.py | 6 ++--- 86 files changed, 292 insertions(+), 212 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 20b221aa6..436698be0 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,10 @@ jobs: pip install flake8 flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --max-line-length=120 --show-source --statistics + - name: Check isort + run: | + pip install isort + isort --check --profile=django . - name: Set secret key run: ./sbin/patchman-set-secret-key - name: Test with django diff --git a/arch/admin.py b/arch/admin.py index 624a3720c..5224711c2 100644 --- a/arch/admin.py +++ b/arch/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from arch.models import PackageArchitecture, MachineArchitecture + +from arch.models import MachineArchitecture, PackageArchitecture admin.site.register(PackageArchitecture) admin.site.register(MachineArchitecture) diff --git a/arch/serializers.py b/arch/serializers.py index 5319e7960..a57651289 100644 --- a/arch/serializers.py +++ b/arch/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer): diff --git a/arch/utils.py b/arch/utils.py index 04d0b3506..3db6ac70c 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture from util.logging import info_message diff --git a/arch/views.py b/arch/views.py index 56a2a1507..21f6b7c78 100644 --- a/arch/views.py +++ b/arch/views.py @@ -16,9 +16,10 @@ from rest_framework import viewsets -from arch.models import PackageArchitecture, MachineArchitecture -from arch.serializers import PackageArchitectureSerializer, \ - MachineArchitectureSerializer +from arch.models import MachineArchitecture, PackageArchitecture +from arch.serializers import ( + MachineArchitectureSerializer, PackageArchitectureSerializer, +) class PackageArchitectureViewSet(viewsets.ModelViewSet): diff --git a/domains/admin.py b/domains/admin.py index 2ef883e30..5cb0fee39 100644 --- a/domains/admin.py +++ b/domains/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from domains.models import Domain admin.site.register(Domain) diff --git a/errata/admin.py b/errata/admin.py index 88190ff66..ac4b8a50a 100644 --- a/errata/admin.py +++ b/errata/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from errata.models import Erratum diff --git a/errata/models.py b/errata/models.py index cfc9bd0df..8c21bcfa6 100644 --- a/errata/models.py +++ b/errata/models.py @@ -16,17 +16,16 @@ import json -from django.db import models +from django.db import IntegrityError, models from django.urls import reverse -from django.db import IntegrityError +from errata.managers import ErratumManager from packages.models import Package, PackageUpdate from packages.utils import find_evr, get_matching_packages -from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from util.logging import error_message from util import get_url +from util.logging import error_message class Erratum(models.Model): diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index e0f2d4aeb..0091b8bfa 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -22,8 +22,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string -from util import get_url, fetch_content, get_setting_of_type from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_setting_of_type, get_url def update_alma_errata(concurrent_processing=True): diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 87c6c47a6..e22de4034 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,11 +20,13 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from util.logging import error_message -from patchman.signals import pbar_start, pbar_update from packages.models import Package -from packages.utils import find_evr, get_matching_packages, get_or_create_package -from util import get_url, fetch_content +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, +) +from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_url +from util.logging import error_message def update_arch_errata(concurrent_processing=False): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index d2722a6be..8f4aa4a1e 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -15,14 +15,15 @@ # along with Patchman. If not, see import re + from defusedxml import ElementTree from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package -from util.logging import error_message +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type +from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url +from util.logging import error_message def update_centos_errata(): diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index ece3754dd..8025b1bf4 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -18,18 +18,18 @@ import csv import re from datetime import datetime -from debian.deb822 import Dsc from io import StringIO +from debian.deb822 import Dsc from django.db import connections from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, find_evr -from util.logging import error_message, warning_message +from packages.utils import find_evr, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, get_setting_of_type, extract +from util import extract, fetch_content, get_setting_of_type, get_url +from util.logging import error_message, warning_message DSCs = {} diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 16d4d12c0..2d8059851 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -14,19 +14,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import json import concurrent.futures -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +import json from django.db import connections from django.db.utils import OperationalError +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content -from util.logging import info_message, error_message +from util import fetch_content, get_url +from util.logging import error_message, info_message def update_rocky_errata(concurrent_processing=True): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 6fafb40ac..5616331fc 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -16,8 +16,8 @@ import concurrent.futures import csv -import os import json +import os from io import StringIO from urllib.parse import urlparse @@ -26,10 +26,15 @@ from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages -from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from util.logging import error_message +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update +from util import ( + bunzip2, fetch_content, get_setting_of_type, get_sha256, get_url, +) +from util.logging import error_message def update_ubuntu_errata(concurrent_processing=False): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index f361d10e6..8b6732c45 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -16,17 +16,17 @@ import concurrent.futures from io import BytesIO -from defusedxml import ElementTree +from defusedxml import ElementTree from django.db import connections from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from util.logging import error_message from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url +from util.logging import error_message def extract_updateinfo(data, url, concurrent_processing=True): diff --git a/errata/tasks.py b/errata/tasks.py index d9aaf3d42..8ded3a79c 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -15,19 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache -from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata -from errata.sources.distros.debian import update_debian_errata +from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.centos import update_centos_errata +from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type +from util.logging import error_message, warning_message @shared_task diff --git a/errata/utils.py b/errata/utils.py index a8d8d4245..e0a5e01b7 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -18,11 +18,11 @@ from django.db import connections -from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from util.logging import warning_message from patchman.signals import pbar_start, pbar_update +from util import tz_aware_datetime +from util.logging import warning_message def get_or_create_erratum(name, e_type, issue_date, synopsis): diff --git a/errata/views.py b/errata/views.py index 42d12f712..8e1c0b2f6 100644 --- a/errata/views.py +++ b/errata/views.py @@ -14,16 +14,15 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from operatingsystems.models import OSRelease from errata.models import Erratum from errata.serializers import ErratumSerializer +from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index f7b98cf80..40dd2efd6 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -66,8 +66,10 @@ } } -from datetime import timedelta # noqa +from datetime import timedelta # noqa + from celery.schedules import crontab # noqa + CELERY_BEAT_SCHEDULE = { 'process_all_unprocessed_reports': { 'task': 'reports.tasks.process_reports', diff --git a/hooks/yum/patchman.py b/hooks/yum/patchman.py index 343144eb0..52f9cc8bc 100644 --- a/hooks/yum/patchman.py +++ b/hooks/yum/patchman.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see import os + from yum.plugins import TYPE_CORE requires_api_version = '2.1' diff --git a/hooks/zypper/patchman.py b/hooks/zypper/patchman.py index 147815657..d9d478f3d 100755 --- a/hooks/zypper/patchman.py +++ b/hooks/zypper/patchman.py @@ -18,8 +18,9 @@ # # zypp system plugin for patchman -import os import logging +import os + from zypp_plugin import Plugin diff --git a/hosts/admin.py b/hosts/admin.py index 8a42e8ccd..43bf31da7 100644 --- a/hosts/admin.py +++ b/hosts/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from hosts.models import Host, HostRepo diff --git a/hosts/migrations/0001_initial.py b/hosts/migrations/0001_initial.py index 43366684c..0037e094a 100644 --- a/hosts/migrations/0001_initial.py +++ b/hosts/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.db import migrations, models + try: import tagging.fields has_tagging = True diff --git a/hosts/migrations/0002_initial.py b/hosts/migrations/0002_initial.py index cc59a70e5..6c453c492 100644 --- a/hosts/migrations/0002_initial.py +++ b/hosts/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/hosts/migrations/0004_remove_host_tags_host_tags.py b/hosts/migrations/0004_remove_host_tags_host_tags.py index 84e7affec..bf03a84ef 100644 --- a/hosts/migrations/0004_remove_host_tags_host_tags.py +++ b/hosts/migrations/0004_remove_host_tags_host_tags.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.18 on 2025-02-04 23:37 +import taggit.managers from django.apps import apps from django.db import migrations -import taggit.managers + try: import tagging # noqa except ImportError: diff --git a/hosts/migrations/0006_migrate_to_tz_aware.py b/hosts/migrations/0006_migrate_to_tz_aware.py index e36bbf1f4..c14ea50b6 100644 --- a/hosts/migrations/0006_migrate_to_tz_aware.py +++ b/hosts/migrations/0006_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Host = apps.get_model('hosts', 'Host') for host in Host.objects.all(): diff --git a/hosts/migrations/0007_alter_host_tags.py b/hosts/migrations/0007_alter_host_tags.py index 3858b8475..3910a06fe 100644 --- a/hosts/migrations/0007_alter_host_tags.py +++ b/hosts/migrations/0007_alter_host_tags.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.19 on 2025-02-28 19:53 -from django.db import migrations import taggit.managers +from django.db import migrations class Migration(migrations.Migration): diff --git a/hosts/models.py b/hosts/models.py index 650544dca..8ea5e3d50 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -24,6 +24,7 @@ from version_utils.rpm import labelCompare except ImportError: from rpm import labelCompare + from taggit.managers import TaggableManager from arch.models import MachineArchitecture @@ -34,9 +35,9 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo +from util.logging import info_message class Host(models.Model): diff --git a/hosts/tasks.py b/hosts/tasks.py index 1643901d1..226652f6e 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.db.models import Count from hosts.models import Host diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index a28c50588..48d8f966c 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,9 +17,9 @@ from datetime import timedelta from django.template import Library -from django.utils.html import format_html from django.templatetags.static import static from django.utils import timezone +from django.utils.html import format_html from util import get_setting_of_type diff --git a/hosts/utils.py b/hosts/utils.py index d6e663cf8..44441f9be 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from socket import gethostbyaddr, gaierror, herror +from socket import gaierror, gethostbyaddr, herror -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from taggit.models import Tag from util.logging import error_message, info_message diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffac..8f20ab196 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages - -from taggit.models import Tag +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets +from taggit.models import Tag -from util.filterspecs import Filter, FilterBar -from hosts.models import Host, HostRepo -from domains.models import Domain from arch.models import MachineArchitecture -from operatingsystems.models import OSVariant, OSRelease -from reports.models import Report +from domains.models import Domain from hosts.forms import EditHostForm -from hosts.serializers import HostSerializer, HostRepoSerializer +from hosts.models import Host, HostRepo +from hosts.serializers import HostRepoSerializer, HostSerializer +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from util.filterspecs import Filter, FilterBar @login_required diff --git a/modules/admin.py b/modules/admin.py index 33b94d206..9cf21e6cf 100644 --- a/modules/admin.py +++ b/modules/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from modules.models import Module admin.site.register(Module) diff --git a/modules/migrations/0001_initial.py b/modules/migrations/0001_initial.py index 12a8e278a..9c27d425e 100644 --- a/modules/migrations/0001_initial.py +++ b/modules/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/modules/utils.py b/modules/utils.py index 05c57c809..0d6694785 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,10 +15,10 @@ # along with Patchman. If not, see from django.db import IntegrityError -from util.logging import error_message, info_message -from modules.models import Module from arch.models import PackageArchitecture +from modules.models import Module +from util.logging import error_message, info_message def get_or_create_module(name, stream, version, context, arch, repo): diff --git a/modules/views.py b/modules/views.py index b897a709f..2d017220c 100644 --- a/modules/views.py +++ b/modules/views.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - -from rest_framework import viewsets, permissions +from django.shortcuts import get_object_or_404, render +from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer diff --git a/operatingsystems/admin.py b/operatingsystems/admin.py index 15f5e2002..4884b5ebf 100644 --- a/operatingsystems/admin.py +++ b/operatingsystems/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from operatingsystems.models import OSVariant, OSRelease + +from operatingsystems.models import OSRelease, OSVariant class OSReleaseAdmin(admin.ModelAdmin): diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index 548a7d88a..fa319182f 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ModelForm, ModelMultipleChoiceField -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant from repos.models import Repository diff --git a/operatingsystems/migrations/0002_initial.py b/operatingsystems/migrations/0002_initial.py index 517a3f9ac..04cbb411d 100644 --- a/operatingsystems/migrations/0002_initial.py +++ b/operatingsystems/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/migrations/0003_os_arch.py b/operatingsystems/migrations/0003_os_arch.py index 2778ca3fc..4d0e0f935 100644 --- a/operatingsystems/migrations/0003_os_arch.py +++ b/operatingsystems/migrations/0003_os_arch.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:02 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/serializers.py b/operatingsystems/serializers.py index 8418c7206..be178909c 100644 --- a/operatingsystems/serializers.py +++ b/operatingsystems/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant class OSVariantSerializer(serializers.HyperlinkedModelSerializer): diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f921..6009f119f 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,19 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse - from rest_framework import viewsets from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from operatingsystems.forms import AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm -from operatingsystems.serializers import OSVariantSerializer, OSReleaseSerializer +from operatingsystems.forms import ( + AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm, +) +from operatingsystems.models import OSRelease, OSVariant +from operatingsystems.serializers import ( + OSReleaseSerializer, OSVariantSerializer, +) @login_required diff --git a/packages/admin.py b/packages/admin.py index 979ba7792..bc4b1aaae 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from packages.models import Package, PackageName, PackageUpdate diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 07e7bcb0a..bfea3e1a8 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/migrations/0002_auto_20250207_1319.py b/packages/migrations/0002_auto_20250207_1319.py index 1563d1396..4c744203f 100644 --- a/packages/migrations/0002_auto_20250207_1319.py +++ b/packages/migrations/0002_auto_20250207_1319.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:19 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/serializers.py b/packages/serializers.py index 902cb3e09..b6e3fb83a 100644 --- a/packages/serializers.py +++ b/packages/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from packages.models import PackageName, Package, PackageUpdate +from packages.models import Package, PackageName, PackageUpdate class PackageNameSerializer(serializers.HyperlinkedModelSerializer): diff --git a/packages/utils.py b/packages/utils.py index f00f6710f..87395ff68 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -21,7 +21,9 @@ from django.db import IntegrityError, transaction from arch.models import PackageArchitecture -from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString +from packages.models import ( + Package, PackageCategory, PackageName, PackageString, PackageUpdate, +) from util.logging import error_message, info_message, warning_message diff --git a/packages/views.py b/packages/views.py index c55a6c72c..413faee0e 100644 --- a/packages/views.py +++ b/packages/views.py @@ -15,17 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar -from packages.models import PackageName, Package, PackageUpdate from arch.models import PackageArchitecture -from packages.serializers import PackageNameSerializer, PackageSerializer, PackageUpdateSerializer +from packages.models import Package, PackageName, PackageUpdate +from packages.serializers import ( + PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/patchman/__init__.py b/patchman/__init__.py index af122cc6b..321dd7e57 100644 --- a/patchman/__init__.py +++ b/patchman/__init__.py @@ -14,10 +14,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from .receivers import * # noqa - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app +from .receivers import * # noqa __all__ = ('celery_app',) diff --git a/patchman/celery.py b/patchman/celery.py index 3c58edc56..c47f994da 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -15,10 +15,11 @@ # along with Patchman. If not, see import os + from celery import Celery os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa +from django.conf import settings # noqa app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/patchman/receivers.py b/patchman/receivers.py index 8d8893cab..19312ed50 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -15,16 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from colorama import init, Fore, Style -from tqdm import tqdm - +from colorama import Fore, Style, init +from django.conf import settings from django.dispatch import receiver +from tqdm import tqdm -from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, \ - info_message_s, warning_message_s, error_message_s, debug_message_s - -from django.conf import settings +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, + warning_message_s, +) +from util import create_pbar, get_verbosity, update_pbar init(autoreset=True) diff --git a/patchman/urls.py b/patchman/urls.py index ee786566b..2ae64f562 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -15,12 +15,11 @@ # You should have received a copy of the GNU General Public License # along with If not, see -from django.conf.urls import include, handler404, handler500 # noqa from django.conf import settings +from django.conf.urls import handler404, handler500, include # noqa from django.contrib import admin from django.urls import path from django.views import static - from rest_framework import routers from arch import views as arch_views diff --git a/patchman/wsgi.py b/patchman/wsgi.py index 9a9b4b7fc..16f02d5ac 100644 --- a/patchman/wsgi.py +++ b/patchman/wsgi.py @@ -19,7 +19,6 @@ from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa - +from django.conf import settings # noqa application = get_wsgi_application() diff --git a/reports/admin.py b/reports/admin.py index a37ec4d08..66e0bf5b5 100644 --- a/reports/admin.py +++ b/reports/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from reports.models import Report diff --git a/reports/migrations/0004_migrate_to_tz_aware.py b/reports/migrations/0004_migrate_to_tz_aware.py index 98176510d..20510dcdc 100644 --- a/reports/migrations/0004_migrate_to_tz_aware.py +++ b/reports/migrations/0004_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Report = apps.get_model('reports', 'Report') for report in Report.objects.all(): diff --git a/reports/models.py b/reports/models.py index d529804b9..f1e0f6f3a 100644 --- a/reports/models.py +++ b/reports/models.py @@ -104,7 +104,7 @@ def process(self, find_updates=True, verbose=False): info_message(text=f'Report {self.id} has already been processed') return - from reports.utils import get_arch, get_os, get_domain + from reports.utils import get_arch, get_domain, get_os arch = get_arch(self.arch) osvariant = get_os(self.os, arch) domain = get_domain(self.domain) @@ -113,7 +113,9 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import process_packages, process_repos, process_updates, process_modules + from reports.utils import ( + process_modules, process_packages, process_repos, process_updates, + ) process_repos(report=self, host=host) process_modules(report=self, host=host) process_packages(report=self, host=host) diff --git a/reports/tasks.py b/reports/tasks.py index d2a47e8fd..07fb30040 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -16,7 +16,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from django.db.utils import OperationalError diff --git a/reports/utils.py b/reports/utils.py index 76b6e09cb..8b12f0466 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -23,13 +23,18 @@ from domains.models import Domain from hosts.models import HostRepo from modules.utils import get_or_create_module -from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant +from operatingsystems.utils import ( + get_or_create_osrelease, get_or_create_osvariant, +) from packages.models import Package, PackageCategory -from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from util.logging import info_message +from packages.utils import ( + find_evr, get_or_create_package, get_or_create_package_update, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update -from repos.models import Repository, Mirror, MirrorPackage +from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo +from util.logging import info_message def process_repos(report, host): diff --git a/reports/views.py b/reports/views.py index ccef1bb24..f247deb21 100644 --- a/reports/views.py +++ b/reports/views.py @@ -15,20 +15,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -from django.http import HttpResponse, Http404 -from django.views.decorators.csrf import csrf_exempt -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages from django.db.utils import OperationalError +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from util.filterspecs import Filter, FilterBar from reports.models import Report +from util.filterspecs import Filter, FilterBar @retry( diff --git a/repos/admin.py b/repos/admin.py index bea87567e..a516ff8a8 100644 --- a/repos/admin.py +++ b/repos/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from repos.models import Repository, Mirror, MirrorPackage + +from repos.models import Mirror, MirrorPackage, Repository class MirrorAdmin(admin.ModelAdmin): diff --git a/repos/forms.py b/repos/forms.py index 0800a5c31..9cb66897b 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField, TextInput, Form, ModelChoiceField, ValidationError from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ( + Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, + ValidationError, +) -from repos.models import Repository, Mirror +from repos.models import Mirror, Repository class EditRepoForm(ModelForm): diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878b..1ae96a981 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/repos/migrations/0003_migrate_to_tz_aware.py b/repos/migrations/0003_migrate_to_tz_aware.py index dddd78ba3..38e304884 100644 --- a/repos/migrations/0003_migrate_to_tz_aware.py +++ b/repos/migrations/0003_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Mirror = apps.get_model('repos', 'Mirror') for mirror in Mirror.objects.all(): diff --git a/repos/models.py b/repos/models.py index a1db2a934..9b9082af5 100644 --- a/repos/models.py +++ b/repos/models.py @@ -20,13 +20,12 @@ from arch.models import MachineArchitecture from packages.models import Package -from util import get_setting_of_type - -from repos.repo_types.deb import refresh_deb_repo -from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo +from repos.repo_types.deb import refresh_deb_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from util.logging import info_message, warning_message, error_message +from repos.repo_types.rpm import refresh_repo_errata, refresh_rpm_repo +from util import get_setting_of_type +from util.logging import error_message, info_message, warning_message class Repository(models.Model): diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 09719428c..390b321d1 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,10 +18,13 @@ from io import BytesIO from packages.models import PackageString -from util.logging import info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages -from util import get_datetime_now, get_checksum, Checksum +from repos.utils import ( + fetch_mirror_data, find_mirror_url, get_max_mirrors, + update_mirror_packages, +) +from util import Checksum, get_checksum, get_datetime_now +from util.logging import info_message, warning_message def refresh_arch_repo(repo): diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index c6c26d787..33d1f2c4e 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -15,14 +15,17 @@ # along with Patchman. If not, see import re + from debian.deb822 import Packages from debian.debian_support import Version from packages.models import PackageString -from util.logging import error_message, info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url -from util import get_datetime_now, get_checksum, Checksum, extract +from repos.utils import ( + fetch_mirror_data, find_mirror_url, update_mirror_packages, +) +from util import Checksum, extract, get_checksum, get_datetime_now +from util.logging import error_message, info_message, warning_message def extract_deb_packages(data, url): diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index e440f0d5f..a05846180 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -14,22 +14,28 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import git import os import shutil import tarfile import tempfile -from defusedxml import ElementTree from fnmatch import fnmatch from io import BytesIO from pathlib import Path +import git +from defusedxml import ElementTree + from packages.models import PackageString from packages.utils import find_evr -from util.logging import info_message, warning_message, error_message from patchman.signals import pbar_start, pbar_update -from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages -from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid +from repos.utils import ( + add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages, +) +from util import ( + Checksum, extract, fetch_content, get_checksum, get_datetime_now, get_url, + response_is_valid, +) +from util.logging import error_message, info_message, warning_message def refresh_gentoo_main_repo(repo): diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 516618096..5ffbb7087 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,11 +16,14 @@ from django.db.models import Q -from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo -from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data +from repos.utils import ( + check_for_metalinks, check_for_mirrorlists, fetch_mirror_data, + find_mirror_url, get_max_mirrors, +) from util import get_datetime_now +from util.logging import info_message, warning_message def refresh_repo_errata(repo): diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index bf5940403..e37b9934f 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,10 +17,10 @@ import re from packages.models import PackageString -from util.logging import info_message from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract +from util.logging import info_message def refresh_yast_repo(mirror, data): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index bc0fbc4bb..1e96db399 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -15,17 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from repos.models import Repository diff --git a/repos/utils.py b/repos/utils.py index c11c41ada..29e9cdb3d 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -17,17 +17,24 @@ import re from io import BytesIO -from defusedxml import ElementTree -from tenacity import RetryError +from defusedxml import ElementTree from django.db import IntegrityError from django.db.models import Q +from tenacity import RetryError from packages.models import Package -from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package -from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from util.logging import info_message, warning_message, error_message, debug_message +from packages.utils import ( + convert_package_to_packagestring, convert_packagestring_to_package, +) from patchman.signals import pbar_start, pbar_update +from util import ( + Checksum, extract, fetch_content, get_checksum, get_setting_of_type, + get_url, response_is_valid, +) +from util.logging import ( + debug_message, error_message, info_message, warning_message, +) def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -176,6 +183,7 @@ def add_mirrors_from_urls(repo, mirror_urls): warning_message(text=text) break from repos.models import Mirror + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? # that is what metalink urls return now m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) diff --git a/repos/views.py b/repos/views.py index 199c834ed..1f0c2bfac 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,24 +15,27 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect -from django.http import HttpResponse -from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse -from django.db.models import Q from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError - +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar +from arch.models import MachineArchitecture from hosts.models import HostRepo -from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease -from arch.models import MachineArchitecture -from repos.forms import EditRepoForm, LinkRepoForm, CreateRepoForm, EditMirrorForm -from repos.serializers import RepositorySerializer, MirrorSerializer, MirrorPackageSerializer +from repos.forms import ( + CreateRepoForm, EditMirrorForm, EditRepoForm, LinkRepoForm, +) +from repos.models import Mirror, MirrorPackage, Repository +from repos.serializers import ( + MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/sbin/patchman b/sbin/patchman index 47d89b06b..c415abec2 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -17,32 +17,37 @@ # along with Patchman. If not, see +import argparse import os import sys -import argparse +from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count -from django import setup as django_setup os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa + django_setup() from arch.utils import clean_architectures -from errata.utils import mark_errata_security_updates, enrich_errata, \ - scan_package_updates_for_affected_packages from errata.tasks import update_errata +from errata.utils import ( + enrich_errata, mark_errata_security_updates, + scan_package_updates_for_affected_packages, +) from hosts.models import Host from hosts.utils import clean_tags from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames -from repos.models import Repository -from repos.utils import clean_repos +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from reports.models import Report from reports.tasks import clean_reports_with_no_hosts +from repos.models import Repository +from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import set_verbosity, get_datetime_now +from util import get_datetime_now, set_verbosity from util.logging import info_message diff --git a/security/admin.py b/security/admin.py index 196a9468f..aedeaea9b 100644 --- a/security/admin.py +++ b/security/admin.py @@ -15,8 +15,8 @@ # along with Patchman. If not, see from django.contrib import admin -from security.models import CWE, CVSS, CVE, Reference +from security.models import CVE, CVSS, CWE, Reference admin.site.register(CWE) admin.site.register(CVSS) diff --git a/security/models.py b/security/models.py index 7f674a0b7..0f8482609 100644 --- a/security/models.py +++ b/security/models.py @@ -16,14 +16,14 @@ import json import re -from cvss import CVSS2, CVSS3, CVSS4 from time import sleep +from cvss import CVSS2, CVSS3, CVSS4 from django.db import models from django.urls import reverse from security.managers import CVEManager -from util import get_url, fetch_content, tz_aware_datetime, error_message +from util import error_message, fetch_content, get_url, tz_aware_datetime class Reference(models.Model): diff --git a/security/tasks.py b/security/tasks.py index ce60df83c..7bff4149d 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from security.models import CVE, CWE diff --git a/security/views.py b/security/views.py index 58a686b55..c9e606a6d 100644 --- a/security/views.py +++ b/security/views.py @@ -14,17 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from packages.models import Package from operatingsystems.models import OSRelease +from packages.models import Package from security.models import CVE, CWE, Reference -from security.serializers import CVESerializer, CWESerializer, ReferenceSerializer +from security.serializers import ( + CVESerializer, CWESerializer, ReferenceSerializer, +) from util.filterspecs import Filter, FilterBar diff --git a/setup.py b/setup.py index 6ec6d9744..8e18eaf98 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ # along with Patchman. If not, see import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup with open('VERSION.txt', 'r', encoding='utf_8') as v: version = v.readline().strip() diff --git a/util/__init__.py b/util/__init__.py index c3dfd6bd5..c6c9aa0d4 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -15,30 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import requests import bz2 -import magic -import zlib import lzma import os +import zlib + +import magic +import requests + try: # python 3.14+ - can also remove the dependency at that stage from compression import zstd except ImportError: import zstandard as zstd + from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 -from requests.exceptions import HTTPError, Timeout, ConnectionError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from time import time -from tqdm import tqdm -from django.utils.timezone import make_aware -from django.utils.dateparse import parse_datetime from django.conf import settings +from django.utils.dateparse import parse_datetime +from django.utils.timezone import make_aware +from requests.exceptions import ConnectionError, HTTPError, Timeout +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) +from tqdm import tqdm -from util.logging import error_message, info_message, debug_message +from util.logging import debug_message, error_message, info_message pbar = None verbose = None diff --git a/util/filterspecs.py b/util/filterspecs.py index 722b45dfc..eac0f7477 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.utils.safestring import mark_safe -from django.db.models.query import QuerySet from operator import itemgetter +from django.db.models.query import QuerySet +from django.utils.safestring import mark_safe + def get_query_string(qs): new_qs = [f'{k}={v}' for k, v in list(qs.items())] diff --git a/util/logging.py b/util/logging.py index dd79d296c..cb00ccced 100644 --- a/util/logging.py +++ b/util/logging.py @@ -14,12 +14,12 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see + from datetime import datetime -from patchman.signals import info_message_s -from patchman.signals import warning_message_s -from patchman.signals import error_message_s -from patchman.signals import debug_message_s +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, warning_message_s, +) def info_message(text): diff --git a/util/tasks.py b/util/tasks.py index f650e3e2e..bd76bac62 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -18,7 +18,9 @@ from arch.utils import clean_architectures from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from repos.utils import clean_repos, remove_mirror_trailing_slashes diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 2aea1e5ec..674e1721a 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,16 +15,15 @@ # along with Patchman. If not, see import re - -from humanize import naturaltime from datetime import datetime, timedelta from urllib.parse import urlencode +from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.utils.html import format_html from django.templatetags.static import static -from django.core.paginator import Paginator +from django.utils.html import format_html +from humanize import naturaltime from util import get_setting_of_type diff --git a/util/views.py b/util/views.py index b66db6b06..fd003bca9 100644 --- a/util/views.py +++ b/util/views.py @@ -17,16 +17,16 @@ from datetime import datetime, timedelta -from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F +from django.shortcuts import render from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from repos.models import Repository, Mirror +from operatingsystems.models import OSRelease, OSVariant from packages.models import Package from reports.models import Report +from repos.models import Mirror, Repository from util import get_setting_of_type From 5fa1ef0e3d7484275238b761b05d07782e08a2f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 00:40:08 +0000 Subject: [PATCH 09/21] Bump django from 4.2.25 to 4.2.26 Bumps [django](https://github.com/django/django) from 4.2.25 to 4.2.26. - [Commits](https://github.com/django/django/compare/4.2.25...4.2.26) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.26 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 788f42413..f67fc2cee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.25 +Django==4.2.26 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From aac552d7cf22ee1f6037a9604813cc3bfa870571 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:57:03 +0000 Subject: [PATCH 10/21] Bump django from 4.2.26 to 4.2.27 Bumps [django](https://github.com/django/django) from 4.2.26 to 4.2.27. - [Commits](https://github.com/django/django/compare/4.2.26...4.2.27) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.27 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f67fc2cee..08ce45730 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.26 +Django==4.2.27 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From da1f44e8723d3feeb2982a2251ac08bc9d2b4940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Sat, 20 Dec 2025 08:24:21 +0000 Subject: [PATCH 11/21] Modified tag handling to preserve case --- hosts/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/models.py b/hosts/models.py index 8ea5e3d50..4689ccc5c 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -86,7 +86,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {" ".join(self.tags.slugs())}\n' + text += f'Tags : {" ".join(self.tags.names())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' From d8f3ce4203b16f60b6ffdd55edff1ca354f006e9 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 17 Dec 2025 14:11:34 -0500 Subject: [PATCH 12/21] fix same module in different repos --- .../0005_alter_module_unique_together.py | 19 +++++++++++++++++++ modules/models.py | 2 +- modules/utils.py | 5 ++--- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 modules/migrations/0005_alter_module_unique_together.py diff --git a/modules/migrations/0005_alter_module_unique_together.py b/modules/migrations/0005_alter_module_unique_together.py new file mode 100644 index 000000000..046a0c54d --- /dev/null +++ b/modules/migrations/0005_alter_module_unique_together.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.25 on 2025-11-25 16:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('arch', '0001_initial'), + ('repos', '0006_mirror_errata_checksum_mirror_modules_checksum'), + ('modules', '0004_alter_module_options'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='module', + unique_together={('name', 'stream', 'version', 'context', 'arch', 'repo')}, + ), + ] diff --git a/modules/models.py b/modules/models.py index 931a41c3c..e1f8071b2 100644 --- a/modules/models.py +++ b/modules/models.py @@ -35,7 +35,7 @@ class Module(models.Model): class Meta: verbose_name = 'Module' verbose_name_plural = 'Modules' - unique_together = ['name', 'stream', 'version', 'context', 'arch'] + unique_together = ['name', 'stream', 'version', 'context', 'arch', 'repo'] ordering = ['name', 'stream'] def __str__(self): diff --git a/modules/utils.py b/modules/utils.py index 0d6694785..248b8b453 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -25,10 +25,9 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object Returns the module """ - created = False - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + m_arch, _ = PackageArchitecture.objects.get_or_create(name=arch) try: - module, created = Module.objects.get_or_create( + module, _ = Module.objects.get_or_create( name=name, stream=stream, version=version, From 65e88519e1967b68a767d7a38af8e4792e80aa8a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 1 Jan 2026 17:48:35 -0500 Subject: [PATCH 13/21] add priority queues for tasks (#724) * add priority queues for tasks * Update repos/tasks.py Co-authored-by: code-review-doctor[bot] <72320148+code-review-doctor[bot]@users.noreply.github.com> --- errata/tasks.py | 6 ++--- etc/patchman/local_settings.py | 6 ++--- hosts/tasks.py | 6 ++--- patchman/settings.py | 5 ++++ reports/tasks.py | 47 ++++++++++++++++++++++++++-------- repos/tasks.py | 18 ++++++++++--- sbin/patchman | 4 +-- security/tasks.py | 38 +++++++++++++++++++++------ util/tasks.py | 2 +- 9 files changed, 97 insertions(+), 35 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index 8ded3a79c..9d1f3ed92 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -29,7 +29,7 @@ from util.logging import error_message, warning_message -@shared_task +@shared_task(priority=1) def update_yum_repo_errata(repo_id=None, force=False): """ Update all yum repos errata """ @@ -41,7 +41,7 @@ def update_yum_repo_errata(repo_id=None, force=False): repo.refresh_errata(force) -@shared_task +@shared_task(priority=1) def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ @@ -85,7 +85,7 @@ def update_errata(erratum_type=None, force=False, repo=None): warning_message('Already updating Errata, skipping task.') -@shared_task +@shared_task(priority=2) def update_errata_and_cves(): """ Task to update all errata """ diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 40dd2efd6..15fcb60af 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -56,15 +56,15 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False -# Set the default timeout to e.g. 30 seconds to enable UI caching -# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.redis.RedisCache', 'LOCATION': 'redis://127.0.0.1:6379', - 'TIMEOUT': 0, } } +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time +CACHE_MIDDLEWARE_SECONDS = 0 from datetime import timedelta # noqa diff --git a/hosts/tasks.py b/hosts/tasks.py index 226652f6e..f186760f2 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -22,7 +22,7 @@ from util.logging import info_message -@shared_task +@shared_task(priority=0) def find_host_updates(host_id): """ Task to find updates for a host """ @@ -30,7 +30,7 @@ def find_host_updates(host_id): host.find_updates() -@shared_task +@shared_task(priority=1) def find_all_host_updates(): """ Task to find updates for all hosts """ @@ -38,7 +38,7 @@ def find_all_host_updates(): find_host_updates.delay(host.id) -@shared_task +@shared_task(priority=1) def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ diff --git a/patchman/settings.py b/patchman/settings.py index 557e8c687..23e932c4d 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,6 +13,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', + 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -109,6 +110,10 @@ TAGGIT_CASE_INSENSITIVE = True CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' +CELERY_BROKER_TRANSPORT_OPTIONS = { + 'queue_order_strategy': 'priority', +} +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 LOGIN_REDIRECT_URL = '/patchman/' LOGOUT_REDIRECT_URL = '/patchman/login/' diff --git a/reports/tasks.py b/reports/tasks.py index 07fb30040..6bf16e7ce 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -24,25 +24,50 @@ from util.logging import info_message, warning_message -@shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) +@shared_task( + bind=True, + priority=0, + autoretry_for=(OperationalError,), + retry_backoff=True, + retry_kwargs={'max_retries': 5} +) def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - lock_key = f'process_report_lock_{report_id}' - # lock will expire after 1 hour - lock_expire = 60 * 60 + report_id_lock_key = f'process_report_id_lock_{report_id}' + if report.host: + report_host_lock_key = f'process_report_host_lock_{report.host}' + else: + report_host_lock_key = f'process_report_host_lock_{report.report_ip}' + # locks will expire after 2 hours + lock_expire = 60 * 60 * 2 - if cache.add(lock_key, 'true', lock_expire): + if cache.add(report_id_lock_key, 'true', lock_expire): try: - report.process() + processing_report_id = cache.get(report_host_lock_key) + if processing_report_id: + if processing_report_id > report.id: + warning_message(f'Currently processing a newer report for {report.host} or {report.report_ip}, \ + marking report {report.id} as processed.') + report.processed = True + report.save() + else: + warning_message(f'Currently processing an older report for {report.host} or {report.report_ip}, \ + will skip processing this report.') + else: + try: + cache.set(report_host_lock_key, report.id, lock_expire) + report.process() + finally: + cache.delete(report_host_lock_key) finally: - cache.delete(lock_key) + cache.delete(report_id_lock_key) else: warning_message(f'Already processing report {report_id}, skipping task.') -@shared_task +@shared_task(priority=1) def process_reports(): """ Task to process all unprocessed reports """ @@ -51,9 +76,9 @@ def process_reports(): process_report.delay(report.id) -@shared_task -def clean_reports_with_no_hosts(): - """ Task to clean processed reports where the host no longer exists +@shared_task(priority=2) +def remove_reports_with_no_hosts(): + """ Task to remove processed reports where the host no longer exists """ for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): diff --git a/repos/tasks.py b/repos/tasks.py index bc1776538..a9fdd5f44 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -21,15 +21,25 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=0) def refresh_repo(repo_id, force=False): """ Refresh metadata for a single repo """ - repo = Repository.objects.get(id=repo_id) - repo.refresh(force) + repo_id_lock_key = f'refresh_repos_{repo_id}_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(repo_id_lock_key, 'true', lock_expire): + try: + repo = Repository.objects.get(id=repo_id) + repo.refresh(force) + finally: + cache.delete(repo_id_lock_key) + else: + warning_message(f'Already refreshing repo {repo_id}, skipping task.') -@shared_task +@shared_task(priority=1) def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ diff --git a/sbin/patchman b/sbin/patchman index c415abec2..b76272a27 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from packages.utils import ( clean_packagenames, clean_packages, clean_packageupdates, ) from reports.models import Report -from reports.tasks import clean_reports_with_no_hosts +from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes @@ -156,7 +156,7 @@ def clean_reports(hoststr=None): host.clean_reports() if not hoststr: - clean_reports_with_no_hosts() + remove_reports_with_no_hosts() def host_updates_alt(host=None): diff --git a/security/tasks.py b/security/tasks.py index 7bff4149d..0cfbc2f1c 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -21,15 +21,26 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=3) def update_cve(cve_id): """ Task to update a CVE """ - cve = CVE.objects.get(id=cve_id) - cve.fetch_cve_data() + cve_id_lock_key = f'update_cve_id_lock_{cve_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cve_id_lock_key, 'true', lock_expire): + try: + cve = CVE.objects.get(id=cve_id) + cve.fetch_cve_data() + finally: + cache.delete(cve_id_lock_key) + else: + warning_message(f'Already updating CVE {cve_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cves(): """ Task to update all CVEs """ @@ -47,15 +58,26 @@ def update_cves(): warning_message('Already updating CVEs, skipping task.') -@shared_task +@shared_task(priority=3) def update_cwe(cwe_id): """ Task to update a CWE """ - cwe = CWE.objects.get(id=cwe_id) - cwe.fetch_cwe_data() + cwe_id_lock_key = f'update_cwe_id_lock_{cwe_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cwe_id_lock_key, 'true', lock_expire): + try: + cwe = CWE.objects.get(id=cwe_id) + cwe.fetch_cwe_data() + finally: + cache.delete(cwe_id_lock_key) + else: + warning_message(f'Already updating CWE {cwe_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cwes(): """ Task to update all CWEs """ diff --git a/util/tasks.py b/util/tasks.py index bd76bac62..12825a8c1 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -24,7 +24,7 @@ from repos.utils import clean_repos, remove_mirror_trailing_slashes -@shared_task +@shared_task(priority=1) def clean_database(remove_duplicate_packages=False): """ Task to check the database and remove orphaned objects Runs all clean_* functions to check database consistency From 0338987b7189f89c83634c3a605aa8984342fb75 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 7 Jan 2026 20:46:50 -0500 Subject: [PATCH 14/21] update celery services handling (#726) Signed-off-by: Marcus Furlong --- debian/python3-patchman.install | 3 +- debian/python3-patchman.postinst | 29 +++++++++++++++ etc/patchman/celery.conf | 3 ++ .../system/patchman-celery-beat.service | 19 ++++++++++ .../system/patchman-celery-worker.service | 21 +++++++++++ etc/systemd/system/patchman-celery.service | 14 -------- patchman-client.spec | 17 ++++----- scripts/rpm-post-install.sh | 36 +++++++++++++++---- setup.py | 9 ++++- 9 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 etc/systemd/system/patchman-celery-beat.service create mode 100644 etc/systemd/system/patchman-celery-worker.service delete mode 100644 etc/systemd/system/patchman-celery.service diff --git a/debian/python3-patchman.install b/debian/python3-patchman.install index e13b11ca1..71f47b3a1 100755 --- a/debian/python3-patchman.install +++ b/debian/python3-patchman.install @@ -1,4 +1,5 @@ #!/usr/bin/dh-exec etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf etc/patchman/local_settings.py etc/patchman -etc/systemd/system/patchman-celery.service => lib/systemd/system/patchman-celery.service +etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service +etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index b64cb8165..bce660108 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -25,8 +25,37 @@ if [ "$1" = "configure" ] ; then chown -R www-data:www-data /var/lib/patchman adduser --system --group patchman-celery usermod -a -G www-data patchman-celery + chown root:patchman-celery /etc/patchman/celery.conf + chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + WORKER_COUNT=1 + if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} + fi + + if [ -d /run/systemd/system ]; then + systemctl daemon-reload >/dev/null || true + for i in $(seq 1 "${WORKER_COUNT}"); do + deb-systemd-helper enable "patchman-celery-worker@$i.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-worker@$i.service" >/dev/null || true + done + + active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') + + for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + deb-systemd-invoke stop "$service" >/dev/null || true + deb-systemd-helper disable "$service" >/dev/null || true + fi + done + + deb-systemd-helper enable "patchman-celery-beat.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-beat.service" >/dev/null || true + fi + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/etc/patchman/celery.conf b/etc/patchman/celery.conf index 7afc96eec..2e6f98556 100644 --- a/etc/patchman/celery.conf +++ b/etc/patchman/celery.conf @@ -1,2 +1,5 @@ REDIS_HOST=127.0.0.1 REDIS_PORT=6379 +CELERY_POOL_TYPE=solo +CELERY_WORKER_COUNT=1 +CELERY_CONCURRENCY=1 diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service new file mode 100644 index 000000000..b4ba9f3d2 --- /dev/null +++ b/etc/systemd/system/patchman-celery-beat.service @@ -0,0 +1,19 @@ +[Unit] +Description=Patchman Celery Beat Scheduler Service +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + beat \ + --loglevel info \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service new file mode 100644 index 000000000..8807cbffc --- /dev/null +++ b/etc/systemd/system/patchman-celery-worker.service @@ -0,0 +1,21 @@ +[Unit] +Description=Patchman Celery Worker Service %i +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + worker \ + --task-events \ + --pool ${CELERY_POOL_TYPE:-solo} \ + --concurrency ${CELERY_CONCURRENCY:-1} \ + --hostname patchman-celery-worker%i@%%h + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery.service b/etc/systemd/system/patchman-celery.service deleted file mode 100644 index 6408d818b..000000000 --- a/etc/systemd/system/patchman-celery.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Patchman Celery Service -Requires=network-online.target -After=network-onlne.target - -[Service] -Type=simple -User=patchman-celery -Group=patchman-celery -EnvironmentFile=/etc/patchman/celery.conf -ExecStart=/usr/bin/celery --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads - -[Install] -WantedBy=multi-user.target diff --git a/patchman-client.spec b/patchman-client.spec index 68736038d..f2f8279a9 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -10,7 +10,7 @@ Source: %{expand:%%(pwd)} BuildArch: noarch Requires: curl which coreutils util-linux gawk -%define binary_payload w9.gzdio +%define _binary_payload w9.gzdio %description patchman-client provides a client that uploads reports to a patchman server @@ -20,14 +20,15 @@ find . -mindepth 1 -delete cp -af %{SOURCEURL0}/. . %install -mkdir -p %{buildroot}/usr/sbin -mkdir -p %{buildroot}/etc/patchman -cp ./client/%{name} %{buildroot}/usr/sbin -cp ./client/%{name}.conf %{buildroot}/etc/patchman +mkdir -p %{buildroot}%{_sbindir} +mkdir -p %{buildroot}%{_sysconfdir}/patchman +install -m 755 client/%{name} %{buildroot}%{_sbindir}/%{name} +install -m 644 client/%{name}.conf %{buildroot}%{_sysconfdir}/patchman/%{name}.conf %files -%defattr(755,root,root) -/usr/sbin/patchman-client -%config(noreplace) /etc/patchman/patchman-client.conf +%defattr(-,root,root) +%{_sbindir}/patchman-client +%dir %{_sysconfdir}/patchman +%config(noreplace) %{_sysconfdir}/patchman/patchman-client.conf %changelog diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 24ade8afa..798c43ea9 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -4,8 +4,9 @@ if [ ! -e /etc/httpd/conf.d/patchman.conf ] ; then cp /etc/patchman/apache.conf.example /etc/httpd/conf.d/patchman.conf fi -if ! grep /usr/lib/python3.9/site-packages /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then - sed -i -e "s/^\(Define patchman_pythonpath\).*/\1 \/usr\/lib\/python3.9\/site-packages/" \ +PYTHON_SITEPACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])") +if ! grep "${PYTHON_SITEPACKAGES}" /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then + sed -i -e "s|^\(Define patchman_pythonpath\).*|\1 ${PYTHON_SITEPACKAGES}|" \ /etc/httpd/conf.d/patchman.conf fi @@ -24,15 +25,38 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -chown -R apache:apache /var/lib/patchman adduser --system --group patchman-celery usermod -a -G apache patchman-celery -chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db -chcon --type httpd_sys_rw_content_t /var/lib/patchman/db/patchman.db -semanage port -a -t http_port_t -p tcp 5672 +chown root:patchman-celery /etc/patchman/celery.conf +chmod 640 /etc/patchman/celery.conf + +chown -R apache:apache /var/lib/patchman +semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" +restorecon -Rv /var/lib/patchman/db setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 +WORKER_COUNT=1 +if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} +fi + +for i in $(seq 1 "${WORKER_COUNT}"); do + systemctl enable --now "patchman-celery-worker@$i.service" +done + +active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') +for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + systemctl stop "$service" + systemctl disable "$service" + fi +done + +systemctl enable --now patchman-celery-beat.service + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/setup.py b/setup.py index 8e18eaf98..a7dbfc683 100755 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ # along with Patchman. If not, see import os +import sys from setuptools import find_packages, setup @@ -29,8 +30,14 @@ with open('requirements.txt', 'r', encoding='utf_8') as rt: install_requires = rt.read().splitlines() - data_files = [] +if 'bdist_rpm' in sys.argv: + data_files.append( + ('/usr/lib/systemd/system', [ + 'etc/systemd/system/patchman-celery-worker@.service', + 'etc/systemd/system/patchman-celery-beat.service' + ]) + ) for dirpath, dirnames, filenames in os.walk('etc'): # Ignore dirnames that start with '.' From ff7f293855183e0c23e55a49d04be0d093738ff7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:10 -0500 Subject: [PATCH 15/21] remove non-present middleware (#729) --- patchman/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/patchman/settings.py b/patchman/settings.py index 23e932c4d..c3089caac 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,7 +13,6 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', - 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', From 5e2bb76493da7f9cceb5dbb93f6673857e714175 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:20 -0500 Subject: [PATCH 16/21] fix wsgi so rpm module is only loaded once (#728) --- etc/patchman/apache.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/patchman/apache.conf.example b/etc/patchman/apache.conf.example index 5434cf975..c055eba06 100644 --- a/etc/patchman/apache.conf.example +++ b/etc/patchman/apache.conf.example @@ -1,5 +1,5 @@ Define patchman_pythonpath /srv/patchman/ -WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py +WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py application-group=%{GLOBAL} WSGIPythonPath ${patchman_pythonpath} From 33519f2f465ae2bc9e3b867cc6b3dd630637ff33 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:55:12 -0500 Subject: [PATCH 17/21] give systemd units usable defaults (#727) --- etc/systemd/system/patchman-celery-beat.service | 4 +++- etc/systemd/system/patchman-celery-worker.service | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index b4ba9f3d2..e219d38db 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -7,9 +7,11 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ beat \ --loglevel info \ diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 8807cbffc..51fa9a0e5 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -7,14 +7,18 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" +Environment="CELERY_POOL_TYPE=solo" +Environment="CELERY_CONCURRENCY=1" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ worker \ --task-events \ - --pool ${CELERY_POOL_TYPE:-solo} \ - --concurrency ${CELERY_CONCURRENCY:-1} \ + --pool ${CELERY_POOL_TYPE} \ + --concurrency ${CELERY_CONCURRENCY} \ --hostname patchman-celery-worker%i@%%h [Install] From 6e9b21f804e996da5865c773994bfd25b0e9d899 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 18:38:05 -0500 Subject: [PATCH 18/21] use consistent users/groups on rhel/debian (#730) --- debian/python3-patchman.postinst | 8 ++++---- etc/systemd/system/patchman-celery-beat.service | 4 ++-- etc/systemd/system/patchman-celery-worker.service | 4 ++-- scripts/rpm-post-install.sh | 10 ++++------ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index bce660108..b36a5422c 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -22,12 +22,12 @@ if [ "$1" = "configure" ] ; then patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' - chown -R www-data:www-data /var/lib/patchman - adduser --system --group patchman-celery - usermod -a -G www-data patchman-celery - chown root:patchman-celery /etc/patchman/celery.conf + adduser --quiet --system --group patchman + adduser --quiet www-data patchman + chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + chown -R patchman:patchman /var/lib/patchman WORKER_COUNT=1 if [ -f /etc/patchman/celery.conf ]; then diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index e219d38db..c9bce7228 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 51fa9a0e5..b2d6f6b71 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" Environment="CELERY_POOL_TYPE=solo" diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 798c43ea9..451f7d487 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -25,15 +25,13 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -adduser --system --group patchman-celery -usermod -a -G apache patchman-celery -chown root:patchman-celery /etc/patchman/celery.conf +adduser --system --shell /sbin/nologin patchman +usermod -a -G patchman apache +chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf - -chown -R apache:apache /var/lib/patchman +chown -R patchman:patchman /var/lib/patchman semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" restorecon -Rv /var/lib/patchman/db -setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 WORKER_COUNT=1 From d450af7157305ccc760eb4df622b06f7de2f1799 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 15:10:12 -0500 Subject: [PATCH 19/21] fixes for dumping/loading fixtures from sqlite (#731) --- operatingsystems/managers.py | 4 ++-- security/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/operatingsystems/managers.py b/operatingsystems/managers.py index 630484a18..b7b9f24f0 100644 --- a/operatingsystems/managers.py +++ b/operatingsystems/managers.py @@ -18,5 +18,5 @@ class OSReleaseManager(models.Manager): - def get_by_natural_key(self, name, codename): - return self.get(name=name, codename=codename) + def get_by_natural_key(self, name, codename, cpe_name): + return self.get(name=name, codename=codename, cpe_name=cpe_name) diff --git a/security/models.py b/security/models.py index 0f8482609..405c8db63 100644 --- a/security/models.py +++ b/security/models.py @@ -125,6 +125,8 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] + if isinstance(severity, str): + severity = severity.capitalize() try: cvss, created = CVSS.objects.get_or_create( version=version, From f798882834ed58d2f8e2c9f1c03a7cc472605b57 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 16:54:39 -0500 Subject: [PATCH 20/21] update logging to log to console and celery systemd units (#732) --- etc/patchman/local_settings.py | 17 +++++++++ patchman/receivers.py | 25 ++++++++------ reports/utils.py | 4 ++- repos/repo_types/arch.py | 2 +- repos/repo_types/deb.py | 2 +- repos/repo_types/gentoo.py | 4 +-- repos/repo_types/rpm.py | 2 +- repos/utils.py | 2 +- sbin/patchman | 6 ++-- util/__init__.py | 42 +++-------------------- util/logging.py | 63 +++++++++++++++++++++++++++++----- 11 files changed, 103 insertions(+), 66 deletions(-) diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 15fcb60af..cb19e2685 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -96,3 +96,20 @@ 'schedule': timedelta(hours=24), }, } + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + }, + 'loggers': { + 'urllib3': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + 'git': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + } +} diff --git a/patchman/receivers.py b/patchman/receivers.py index 19312ed50..9393d8912 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -18,13 +18,13 @@ from colorama import Fore, Style, init from django.conf import settings from django.dispatch import receiver -from tqdm import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, warning_message_s, ) -from util import create_pbar, get_verbosity, update_pbar +from util.logging import create_pbar, get_quiet_mode, logger, update_pbar init(autoreset=True) @@ -53,8 +53,10 @@ def print_info_message(**kwargs): """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.RESET_ALL + Fore.RESET + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(loggers=[logger]): + for line in text.splitlines(): + logger.info(Style.RESET_ALL + Fore.RESET + line) @receiver(warning_message_s) @@ -62,8 +64,9 @@ def print_warning_message(**kwargs): """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.BRIGHT + Fore.YELLOW + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(): + logger.warning(Style.BRIGHT + Fore.YELLOW + text) @receiver(error_message_s) @@ -72,13 +75,15 @@ def print_error_message(**kwargs): """ text = str(kwargs.get('text')) if text: - tqdm.write(Style.BRIGHT + Fore.RED + text) + with logging_redirect_tqdm(): + logger.error(Style.BRIGHT + Fore.RED + text) @receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to handle a debug message, blue text if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if DEBUG is set """ text = str(kwargs.get('text')) - if get_verbosity() and settings.DEBUG and text: - tqdm.write(Style.BRIGHT + Fore.BLUE + text) + if settings.DEBUG and text: + with logging_redirect_tqdm(loggers=[logger]): + logger.debug(Style.BRIGHT + Fore.BLUE + text) diff --git a/reports/utils.py b/reports/utils.py index 8b12f0466..1a08cf3c2 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -34,7 +34,7 @@ from patchman.signals import pbar_start, pbar_update from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo -from util.logging import info_message +from util.logging import debug_message, info_message def process_repos(report, host): @@ -47,6 +47,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): + debug_message(f'Processing report {report.id} repo: {repo_str}') repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) @@ -93,6 +94,7 @@ def process_packages(report, host): packages = parse_packages(report.packages) pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): + debug_message(f'Processing report {report.id} package: {pkg_str}') package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 390b321d1..b339ee6a8 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -51,7 +51,7 @@ def refresh_arch_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Arch Repo data') if not package_data: continue diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 33d1f2c4e..eea6593f6 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -95,7 +95,7 @@ def refresh_deb_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Debian Repo data') if not package_data: continue diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index a05846180..61966f7a1 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -51,7 +51,7 @@ def refresh_gentoo_main_repo(repo): continue res = get_url(mirror.url + '.md5sum') - data = fetch_content(res, 'Fetching Repo checksum') + data = fetch_content(res, 'Fetching Gentoo Repo checksum') if data is None: mirror.fail() continue @@ -72,7 +72,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Gentoo Repo data') if data is None: mirror.fail() continue diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 5ffbb7087..d1482272a 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -84,7 +84,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): repo_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching rpm Repo data') if not repo_data: continue diff --git a/repos/utils.py b/repos/utils.py index 29e9cdb3d..13cee149e 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -155,7 +155,7 @@ def get_mirrorlist_urls(url): return if response_is_valid(res): try: - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Repo data to check for mirrorlist') if data is None: return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) diff --git a/sbin/patchman b/sbin/patchman index b76272a27..06bef981a 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -47,8 +47,8 @@ from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import get_datetime_now, set_verbosity -from util.logging import info_message +from util import get_datetime_now +from util.logging import info_message, set_quiet_mode def get_host(host=None, action='Performing action'): @@ -547,7 +547,7 @@ def main(): parser = collect_args() args = parser.parse_args() - set_verbosity(not args.quiet) + set_quiet_mode(args.quiet) showhelp = process_args(args) if showhelp: parser.print_help() diff --git a/util/__init__.py b/util/__init__.py index c6c9aa0d4..b85e5e379 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -41,12 +41,14 @@ from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) -from tqdm import tqdm -from util.logging import debug_message, error_message, info_message +from util.logging import ( + create_pbar, debug_message, error_message, info_message, quiet_mode, + update_pbar, +) pbar = None -verbose = None +verbose = not quiet_mode Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') http_proxy = os.getenv('http_proxy') @@ -57,40 +59,6 @@ } -def get_verbosity(): - """ Get the global verbosity level - """ - return verbose - - -def set_verbosity(value): - """ Set the global verbosity level - """ - global verbose - verbose = value - - -def create_pbar(ptext, plength, ljust=35, **kwargs): - """ Create a global progress bar if global verbose is True - """ - global pbar - if verbose and plength > 0: - jtext = str(ptext).ljust(ljust) - pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') - return pbar - - -def update_pbar(index, **kwargs): - """ Update the global progress bar if global verbose is True - """ - global pbar - if verbose and pbar: - pbar.update(n=index-pbar.n) - if index >= pbar.total: - pbar.close() - pbar = None - - def fetch_content(response, text='', ljust=35): """ Display a progress bar to fetch the request content if verbose is True. Otherwise, just return the request content diff --git a/util/logging.py b/util/logging.py index cb00ccced..bf532e549 100644 --- a/util/logging.py +++ b/util/logging.py @@ -15,28 +15,73 @@ # along with Patchman. If not, see -from datetime import datetime +import logging + +from django.conf import settings +from tqdm import tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, warning_message_s, ) +log_format = '[%(asctime)s] %(levelname)s: %(message)s' +if settings.DEBUG: + logging_level = logging.DEBUG +else: + logging_level = logging.INFO +logging.basicConfig(level=logging_level, format=log_format) +logger = logging.getLogger() +logging.getLogger('git.cmd').setLevel(logging.WARNING) + +quiet_mode = False +pbar = None + + +def get_quiet_mode(): + """ Get the global quiet_mode + """ + return quiet_mode + + +def set_quiet_mode(value): + """ Set the global quiet_mode + """ + global quiet_mode + quiet_mode = value + + +def create_pbar(ptext, plength, ljust=35, **kwargs): + """ Create a global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and plength > 0: + jtext = str(ptext).ljust(ljust) + pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') + return pbar + + +def update_pbar(index, **kwargs): + """ Update the global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and pbar: + pbar.update(n=index-pbar.n) + if index >= pbar.total: + pbar.close() + pbar = None + def info_message(text): - ts = datetime.now() - info_message_s.send(sender=None, text=text, ts=ts) + info_message_s.send(sender=None, text=text) def warning_message(text): - ts = datetime.now() - warning_message_s.send(sender=None, text=text, ts=ts) + warning_message_s.send(sender=None, text=text) def debug_message(text): - ts = datetime.now() - debug_message_s.send(sender=None, text=text, ts=ts) + debug_message_s.send(sender=None, text=text) def error_message(text): - ts = datetime.now() - error_message_s.send(sender=None, text=text, ts=ts) + error_message_s.send(sender=None, text=text) From aa43a631bcf87b8c7c108532cd2c832f59174131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 15 Jan 2026 16:38:31 +0000 Subject: [PATCH 21/21] Changed entrypoint script to adapt changes made in 8fa3eb2 --- docker/docker-entrypoint.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index cf0ac72f1..97a75152d 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -92,17 +92,16 @@ if "${USE_CACHE}"; then redisPort="6379" fi - # Comment DummyCache Block - sed -i '47,51 {/^#/ ! s/\(.*\)/#\1/}' "$conf" - - # Uncomment RedisCache Block - sed -i '55,61 {s/^# //}' "$conf" - - sed -i "58 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" + # Change RedisCache LOCATION + sed -i "62 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" if [ -n "${CACHE_TIMEOUT}" ]; then - sed -i "59 {s/30/${CACHE_TIMEOUT}/}" "$conf" + sed -i "67 {s/0/${CACHE_TIMEOUT}/}" "$conf" fi +else + # Change RedisCache to DummyCache to avoid ConnectionError and comment LOCATION + sed -i '61 {s/redis.RedisCache/dummy.DummyCache/}' "$conf" + sed -i '62 {/^#/ ! s/\(.*\)/#\1/}' "$conf" fi # Sync database on container first start