From 129ad243c8d591e7a3da1cca1b51ec3b83b281d9 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 24 Jul 2023 22:24:31 +0300 Subject: [PATCH 1/8] Move base file structure to dedicated repo. Signed-off-by: Nashwan Azhari --- README.rst | 5 ++ autolabeler/__init__.py | 0 autolabeler/main.py | 126 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 35 +++++++++++ setup.py | 18 ++++++ 5 files changed, 184 insertions(+) create mode 100644 README.rst create mode 100644 autolabeler/__init__.py create mode 100644 autolabeler/main.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..85480c5 --- /dev/null +++ b/README.rst @@ -0,0 +1,5 @@ +Cloudbase GitHub Auto-Labeler +============================= + +Simple Python3 utility for automatically labelling issues and pull requests +with minimal configuration. diff --git a/autolabeler/__init__.py b/autolabeler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/autolabeler/main.py b/autolabeler/main.py new file mode 100644 index 0000000..58dfcd2 --- /dev/null +++ b/autolabeler/main.py @@ -0,0 +1,126 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os + + +def _add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + if not parser: + raise ValueError("No parser supplied") + + parser.add_argument( + "target", + help="""The full 'user/repo' shorthand of the target GitHub repository. + Optional suffixes may also be used, such as: + Adding 'issue/123' or 'pull/456' will target a specific issues/PR. + Adding 'issues/open' (note the plural) will target all open issues. + Adding 'issues' or 'pulls' will target all issues/PRs.""") + parser.add_argument( + "-t", "--github-token", default=os.environ.get("GITHUB_TOKEN"), + help="String GitHub API token to make labelling API calls. " + "It can be both a 'classic' GitHub tokens, or with the new " + "'fine-grained' GitHub tokens which include R/W access to " + "the Issues and Pull Requests.") + parser.add_argument( + "-l", "--label-definitions-file", type=argparse.FileType('r'), + help="String path to a JSON/YAML file containing label definitions.") + parser.add_argument( + "-r", "--replies-definitions-file", type=argparse.FileType('r'), + help="String path to a JSON/YAML file containing issue/PR autoreply " + "rules.") + + return parser + + +def _parse_target_argument(target: str) -> dict: + """ Parses the prvided target for the labelling action. + + Supports inputs of the form: + - username/repository_name + - user/repo/{issues/pulls} for all issues/pulls + - user/repo/{issue/pull}/123 for a specific issue/pull + + Returns dict of the form: { + "original": "", + "user": "", + "repo": "", + "type": None/"issue"/"pull", + "id": None/int, + } + """ + parts = target.split('/') + if len(parts) not in range(2, 5): + raise ValueError( + "Target format must be a slash-separated string with the " + "following path elements: username/repository[/type[/id]]. " + f"Got: {target} ({len(parts)} path elements)") + + def _safe_index(n: int): + try: + return parts[n] + except IndexError: + return None + + target_type = None + match _safe_index(2): + case None: + pass + # NOTE: special concession to accept 'pulls/issues' as plurals as well. + case 'pull' | 'pulls': + target_type = 'pull' + case 'issue' | 'issues': + target_type = 'issue' + case other: + accepted = ['pull(s)', 'issue(s)'] + raise ValueError( + f"Unsupported target type '{other}'. Must be one of: {accepted}") + + target_id = None + match _safe_index(3): + case None: + pass + case some: + target_id = int(some) + + return { + "original": target, + "user": parts[0], + "repo": parts[1], + "type": target_type, + "id": target_id, + } + + +def main(): + parser = argparse.ArgumentParser( + "github-autolabeler", + description="Python 3 utility for automatically labelling/triaging " + "GitHub issues and pull requests.") + + parser = _add_arguments(parser) + + args = parser.parse_args() + + if not args.github_token: + raise ValueError( + f"No GitHub API token provided via '-t/--github-token' argument " + "or 'GITHUB_TOKEN' environment variable.") + + # TODO(aznashwan): + # - parse config files + # - load rules + # - check labels generated + + print(f"{_parse_target_argument(args.target)}") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..929b9d2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "github-autolabeler" +version = "0.0.1" +description = "Python3 utility for easy labelling/triaging of GitHub PRs/Issues." +authors = [ + { name = "Cloudbase Solutions Srl", email = "support@cloudbasesolutions.com" }, +] +license = { file = "LICENSE" } +readme = "README.rst" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GPLv3", + "Operating System :: OS Independent", +] + +# NOTE(aznashwan): `match` statements require 3.10. +requires-python = ">=3.10" +dependencies = [ + "pyyaml", + "PyGithub", +] + +[project.urls] +"Homepage" = "https://github.com/cloudbase/github-autolabeler" +"Bug Tracker" = "https://github.com/cloudbase/github-autolabeler/issues" + +[build-system] +requires = ["setuptools >= 61.0.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["autolabeler"] + +[project.scripts] +github-autolabeler = "autolabeler.main:main" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d292d16 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import setuptools + +if __name__ == "__main__": + setuptools.setup() From abe616910bc0ecf3fbf66253b9baa72a404f3caa Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Tue, 25 Jul 2023 02:28:42 +0300 Subject: [PATCH 2/8] Add targets skeleton. Signed-off-by: Nashwan Azhari --- autolabeler/main.py | 44 +++++++++++++++++++++++++--- autolabeler/targets.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 autolabeler/targets.py diff --git a/autolabeler/main.py b/autolabeler/main.py index 58dfcd2..29b2396 100644 --- a/autolabeler/main.py +++ b/autolabeler/main.py @@ -13,8 +13,35 @@ # under the License. import argparse +import logging import os +import github +from github import Auth + +from autolabeler import targets + + +def _setup_logging(): + # create logger + logger = logging.getLogger('github-autolabeler') + logger.setLevel(logging.DEBUG) + + # create console handler and set level to debug + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + + return logger + def _add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: if not parser: @@ -30,8 +57,8 @@ def _add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: parser.add_argument( "-t", "--github-token", default=os.environ.get("GITHUB_TOKEN"), help="String GitHub API token to make labelling API calls. " - "It can be both a 'classic' GitHub tokens, or with the new " - "'fine-grained' GitHub tokens which include R/W access to " + "It can be both a 'classic' GitHub token, or a new " + "'fine-grained' GitHub token which includes R/W access to " "the Issues and Pull Requests.") parser.add_argument( "-l", "--label-definitions-file", type=argparse.FileType('r'), @@ -45,7 +72,7 @@ def _add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def _parse_target_argument(target: str) -> dict: - """ Parses the prvided target for the labelling action. + """ Parses the provided target for the labelling action. Supports inputs of the form: - username/repository_name @@ -104,6 +131,8 @@ def _safe_index(n: int): def main(): + LOG = _setup_logging() + parser = argparse.ArgumentParser( "github-autolabeler", description="Python 3 utility for automatically labelling/triaging " @@ -123,4 +152,11 @@ def main(): # - load rules # - check labels generated - print(f"{_parse_target_argument(args.target)}") + LOG.info(f"{_parse_target_argument(args.target)}") + + gh = github.Github(login_or_token=args.github_token) + # NOTE(aznashwan): required to load user: + _ = gh.get_user().login + gh.load(f) + + _ = targets.RepoLabelsTarget(gh, "aznashwan", "cloudbase-init") diff --git a/autolabeler/targets.py b/autolabeler/targets.py new file mode 100644 index 0000000..ca3b400 --- /dev/null +++ b/autolabeler/targets.py @@ -0,0 +1,66 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging + +from github import Github +from github.Label import Label + +LOG = logging.getLogger() + + +class BaseLabelsTarget(metaclass=abc.ABCMeta): + """ ABC for GitHub API entities which can have labels applies to them. """ + + # @abc.abstractclassmethod + # def from_str(cls, client: Github, target_str: str) -> Self: + # LOG.debug(f"BaseLabelsTarget.from_str({client}, {target_str})") + # raise NotImplemented + + @abc.abstractmethod + def get_labels(self) -> list[Label]: + raise NotImplemented + + @abc.abstractmethod + def set_labels(self, labels: list[Label]): + raise NotImplemented + + @abc.abstractmethod + def remove_labels(self, labels: list[str]): + raise NotImplemented + + +class RepoLabelsTarget(BaseLabelsTarget): + + def __init__(self, client: Github, repo_user: str, repo_name: str): + self._client = client + self._repo_user = repo_user + self._repo_name = repo_name + self._repo_handle = client.get_repo(f"{repo_user}/{repo_name}") + + def get_labels(self) -> list[Label]: + return list(self._repo_handle.get_labels()) + + def set_labels(self, labels: list[Label]): + for label in labels: + self._repo_handle.create_label(label.name, label.color) + + def remove_labels(self, labels: list[str]): + existing_labels_map = {l.name: l for l in self.get_labels()} + + for label in labels: + label_obj = existing_labels_map.get(label) + if label_obj: + label_obj.delete() From c998b2f32a990491ffe87e1b33ccb0bd613606e8 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Tue, 25 Jul 2023 19:11:30 +0300 Subject: [PATCH 3/8] Save. Signed-off-by: Nashwan Azhari --- autolabeler/labelers.py | 94 +++++++++++++++ autolabeler/selectors.py | 192 +++++++++++++++++++++++++++++++ samples/autolabels-showcase.yaml | 74 ++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 autolabeler/labelers.py create mode 100644 autolabeler/selectors.py create mode 100644 samples/autolabels-showcase.yaml diff --git a/autolabeler/labelers.py b/autolabeler/labelers.py new file mode 100644 index 0000000..843a644 --- /dev/null +++ b/autolabeler/labelers.py @@ -0,0 +1,94 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging +from typing import Self + +from github.Issue import Issue +from github.Label import Label +from github.PullRequest import PullRequest +from github.Repository import Repository + +from selectors import Selector + + +LOG = logging.getLogger() + + +class BaseLabeler(metaclass=abc.ABCMeta): + + @abc.abstractmethod + def get_labels_for_repo(self, repo: Repository) -> list[Label]: + raise NotImplemented + + @abc.abstractmethod + def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: + raise NotImplemented + + @abc.abstractmethod + def get_lebels_for_issue(self, issue: Issue) -> list[Label]: + raise NotImplemented + + +class Labeler(BaseLabeler): + def __init__(self, + label_name: str, + label_color: str, + selectors: list[Selector], + prefix: str=""): + self._name = label_name + self._color = label_color + self._selectors = selectors + self._prefix = prefix + + @classmethod + def from_dict(cls, label_name: str, val: dict, prefix: str="") -> Self: + required_fields = [ + "label-color", "label-description"] + if not type(val) is dict: + raise TypeError( + f"Expected dict with keys {required_fields}, " + f"got {val} ({type(val)})") + + missing = [field in val for field in required_fields] + if missing: + raise ValueError( + f"Missing required fields {missing} in Labeler definition: {val}") + + sels_defs = val.get("selectors", []) + sels = [Selector.from_dict(s) for s in sels_defs] + + return cls(label_name, val['label-color'], sels, prefix=prefix) + + + + +class PrefixLabeler(BaseLabeler): + """ Class for handling label prefix groups. + + Simply aggregates/delegates labels generation to the contained concrete + label generators and just adds its prefix to all labels. + """ + + def __init__(self, prefix: str, sublabelers: list[BaseLabeler], + separator='/'): + self._prefix = prefix + self._separator = separator + self._sublabelers = sublabelers + + def _prefix_labels(self, labels: list[Label]) -> list[Label]: + for l in labels: + l.name = f"{self._prefix}{self._separator}{l.name}" + return labels diff --git a/autolabeler/selectors.py b/autolabeler/selectors.py new file mode 100644 index 0000000..33ad57e --- /dev/null +++ b/autolabeler/selectors.py @@ -0,0 +1,192 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging +import re +from typing import Self + +from github.Issue import Issue +from github.PullRequest import PullRequest +from github.Repository import Repository + + +LOG = logging.getLogger() + + +class Selector(withmetaclass=abc.ABCMeta): + + @abc.abstractclassmethod + def from_dict(cls, val: dict): + raise NotImplemented + + @abc.abstractmethod + def match(self, obj: object) -> dict: + """ Returns a dict of str key-values if the Github object is matched. + + Returned keys should follow a predictable namespacing model based on the + checked property, so for example if a selector has "check1"="", + the returned dict should contain: { + "check1": "" + } + """ + raise NotImplemented + + +class RegexSelector(Selector): + """ Selects Issues/PRs based on regexes of their title/comments. """ + + def __init__(self, + re_title: str="", re_description: str="", + re_comments: str="", maintainer_comments_only: bool=True, + case_insensitive: bool=False): + self._re_title = re_title + self._re_description = re_description + self._re_comments = re_comments + self._re_maintainer_comments = maintainer_comments_only + self._case_insensitive = case_insensitive + + @classmethod + def from_dict(cls, val: dict) -> Self: + """ Supports dict with following keys: + + case_insensitive: bool, + title: "", + description: "", + comments: "", + maintainer-comments-only: bool, + """ + supported_keys = [ + "case_insensitive", "title", "description", + "comments", "maintainer-comments", "maintainer-comments-only"] + if not any(k in val for k in supported_keys): + raise ValueError( + f"FileRegexSelector requires at least one regex key " + f"({supported_keys}). Got {val}") + + kwargs = { + "case_insensitive": val.get("case_insensitive", False), + "re_title": val.get("title", ""), + "re_description": val.get("description", ""), + "re_comments": val.get("comments", ""), + "maintainer_comments_only": val.get( + "maintainer-comments-only", True)} + + return cls(**kwargs) + + def _get_title_description_matches(self, obj: PullRequest|Issue) -> dict: + res = {} + + if self._re_title: + res.update( + _get_match_groups( + self._re_title, obj.title, prefix="title-", + case_insensitive=self._case_insensitive)) + + if self._re_description: + res.update( + _get_match_groups( + # NOTE: PyGithub refers to issue/PR descriptions as their "body". + self._re_description, obj.body, prefix="description-", + case_insensitive=self._case_insensitive)) + + return res + + def _get_comment_matches(self, obj: Issue|PullRequest) -> dict: + if self._re_comments: + raise NotImplemented + + # TODO(aznashwan): + # pr.get_issue_comments/get_comments/get_review_comments() and match + return {} + + def match(self, obj: object) -> dict: + # NOTE(aznashwan): Only Issues/PRs have the concept of comments. + if not isinstance(obj, (Issue, PullRequest)): + LOG.warn( + f"RegexSelector got unsupported object type {type(obj)}: {obj}") + return {} + + res = {} + res.update(self._get_comment_matches(obj)) + res.update(self._get_title_description_matches(obj)) + + return res + + +class FilesSelector(Selector): + """ Selects Repos/PRs based on contained file properties. """ + + def __init__(self, file_name_re: str="", file_type: str=""): + self._file_type = file_type + self._file_name_re = file_name_re + + @classmethod + def from_dict(cls, val: dict) -> Self: + supported_keys = ["name-regex", "type"] + if not any(k in val for k in supported_keys): + raise ValueError( + f"FilesSelector requires at least one options key " + f"({supported_keys}). Got {val}") + + kwargs = { + "file_name_re": val.get("name-regex", ""), + "file_type": val.get("type", "")} + + return cls(**kwargs) + + def match(self, obj: Repository|PullRequest) -> dict: + # NOTE(aznashwan): Only Repositories/PRs have associated files. + if not isinstance(obj, (Repository, PullRequest)): + LOG.warn( + f"FileSelector got unsupported object type {type(obj)}: {obj}") + return {} + + res = {} + + # TODO: + # - recusively gather all files and directories. + # - match them all + + return res + + +SELECTORS_NAME_MAP = { + "regex": RegexSelector, + "files": FilesSelector +} + + +def get_selector_cls(selector_name, raise_if_missing=True): + selector = SELECTORS_NAME_MAP.get(selector_name) + if raise_if_missing and not selector: + raise ValueError( + f"Unknown selector type {selector_name}. Supported selectors are: " + f"{list(SELECTORS_NAME_MAP.keys())}") + + +def _get_match_groups( + regex: str, value: str, prefix="", case_insensitive=False) -> dict: + """ Returns dict with all match groups and keys of the form 'prefix-group-N'""" + res = {} + flags = [] + if case_insensitive: + flags = [re.IGNORECASE] + + match = re.match(regex, value, *flags) + if match: + for i, group in enumerate(match.groups()): + res[f"{prefix}group-{i}"] = group + + return res diff --git a/samples/autolabels-showcase.yaml b/samples/autolabels-showcase.yaml new file mode 100644 index 0000000..4679cf2 --- /dev/null +++ b/samples/autolabels-showcase.yaml @@ -0,0 +1,74 @@ +# This defines a static static label named 'example-label' with the given props. +example-label: + color: 0075ca # blue + description: This is a simple example static label. + + +# This defines two "namespaced" labels named: label-prefix/sublabel-{1,2}. +label-prefix: + sublabel-1: + color: 0075ca + description: This is a simple first sublabel. + sublabel-2: + color: d73a4a # red + description: This is a simple second sublabel. + + +# This will define as many labels as there are filepath regex matches. +label-for-sample-file-{files-name-regex-group-1}: + color: 0075ca + description: This label was created for file {files-name-regex-group-1}. + selectors: + files: + # This will specify a regex to be matched against full filepaths in + # the repo tree. 'files-name-group-0' will be the entire filename. + name-regex: "samples/(.*).yaml" + + # TODO(aznashwan): + # type: "" + # size: min/max filesizes for some "kind/bigblob"-type labels? + + +# This will define a label named "bug" which will automatically be +# associated with issues/PRs matching the provided regexes. +bug: + color: d73a4a + description: This label signifies the Issue/PR is about a bug. + selectors: + regex: + # This allows for case insensitive searches. (default is false) + case-insensitive: true + + # This will make the label be applied to PRs/Issues whose titles/descriptions + # match the given regex. (in this case, some simple keywords) + title: "(issue|bug|fix|problem|failure|error)" + description: "(issue|bug|problem|failure|error)" + + # This will make the label applied to PRs/Issues where anyone comments + # the given magic string on a single line within the comment. + comments: "^(/label-me-as bug)$" + # Only allows maintainers to add this label via comment. (default is true) + maintainer-comments-only: false + + +# This will define two labels named "pr-size-measure/{small,large}" that will +# automatically get applied to Pull Requsts whose change counts match. +pr-size-measure: + small: + color: 0075ca + description: | + This PR is between {lines-changed-min} and {lines-changed-max} lines of code. + selectors: + lines-changed: + min: 0 + max: 1000 + + large: + color: d73a4a + description: | + This PR is between {lines-changed-min} and {lines-changed-max} lines of code. + selectors: + lines-changed: + min: 1000 + # NOTE: setting to zero will remove any upper bound. + max: 0 From df39f2566684bd32127792e7f2d0cbcb3d7fb43a Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Tue, 25 Jul 2023 20:45:07 +0300 Subject: [PATCH 4/8] Add baseline selectors. Signed-off-by: Nashwan Azhari --- autolabeler/labelers.py | 51 ++++++++++++++++++++++++-------- autolabeler/selectors.py | 3 ++ samples/autolabels-showcase.yaml | 2 +- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/autolabeler/labelers.py b/autolabeler/labelers.py index 843a644..d45596e 100644 --- a/autolabeler/labelers.py +++ b/autolabeler/labelers.py @@ -21,7 +21,7 @@ from github.PullRequest import PullRequest from github.Repository import Repository -from selectors import Selector +from autolabeler import selectors LOG = logging.getLogger() @@ -38,15 +38,15 @@ def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: raise NotImplemented @abc.abstractmethod - def get_lebels_for_issue(self, issue: Issue) -> list[Label]: + def get_labels_for_issue(self, issue: Issue) -> list[Label]: raise NotImplemented -class Labeler(BaseLabeler): +class SelectorLabeler(BaseLabeler): def __init__(self, label_name: str, label_color: str, - selectors: list[Selector], + selectors: list[selectors.Selector], prefix: str=""): self._name = label_name self._color = label_color @@ -67,14 +67,19 @@ def from_dict(cls, label_name: str, val: dict, prefix: str="") -> Self: raise ValueError( f"Missing required fields {missing} in Labeler definition: {val}") - sels_defs = val.get("selectors", []) - sels = [Selector.from_dict(s) for s in sels_defs] + sels = [] + sels_defs = val.get("selectors", {}) + for sname, sbody in sels_defs.items(): + try: + scls = selectors.get_selector_cls(sname, raise_if_missing=True) + sels.append(scls.from_dict(sbody)) + except Exception as ex: + raise ValueError( + f"Failed to load selector '{sname}' from payload {sbody}") from ex return cls(label_name, val['label-color'], sels, prefix=prefix) - - class PrefixLabeler(BaseLabeler): """ Class for handling label prefix groups. @@ -88,7 +93,29 @@ def __init__(self, prefix: str, sublabelers: list[BaseLabeler], self._separator = separator self._sublabelers = sublabelers - def _prefix_labels(self, labels: list[Label]) -> list[Label]: - for l in labels: - l.name = f"{self._prefix}{self._separator}{l.name}" - return labels + def get_labels_for_repo(self, repo: Repository) -> list[Label]: + raise NotImplemented + + def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: + raise NotImplemented + + def get_labels_for_issue(self, issue: Issue) -> list[Label]: + raise NotImplemented + + + +def load_labelers_from_config(config: dict) -> list[BaseLabeler]: + toplevel_lebelers = [] + for key, val in config.items(): + # Assume it's a plain labeler until proven otherwise. + try: + toplevel_lebelers.append(SelectorLabeler.from_dict(key, val)) + continue + except ValueError as err: + LOG.info( + f"Failed to load lebeler on key {key}, assuming it's a prefix: {err}") + + prefixer = PrefixLabeler(key, load_labelers_from_config(val)) + toplevel_lebelers.append(prefixer) + + return toplevel_lebelers diff --git a/autolabeler/selectors.py b/autolabeler/selectors.py index 33ad57e..332ff24 100644 --- a/autolabeler/selectors.py +++ b/autolabeler/selectors.py @@ -29,6 +29,7 @@ class Selector(withmetaclass=abc.ABCMeta): @abc.abstractclassmethod def from_dict(cls, val: dict): + _ = val raise NotImplemented @abc.abstractmethod @@ -104,6 +105,7 @@ def _get_title_description_matches(self, obj: PullRequest|Issue) -> dict: return res def _get_comment_matches(self, obj: Issue|PullRequest) -> dict: + _ = obj if self._re_comments: raise NotImplemented @@ -174,6 +176,7 @@ def get_selector_cls(selector_name, raise_if_missing=True): raise ValueError( f"Unknown selector type {selector_name}. Supported selectors are: " f"{list(SELECTORS_NAME_MAP.keys())}") + return selector def _get_match_groups( diff --git a/samples/autolabels-showcase.yaml b/samples/autolabels-showcase.yaml index 4679cf2..74a27fa 100644 --- a/samples/autolabels-showcase.yaml +++ b/samples/autolabels-showcase.yaml @@ -26,7 +26,7 @@ label-for-sample-file-{files-name-regex-group-1}: # TODO(aznashwan): # type: "" - # size: min/max filesizes for some "kind/bigblob"-type labels? + # size: min/max filesizes for some "kind/bigblob"-type label combos? # This will define a label named "bug" which will automatically be From 38e546083f5b8132de48e979296c89ecf65ea965 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Wed, 26 Jul 2023 00:12:41 +0300 Subject: [PATCH 5/8] Got labelers loading. Signed-off-by: Nashwan Azhari --- autolabeler/labelers.py | 37 +++++++++++++++------- autolabeler/main.py | 54 +++++++++++++------------------- autolabeler/selectors.py | 24 +++++++++----- autolabeler/utils.py | 46 +++++++++++++++++++++++++++ samples/autolabels-showcase.yaml | 28 ++++++++--------- 5 files changed, 122 insertions(+), 67 deletions(-) create mode 100644 autolabeler/utils.py diff --git a/autolabeler/labelers.py b/autolabeler/labelers.py index d45596e..a931250 100644 --- a/autolabeler/labelers.py +++ b/autolabeler/labelers.py @@ -13,7 +13,6 @@ # under the License. import abc -import logging from typing import Self from github.Issue import Issue @@ -22,9 +21,10 @@ from github.Repository import Repository from autolabeler import selectors +from autolabeler import utils -LOG = logging.getLogger() +LOG = utils.getStdoutLogger(__name__) class BaseLabeler(metaclass=abc.ABCMeta): @@ -62,7 +62,7 @@ def from_dict(cls, label_name: str, val: dict, prefix: str="") -> Self: f"Expected dict with keys {required_fields}, " f"got {val} ({type(val)})") - missing = [field in val for field in required_fields] + missing = [field for field in required_fields if field not in val] if missing: raise ValueError( f"Missing required fields {missing} in Labeler definition: {val}") @@ -71,14 +71,26 @@ def from_dict(cls, label_name: str, val: dict, prefix: str="") -> Self: sels_defs = val.get("selectors", {}) for sname, sbody in sels_defs.items(): try: - scls = selectors.get_selector_cls(sname, raise_if_missing=True) - sels.append(scls.from_dict(sbody)) + # HACK(aznashwan): disable raising for testing. + scls = selectors.get_selector_cls(sname, raise_if_missing=False) + if scls: + sels.append(scls.from_dict(sbody)) except Exception as ex: raise ValueError( - f"Failed to load selector '{sname}' from payload {sbody}") from ex + f"Failed to load selector '{sname}' from " + f"payload {sbody}:\n{ex}") from ex return cls(label_name, val['label-color'], sels, prefix=prefix) + def get_labels_for_repo(self, repo: Repository) -> list[Label]: + raise NotImplemented + + def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: + raise NotImplemented + + def get_labels_for_issue(self, issue: Issue) -> list[Label]: + raise NotImplemented + class PrefixLabeler(BaseLabeler): """ Class for handling label prefix groups. @@ -105,17 +117,18 @@ def get_labels_for_issue(self, issue: Issue) -> list[Label]: def load_labelers_from_config(config: dict) -> list[BaseLabeler]: - toplevel_lebelers = [] + toplevel_labelers = [] for key, val in config.items(): + LOG.info(f"Attempting to load labeler from: {config}") # Assume it's a plain labeler until proven otherwise. try: - toplevel_lebelers.append(SelectorLabeler.from_dict(key, val)) + toplevel_labelers.append(SelectorLabeler.from_dict(key, val)) continue - except ValueError as err: + except (ValueError, TypeError) as err: LOG.info( - f"Failed to load lebeler on key {key}, assuming it's a prefix: {err}") + f"Failed to load labeler on key {key}, assuming it's a prefix: {err}") prefixer = PrefixLabeler(key, load_labelers_from_config(val)) - toplevel_lebelers.append(prefixer) + toplevel_labelers.append(prefixer) - return toplevel_lebelers + return toplevel_labelers diff --git a/autolabeler/main.py b/autolabeler/main.py index 29b2396..bb39ec9 100644 --- a/autolabeler/main.py +++ b/autolabeler/main.py @@ -13,34 +13,17 @@ # under the License. import argparse -import logging +from io import IOBase import os import github +import yaml from github import Auth -from autolabeler import targets +from autolabeler import labelers +from autolabeler import utils - -def _setup_logging(): - # create logger - logger = logging.getLogger('github-autolabeler') - logger.setLevel(logging.DEBUG) - - # create console handler and set level to debug - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - # create formatter - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - # add formatter to ch - ch.setFormatter(formatter) - - # add ch to logger - logger.addHandler(ch) - - return logger +LOG = utils.getStdoutLogger(__name__) def _add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: @@ -130,9 +113,14 @@ def _safe_index(n: int): } -def main(): - LOG = _setup_logging() +def load_yaml_file(path_or_file: str|IOBase) -> dict: + file = path_or_file + if isinstance(path_or_file, str): + file = open(path_or_file, 'r') + return yaml.safe_load(file) + +def main(): parser = argparse.ArgumentParser( "github-autolabeler", description="Python 3 utility for automatically labelling/triaging " @@ -147,16 +135,16 @@ def main(): f"No GitHub API token provided via '-t/--github-token' argument " "or 'GITHUB_TOKEN' environment variable.") - # TODO(aznashwan): - # - parse config files - # - load rules - # - check labels generated + # gh = github.Github(login_or_token=args.github_token) + # _ = gh.get_user().login + # gh.load(f) + gh = github.Github() LOG.info(f"{_parse_target_argument(args.target)}") - gh = github.Github(login_or_token=args.github_token) - # NOTE(aznashwan): required to load user: - _ = gh.get_user().login - gh.load(f) + lblrs = [] + if args.label_definitions_file: + rules_config = load_yaml_file(args.label_definitions_file) + lblrs = labelers.load_labelers_from_config(rules_config) - _ = targets.RepoLabelsTarget(gh, "aznashwan", "cloudbase-init") + LOG.info(f"{lblrs}") diff --git a/autolabeler/selectors.py b/autolabeler/selectors.py index 332ff24..38a5222 100644 --- a/autolabeler/selectors.py +++ b/autolabeler/selectors.py @@ -13,19 +13,21 @@ # under the License. import abc -import logging import re from typing import Self +import typing from github.Issue import Issue from github.PullRequest import PullRequest from github.Repository import Repository +from autolabeler import utils -LOG = logging.getLogger() +LOG = utils.getStdoutLogger(__name__) -class Selector(withmetaclass=abc.ABCMeta): + +class Selector(metaclass=abc.ABCMeta): @abc.abstractclassmethod def from_dict(cls, val: dict): @@ -166,16 +168,22 @@ def match(self, obj: Repository|PullRequest) -> dict: SELECTORS_NAME_MAP = { "regex": RegexSelector, - "files": FilesSelector + "files": FilesSelector, } -def get_selector_cls(selector_name, raise_if_missing=True): +def get_selector_cls(selector_name: str, raise_if_missing: bool=True) -> typing.Type: selector = SELECTORS_NAME_MAP.get(selector_name) - if raise_if_missing and not selector: - raise ValueError( - f"Unknown selector type {selector_name}. Supported selectors are: " + + if not selector: + msg = ( + f"Unknown selector type '{selector_name}'. Supported selectors are: " f"{list(SELECTORS_NAME_MAP.keys())}") + if raise_if_missing: + raise ValueError(msg) + else: + LOG.warn(msg) + return selector diff --git a/autolabeler/utils.py b/autolabeler/utils.py new file mode 100644 index 0000000..d3ab489 --- /dev/null +++ b/autolabeler/utils.py @@ -0,0 +1,46 @@ +# Copyright 2023 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + + +__LOGGER = None + + +def getStdoutLogger(name="", level=logging.DEBUG): + global __LOGGER + if __LOGGER: + return __LOGGER + + # create logger + if not name: + name = "github-autolabeler" + logger = logging.getLogger(name) + logger.setLevel(level) + + # create console handler and set level to debug + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + + __LOGGER = logger + return logger diff --git a/samples/autolabels-showcase.yaml b/samples/autolabels-showcase.yaml index 74a27fa..1e9cc38 100644 --- a/samples/autolabels-showcase.yaml +++ b/samples/autolabels-showcase.yaml @@ -1,23 +1,23 @@ # This defines a static static label named 'example-label' with the given props. example-label: - color: 0075ca # blue - description: This is a simple example static label. + label-color: 0075ca # blue + label-description: This is a simple example static label. # This defines two "namespaced" labels named: label-prefix/sublabel-{1,2}. label-prefix: sublabel-1: - color: 0075ca - description: This is a simple first sublabel. + label-color: 0075ca + label-description: This is a simple first sublabel. sublabel-2: - color: d73a4a # red - description: This is a simple second sublabel. + label-color: d73a4a # red + label-description: This is a simple second sublabel. # This will define as many labels as there are filepath regex matches. label-for-sample-file-{files-name-regex-group-1}: - color: 0075ca - description: This label was created for file {files-name-regex-group-1}. + label-color: 0075ca + label-description: This label was created for file {files-name-regex-group-1}. selectors: files: # This will specify a regex to be matched against full filepaths in @@ -32,8 +32,8 @@ label-for-sample-file-{files-name-regex-group-1}: # This will define a label named "bug" which will automatically be # associated with issues/PRs matching the provided regexes. bug: - color: d73a4a - description: This label signifies the Issue/PR is about a bug. + label-color: d73a4a + label-description: This label signifies the Issue/PR is about a bug. selectors: regex: # This allows for case insensitive searches. (default is false) @@ -55,8 +55,8 @@ bug: # automatically get applied to Pull Requsts whose change counts match. pr-size-measure: small: - color: 0075ca - description: | + label-color: 0075ca + label-description: | This PR is between {lines-changed-min} and {lines-changed-max} lines of code. selectors: lines-changed: @@ -64,8 +64,8 @@ pr-size-measure: max: 1000 large: - color: d73a4a - description: | + label-color: d73a4a + label-description: | This PR is between {lines-changed-min} and {lines-changed-max} lines of code. selectors: lines-changed: From bdd38324121fb2855fcf59db272d2b2402793780 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Wed, 26 Jul 2023 16:28:50 +0300 Subject: [PATCH 6/8] Add labeler selector aggregation logic. Signed-off-by: Nashwan Azhari --- autolabeler/labelers.py | 122 ++++++++++++++++++++++++++----- autolabeler/selectors.py | 24 +++--- autolabeler/targets.py | 2 + samples/autolabels-showcase.yaml | 2 +- 4 files changed, 118 insertions(+), 32 deletions(-) diff --git a/autolabeler/labelers.py b/autolabeler/labelers.py index a931250..834cc73 100644 --- a/autolabeler/labelers.py +++ b/autolabeler/labelers.py @@ -13,7 +13,9 @@ # under the License. import abc -from typing import Self +import dataclasses +import itertools +from typing import Self, Union from github.Issue import Issue from github.Label import Label @@ -27,18 +29,44 @@ LOG = utils.getStdoutLogger(__name__) +LabellableObject = Union[Repository, PullRequest, Issue] + + +@dataclasses.dataclass +class LabelParams: + name: str + color: str + description: str + + @classmethod + def from_label(cls, label: Label) -> Self: + return cls(label.name, label.color, str(label.description)) + + @classmethod + def from_dict(cls, val: dict[str, str]) -> Self: + return cls(val.get("name", ""), + val.get("color", ""), + val.get("description", "")) + + def __eq__(self, other: Self) -> bool: + return self.name == other.name and \ + self.color == other.color and \ + self.description == other.description + + class BaseLabeler(metaclass=abc.ABCMeta): @abc.abstractmethod - def get_labels_for_repo(self, repo: Repository) -> list[Label]: + def get_labels_for_repo(self, repo: Repository) -> list[LabelParams]: + _ = repo raise NotImplemented @abc.abstractmethod - def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: + def get_labels_for_pr(self, pr: PullRequest) -> list[LabelParams]: raise NotImplemented @abc.abstractmethod - def get_labels_for_issue(self, issue: Issue) -> list[Label]: + def get_labels_for_issue(self, issue: Issue) -> list[LabelParams]: raise NotImplemented @@ -46,10 +74,12 @@ class SelectorLabeler(BaseLabeler): def __init__(self, label_name: str, label_color: str, + label_description: str, selectors: list[selectors.Selector], prefix: str=""): self._name = label_name self._color = label_color + self._description = label_description self._selectors = selectors self._prefix = prefix @@ -80,16 +110,62 @@ def from_dict(cls, label_name: str, val: dict, prefix: str="") -> Self: f"Failed to load selector '{sname}' from " f"payload {sbody}:\n{ex}") from ex - return cls(label_name, val['label-color'], sels, prefix=prefix) - - def get_labels_for_repo(self, repo: Repository) -> list[Label]: - raise NotImplemented - - def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: - raise NotImplemented - - def get_labels_for_issue(self, issue: Issue) -> list[Label]: - raise NotImplemented + return cls( + label_name, + val['label-color'], val['label-description'], + sels, prefix=prefix) + + def _run_selectors(self, obj: LabellableObject) -> list[dict]: + # overly-drawn-out code for logging purposes: + selector_matches = [] + for selector in self._selectors: + res = selector.match(obj) + selector_matches.append(res) + LOG.debug(f"{selector}.match({obj}) = {res}") + return selector_matches + + def _get_labels_for_selector_matches( + self, selector_matches: list[dict]) -> list[LabelParams]: + if self._selectors and not selector_matches: + # NOTE(aznashwan): if no selector fired, no labels should be returned. + LOG.debug(f"{self} matched no selector") + return [] + + label_defs = {} + for match in selector_matches: + name = self._name.format(**match) + new = LabelParams( + name, self._color, + self._description.format(**match)) + if name not in label_defs: + label_defs[name] = new + elif label_defs[name] != new: + LOG.warning( + f"{self} got conflicting colors/descriptions for label " + f"{name}: value already present ({label_defs[name]}) " + f"different from new value: {new}") + LOG.debug(f"{self} determined following labels: {label_defs}") + + return list(label_defs.values()) + + def get_labels_for_repo(self, repo: Repository) -> list[LabelParams]: + return self._get_labels_for_selector_matches( + self._run_selectors(repo)) + + def _get_nonstatic_labels(self, obj: PullRequest|Issue): + # NOTE(aznashwan): this prevents static labellers with no selectors + # being from applied to all PRs/Issues. + if not self._selectors: + return [] + + return self._get_labels_for_selector_matches( + self._run_selectors(obj)) + + def get_labels_for_pr(self, pr: PullRequest) -> list[LabelParams]: + return self._get_nonstatic_labels(pr) + + def get_labels_for_issue(self, issue: Issue) -> list[LabelParams]: + return self._get_nonstatic_labels(issue) class PrefixLabeler(BaseLabeler): @@ -101,18 +177,24 @@ class PrefixLabeler(BaseLabeler): def __init__(self, prefix: str, sublabelers: list[BaseLabeler], separator='/'): + if not sublabelers: + raise ValueError( + f"{self} expects at least one sub-labeler. Got: {sublabelers}") self._prefix = prefix self._separator = separator self._sublabelers = sublabelers - def get_labels_for_repo(self, repo: Repository) -> list[Label]: - raise NotImplemented + def get_labels_for_repo(self, repo: Repository) -> list[LabelParams]: + res = [slblr.get_labels_for_repo(repo) for slblr in self._sublabelers] + return list(itertools.chain(*res)) - def get_labels_for_pr(self, pr: PullRequest) -> list[Label]: - raise NotImplemented + def get_labels_for_pr(self, pr: PullRequest) -> list[LabelParams]: + res = [slblr.get_labels_for_pr(pr) for slblr in self._sublabelers] + return list(itertools.chain(*res)) - def get_labels_for_issue(self, issue: Issue) -> list[Label]: - raise NotImplemented + def get_labels_for_issue(self, issue: Issue) -> list[LabelParams]: + res = [slblr.get_labels_for_issue(issue) for slblr in self._sublabelers] + return list(itertools.chain(*res)) diff --git a/autolabeler/selectors.py b/autolabeler/selectors.py index 38a5222..3061c1a 100644 --- a/autolabeler/selectors.py +++ b/autolabeler/selectors.py @@ -35,7 +35,7 @@ def from_dict(cls, val: dict): raise NotImplemented @abc.abstractmethod - def match(self, obj: object) -> dict: + def match(self, obj: object) -> list[dict]: """ Returns a dict of str key-values if the Github object is matched. Returned keys should follow a predictable namespacing model based on the @@ -71,7 +71,7 @@ def from_dict(cls, val: dict) -> Self: maintainer-comments-only: bool, """ supported_keys = [ - "case_insensitive", "title", "description", + "case-insensitive", "title", "description", "comments", "maintainer-comments", "maintainer-comments-only"] if not any(k in val for k in supported_keys): raise ValueError( @@ -79,7 +79,7 @@ def from_dict(cls, val: dict) -> Self: f"({supported_keys}). Got {val}") kwargs = { - "case_insensitive": val.get("case_insensitive", False), + "case_insensitive": val.get("case-insensitive", False), "re_title": val.get("title", ""), "re_description": val.get("description", ""), "re_comments": val.get("comments", ""), @@ -115,16 +115,16 @@ def _get_comment_matches(self, obj: Issue|PullRequest) -> dict: # pr.get_issue_comments/get_comments/get_review_comments() and match return {} - def match(self, obj: object) -> dict: + def match(self, obj: object) -> list[dict]: # NOTE(aznashwan): Only Issues/PRs have the concept of comments. if not isinstance(obj, (Issue, PullRequest)): LOG.warn( f"RegexSelector got unsupported object type {type(obj)}: {obj}") - return {} + return [] - res = {} - res.update(self._get_comment_matches(obj)) - res.update(self._get_title_description_matches(obj)) + res = [] + res.append(self._get_comment_matches(obj)) + res.append(self._get_title_description_matches(obj)) return res @@ -150,14 +150,14 @@ def from_dict(cls, val: dict) -> Self: return cls(**kwargs) - def match(self, obj: Repository|PullRequest) -> dict: + def match(self, obj: Repository|PullRequest) -> list[dict]: # NOTE(aznashwan): Only Repositories/PRs have associated files. if not isinstance(obj, (Repository, PullRequest)): LOG.warn( f"FileSelector got unsupported object type {type(obj)}: {obj}") - return {} + return [] - res = {} + res = [] # TODO: # - recusively gather all files and directories. @@ -200,4 +200,6 @@ def _get_match_groups( for i, group in enumerate(match.groups()): res[f"{prefix}group-{i}"] = group + LOG.debug(f"Matche result for {regex=} to {value=}: {res}") + return res diff --git a/autolabeler/targets.py b/autolabeler/targets.py index ca3b400..98dc910 100644 --- a/autolabeler/targets.py +++ b/autolabeler/targets.py @@ -35,10 +35,12 @@ def get_labels(self) -> list[Label]: @abc.abstractmethod def set_labels(self, labels: list[Label]): + _ = labels raise NotImplemented @abc.abstractmethod def remove_labels(self, labels: list[str]): + _ = labels raise NotImplemented diff --git a/samples/autolabels-showcase.yaml b/samples/autolabels-showcase.yaml index 1e9cc38..c2f2a70 100644 --- a/samples/autolabels-showcase.yaml +++ b/samples/autolabels-showcase.yaml @@ -70,5 +70,5 @@ pr-size-measure: selectors: lines-changed: min: 1000 - # NOTE: setting to zero will remove any upper bound. + # NOTE: setting max to zero will remove any upper bound. max: 0 From 771b776a02653962914366772feeaf2edf0df4f3 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Thu, 27 Jul 2023 12:40:44 +0300 Subject: [PATCH 7/8] Add file listing. Signed-off-by: Nashwan Azhari --- autolabeler/labelers.py | 46 ++++++++++++++++++++++++++-------------- autolabeler/main.py | 11 ++++++---- autolabeler/selectors.py | 31 +++++++++++++++++++++------ 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/autolabeler/labelers.py b/autolabeler/labelers.py index 834cc73..b31fdf7 100644 --- a/autolabeler/labelers.py +++ b/autolabeler/labelers.py @@ -126,25 +126,31 @@ def _run_selectors(self, obj: LabellableObject) -> list[dict]: def _get_labels_for_selector_matches( self, selector_matches: list[dict]) -> list[LabelParams]: + if not self._selectors: + # This is a static label and should be returned. + return [LabelParams( + self._name, self._color, self._description)] + if self._selectors and not selector_matches: # NOTE(aznashwan): if no selector fired, no labels should be returned. LOG.debug(f"{self} matched no selector") return [] label_defs = {} - for match in selector_matches: - name = self._name.format(**match) - new = LabelParams( - name, self._color, - self._description.format(**match)) - if name not in label_defs: - label_defs[name] = new - elif label_defs[name] != new: - LOG.warning( - f"{self} got conflicting colors/descriptions for label " - f"{name}: value already present ({label_defs[name]}) " - f"different from new value: {new}") - LOG.debug(f"{self} determined following labels: {label_defs}") + for match_set in selector_matches: + for match in match_set: + name = self._name.format(**match) + new = LabelParams( + name, self._color, + self._description.format(**match)) + if name not in label_defs: + label_defs[name] = new + elif label_defs[name] != new: + LOG.warning( + f"{self} got conflicting colors/descriptions for label " + f"{name}: value already present ({label_defs[name]}) " + f"different from new value: {new}") + LOG.debug(f"{self} determined following labels: {label_defs}") return list(label_defs.values()) @@ -153,6 +159,7 @@ def get_labels_for_repo(self, repo: Repository) -> list[LabelParams]: self._run_selectors(repo)) def _get_nonstatic_labels(self, obj: PullRequest|Issue): + # TODO(aznashwan): separate `StaticLabeler` class. # NOTE(aznashwan): this prevents static labellers with no selectors # being from applied to all PRs/Issues. if not self._selectors: @@ -184,17 +191,24 @@ def __init__(self, prefix: str, sublabelers: list[BaseLabeler], self._separator = separator self._sublabelers = sublabelers + def _prefix_label(self, label: LabelParams) -> LabelParams: + label.name = f"{self._prefix}{self._separator}{label.name}" + return label + + def _prefix_labels(self, labels: list[LabelParams]) -> list[LabelParams]: + return list(map(self._prefix_label, labels)) + def get_labels_for_repo(self, repo: Repository) -> list[LabelParams]: res = [slblr.get_labels_for_repo(repo) for slblr in self._sublabelers] - return list(itertools.chain(*res)) + return self._prefix_labels(list(itertools.chain(*res))) def get_labels_for_pr(self, pr: PullRequest) -> list[LabelParams]: res = [slblr.get_labels_for_pr(pr) for slblr in self._sublabelers] - return list(itertools.chain(*res)) + return self._prefix_labels(list(itertools.chain(*res))) def get_labels_for_issue(self, issue: Issue) -> list[LabelParams]: res = [slblr.get_labels_for_issue(issue) for slblr in self._sublabelers] - return list(itertools.chain(*res)) + return self._prefix_labels(list(itertools.chain(*res))) diff --git a/autolabeler/main.py b/autolabeler/main.py index bb39ec9..59a4a59 100644 --- a/autolabeler/main.py +++ b/autolabeler/main.py @@ -135,10 +135,8 @@ def main(): f"No GitHub API token provided via '-t/--github-token' argument " "or 'GITHUB_TOKEN' environment variable.") - # gh = github.Github(login_or_token=args.github_token) - # _ = gh.get_user().login - # gh.load(f) - gh = github.Github() + gh = github.Github(login_or_token=args.github_token) + _ = gh.get_user().login LOG.info(f"{_parse_target_argument(args.target)}") @@ -147,4 +145,9 @@ def main(): rules_config = load_yaml_file(args.label_definitions_file) lblrs = labelers.load_labelers_from_config(rules_config) + repo = gh.get_repo("aznashwan/github-autolabeler") + + for labeler in lblrs: + print(f"\n{labeler}=> {labeler.get_labels_for_repo(repo)}\n") + LOG.info(f"{lblrs}") diff --git a/autolabeler/selectors.py b/autolabeler/selectors.py index 3061c1a..91b71ef 100644 --- a/autolabeler/selectors.py +++ b/autolabeler/selectors.py @@ -17,6 +17,7 @@ from typing import Self import typing +from github.ContentFile import ContentFile from github.Issue import Issue from github.PullRequest import PullRequest from github.Repository import Repository @@ -26,6 +27,8 @@ LOG = utils.getStdoutLogger(__name__) +FileContainingObject = typing.Union[Repository, PullRequest] + class Selector(metaclass=abc.ABCMeta): @@ -77,7 +80,7 @@ def from_dict(cls, val: dict) -> Self: raise ValueError( f"FileRegexSelector requires at least one regex key " f"({supported_keys}). Got {val}") - + kwargs = { "case_insensitive": val.get("case-insensitive", False), "re_title": val.get("title", ""), @@ -143,7 +146,7 @@ def from_dict(cls, val: dict) -> Self: raise ValueError( f"FilesSelector requires at least one options key " f"({supported_keys}). Got {val}") - + kwargs = { "file_name_re": val.get("name-regex", ""), "file_type": val.get("type", "")} @@ -157,11 +160,12 @@ def match(self, obj: Repository|PullRequest) -> list[dict]: f"FileSelector got unsupported object type {type(obj)}: {obj}") return [] + all_files = _list_files_recursively(obj, path="") res = [] - - # TODO: - # - recusively gather all files and directories. - # - match them all + for file in all_files: + match = _get_match_groups(self._file_name_re, file.path) + if match: + res.append(match) return res @@ -187,6 +191,21 @@ def get_selector_cls(selector_name: str, raise_if_missing: bool=True) -> typing. return selector +def _list_files_for_object(obj: Repository|PullRequest, path: str="") -> list[ContentFile]: + return list(obj.get_contents(path)) # pyright: ignore + + +def _list_files_recursively(obj: Repository|PullRequest, path="") -> list[ContentFile]: + files = [] + for item in _list_files_for_object(obj, path=path): + if item.type == "dir": + files.extend(_list_files_recursively(obj, path=item.path)) + else: + files.append(item) + + return files + + def _get_match_groups( regex: str, value: str, prefix="", case_insensitive=False) -> dict: """ Returns dict with all match groups and keys of the form 'prefix-group-N'""" From b8659cb42fe4d9504dbc4c614e61f61b068de532 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Thu, 27 Jul 2023 18:36:09 +0300 Subject: [PATCH 8/8] Update autolabels-showcase.yaml --- samples/autolabels-showcase.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/autolabels-showcase.yaml b/samples/autolabels-showcase.yaml index c2f2a70..c7544e1 100644 --- a/samples/autolabels-showcase.yaml +++ b/samples/autolabels-showcase.yaml @@ -3,6 +3,7 @@ example-label: label-color: 0075ca # blue label-description: This is a simple example static label. +# Test to see if labels get applied in PR. # This defines two "namespaced" labels named: label-prefix/sublabel-{1,2}. label-prefix: