From a54c81497890246ff7bb33f2f6a9f031a6828595 Mon Sep 17 00:00:00 2001 From: Shlok Gilda Date: Thu, 18 Dec 2025 20:46:26 -0500 Subject: [PATCH] Remove DEI badging functionality and related components Signed-off-by: Shlok Gilda --- collectoss/api/routes/__init__.py | 1 - collectoss/api/routes/dei.py | 139 ------------------ collectoss/application/db/models/__init__.py | 3 +- .../application/db/models/operations.py | 13 -- .../versions/39_remove_dei_badging_table.py | 32 ++++ collectoss/templates/dei-badging-report.j2 | 17 --- docker/database/collectoss-new-schema.sql | 73 --------- docs/source/rest-api/spec.yml | 98 +----------- 8 files changed, 34 insertions(+), 342 deletions(-) create mode 100644 collectoss/application/schema/alembic/versions/39_remove_dei_badging_table.py diff --git a/collectoss/api/routes/__init__.py b/collectoss/api/routes/__init__.py index 0fc056f58..3163575bd 100644 --- a/collectoss/api/routes/__init__.py +++ b/collectoss/api/routes/__init__.py @@ -7,6 +7,5 @@ from .manager import * from .nonstandard_metrics import * from .user import * -from .dei import * from .util import * from .complexity import * diff --git a/collectoss/api/routes/dei.py b/collectoss/api/routes/dei.py index 5e18dceb8..e69de29bb 100644 --- a/collectoss/api/routes/dei.py +++ b/collectoss/api/routes/dei.py @@ -1,139 +0,0 @@ -""" -Creates routes for DEI badging functionality -""" - -import logging, subprocess - -from flask import request, jsonify, render_template, send_file, current_app -from pathlib import Path - -from collectoss.api.util import api_key_required, ssl_required - -from collectoss.application.db.models import ClientApplication, CollectionStatus, Repo, RepoGroup, BadgingDEI -from collectoss.application.db.session import DatabaseSession - -from collectoss.tasks.util.collection_util import CollectionRequest,CollectionTaskRoutine, get_enabled_phase_names_from_config_session, core_task_success_util -from collectoss.tasks.start_tasks import prelim_phase, primary_repo_collect_phase -from collectoss.tasks.github.util.util import get_repo_weight_by_issue - -from ..server import app - -logger = logging.getLogger(__name__) - -from collectoss.api.routes import API_VERSION -from collectoss.application.db.models.operations import FRONTEND_REPO_GROUP_NAME - -@app.route(f"/{API_VERSION}/dei/repo/add", methods=['POST']) -@ssl_required -@api_key_required -def dei_track_repo(application: ClientApplication): - dei_id = request.args.get("id") - level = request.args.get("level") - repo_url = request.args.get("url") - - if not (dei_id and level and repo_url): - return jsonify({"status": "Missing argument"}), 400 - - repo_url = repo_url.lower() - - # Use context manager to ensure proper session cleanup - with DatabaseSession(logger, engine=current_app.engine) as session: - repo: Repo = session.query(Repo).filter(Repo.repo_git==repo_url).first() - if repo: - # Making the assumption that only new repos will be added with this endpoint - return jsonify({"status": "Repo already exists"}) - - frontend_repo_group: RepoGroup = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first() - repo_id = Repo.insert_github_repo(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="") - if not repo_id: - return jsonify({"status": "Error adding repo"}) - - repo = Repo.get_by_id(session, repo_id) - repo_git = repo.repo_git - pr_issue_count = get_repo_weight_by_issue(logger, repo_git) - - record = { - "repo_id": repo_id, - "issue_pr_sum": pr_issue_count, - "core_weight": -9223372036854775808, - "secondary_weight": -9223372036854775808, - "ml_weight": -9223372036854775808 - } - - collection_status_unique = ["repo_id"] - session.insert_data(record, CollectionStatus, collection_status_unique, on_conflict_update=False) - - record = { - "badging_id": dei_id, - "level": level, - "repo_id": repo_id - } - - enabled_phase_names = get_enabled_phase_names_from_config_session(session, logger) - - # Primary collection hook. - primary_enabled_phases = [] - - # Primary jobs - if prelim_phase.__name__ in enabled_phase_names: - primary_enabled_phases.append(prelim_phase) - - primary_enabled_phases.append(primary_repo_collect_phase) - - #task success is scheduled no matter what the config says. - def core_task_success_util_gen(repo_git): - return core_task_success_util.si(repo_git) - - primary_enabled_phases.append(core_task_success_util_gen) - - record = BadgingDEI(**record) - session.add(record) - - # Explicitly commit the session to persist BadgingDEI record - session.commit() - - deiHook = CollectionRequest("core",primary_enabled_phases) - deiHook.repo_list = [repo_url] - - singleRoutine = CollectionTaskRoutine(logger, session,[deiHook]) - singleRoutine.start_data_collection() - #start_block_of_repos(logger, session, [repo_url], primary_enabled_phases, "new") - - return jsonify({"status": "Success"}) - -@app.route(f"/{API_VERSION}/dei/report", methods=['POST']) -@ssl_required -@api_key_required -def dei_report(application: ClientApplication): - dei_id = request.args.get("id") - - if not dei_id: - return jsonify({"status": "Missing argument"}), 400 - - # Use context manager but scope it carefully to cover lazy-loading - with DatabaseSession(logger, engine=current_app.engine) as session: - project: BadgingDEI = session.query(BadgingDEI).filter(BadgingDEI.badging_id==dei_id).first() - - if not project: - return jsonify({"status": "Invalid ID"}) - - # Render template while session is still open (accesses project.repo via lazy-loading) - md = render_template("dei-badging-report.j2", project=project) - - # Store project.id before session closes - project_id = project.id - - # Session is now closed - proceed with file operations (no database access needed) - cachePath = Path.cwd() / "collectoss" / "static" / "cache" - - source = cachePath / f"{project_id}_badging_report.md" - report = cachePath / f"{project_id}_badging_report.pdf" - source.write_text(md) - - command = f"mdpdf -o {str(report.resolve())} {str(source.resolve())}" - converter = subprocess.Popen(command.split()) - converter.wait() - - # TODO what goes in the report? - - return send_file(report.resolve()) \ No newline at end of file diff --git a/collectoss/application/db/models/__init__.py b/collectoss/application/db/models/__init__.py index 80d3cf9b4..20c74710b 100644 --- a/collectoss/application/db/models/__init__.py +++ b/collectoss/application/db/models/__init__.py @@ -110,8 +110,7 @@ Subscription, SubscriptionType, RefreshToken, - CollectionStatus, - BadgingDEI + CollectionStatus ) DEFAULT_REPO_GROUP_IDS = [1, 10] diff --git a/collectoss/application/db/models/operations.py b/collectoss/application/db/models/operations.py index 19eff3bae..f9f73f2a5 100644 --- a/collectoss/application/db/models/operations.py +++ b/collectoss/application/db/models/operations.py @@ -219,19 +219,6 @@ class WorkerSettingsFacade(Base): comment="For future use when we move all working tables to the operations schema. ", ) -class BadgingDEI(Base): - __tablename__ = 'dei_badging' - __table_args__ = {"schema": "data"} - id = Column(Integer, primary_key=True, nullable=False) - badging_id = Column(Integer, nullable=False) - level = Column(String, nullable=False) - - repo_id = Column( - ForeignKey("data.repo.repo_id", name="user_repo_user_id_fkey"), primary_key=True, nullable=False - ) - - repo = relationship("Repo") - class Config(Base): __tablename__ = 'config' diff --git a/collectoss/application/schema/alembic/versions/39_remove_dei_badging_table.py b/collectoss/application/schema/alembic/versions/39_remove_dei_badging_table.py new file mode 100644 index 000000000..8941a7b4b --- /dev/null +++ b/collectoss/application/schema/alembic/versions/39_remove_dei_badging_table.py @@ -0,0 +1,32 @@ +"""Remove DEI Badging table + +Revision ID: 39 +Revises: 38 +Create Date: 2025-12-18 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '39' +down_revision = '38' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('dei_badging', schema='augur_data') + + +def downgrade(): + op.create_table('dei_badging', + sa.Column('id', sa.Integer, nullable=False, autoincrement=True), + sa.Column('badging_id', sa.Integer(), nullable=False), + sa.Column('level', sa.String(), nullable=False), + sa.Column('repo_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['repo_id'], ['augur_data.repo.repo_id'], name='user_repo_user_id_fkey'), + sa.PrimaryKeyConstraint('id', 'repo_id'), + schema='augur_data' + ) diff --git a/collectoss/templates/dei-badging-report.j2 b/collectoss/templates/dei-badging-report.j2 index 77ba4f7d6..8b1378917 100644 --- a/collectoss/templates/dei-badging-report.j2 +++ b/collectoss/templates/dei-badging-report.j2 @@ -1,18 +1 @@ -# DEI Badging Report -## Project: {{ project.id }} - -- Level: {{ project.level }} -- Repo: - - {{ project.repo.repo_id }} - - {{ project.repo.repo_git }} - -{% if project.level == "gold" %} - -{% elif project.level == "bronze" %} - -{% else %} - -# ERROR - -{% endif %} \ No newline at end of file diff --git a/docker/database/collectoss-new-schema.sql b/docker/database/collectoss-new-schema.sql index 1163a87df..d649b79cb 100644 --- a/docker/database/collectoss-new-schema.sql +++ b/docker/database/collectoss-new-schema.sql @@ -1488,41 +1488,6 @@ CREATE SEQUENCE augur_data.contributors_history_cntrb_history_id_seq ALTER SEQUENCE augur_data.contributors_history_cntrb_history_id_seq OWNER TO augur; --- --- Name: dei_badging; Type: TABLE; Schema: augur_data; Owner: augur --- - -CREATE TABLE augur_data.dei_badging ( - id integer NOT NULL, - badging_id integer NOT NULL, - level character varying NOT NULL, - repo_id bigint NOT NULL -); - - -ALTER TABLE augur_data.dei_badging OWNER TO augur; - --- --- Name: dei_badging_id_seq; Type: SEQUENCE; Schema: augur_data; Owner: augur --- - -CREATE SEQUENCE augur_data.dei_badging_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -ALTER SEQUENCE augur_data.dei_badging_id_seq OWNER TO augur; - --- --- Name: dei_badging_id_seq; Type: SEQUENCE OWNED BY; Schema: augur_data; Owner: augur --- - -ALTER SEQUENCE augur_data.dei_badging_id_seq OWNED BY augur_data.dei_badging.id; - -- -- Name: discourse_insights_msg_discourse_id_seq1; Type: SEQUENCE; Schema: augur_data; Owner: augur @@ -5741,13 +5706,6 @@ ALTER TABLE spdx.scanners OWNER TO augur; ALTER TABLE ONLY augur_data.chaoss_user ALTER COLUMN chaoss_id SET DEFAULT nextval('augur_data.chaoss_user_chaoss_id_seq'::regclass); --- --- Name: dei_badging id; Type: DEFAULT; Schema: augur_data; Owner: augur --- - -ALTER TABLE ONLY augur_data.dei_badging ALTER COLUMN id SET DEFAULT nextval('augur_data.dei_badging_id_seq'::regclass); - - -- -- Name: config id; Type: DEFAULT; Schema: augur_operations; Owner: augur -- @@ -6458,14 +6416,6 @@ COPY augur_data.contributors_aliases (cntrb_alias_id, canonical_email, alias_ema \. --- --- Data for Name: dei_badging; Type: TABLE DATA; Schema: augur_data; Owner: augur --- - -COPY augur_data.dei_badging (id, badging_id, level, repo_id) FROM stdin; -\. - - -- -- Data for Name: discourse_insights; Type: TABLE DATA; Schema: augur_data; Owner: augur -- @@ -7942,13 +7892,6 @@ SELECT pg_catalog.setval('augur_data.contributors_cntrb_id_seq', 25430, false); SELECT pg_catalog.setval('augur_data.contributors_history_cntrb_history_id_seq', 25430, false); --- --- Name: dei_badging_id_seq; Type: SEQUENCE SET; Schema: augur_data; Owner: augur --- - -SELECT pg_catalog.setval('augur_data.dei_badging_id_seq', 1, false); - - -- -- Name: discourse_insights_msg_discourse_id_seq; Type: SEQUENCE SET; Schema: augur_data; Owner: augur -- @@ -8678,14 +8621,6 @@ ALTER TABLE ONLY augur_data.contributors_aliases ADD CONSTRAINT contributors_aliases_pkey PRIMARY KEY (cntrb_alias_id); --- --- Name: dei_badging dei_badging_pkey; Type: CONSTRAINT; Schema: augur_data; Owner: augur --- - -ALTER TABLE ONLY augur_data.dei_badging - ADD CONSTRAINT dei_badging_pkey PRIMARY KEY (id, repo_id); - - -- -- Name: repo_dependencies deps-insert-unique; Type: CONSTRAINT; Schema: augur_data; Owner: augur -- @@ -11335,14 +11270,6 @@ ALTER TABLE ONLY augur_data.repo_sbom_scans ADD CONSTRAINT repo_linker_sbom FOREIGN KEY (repo_id) REFERENCES augur_data.repo(repo_id) ON UPDATE CASCADE ON DELETE CASCADE; --- --- Name: dei_badging user_repo_user_id_fkey; Type: FK CONSTRAINT; Schema: augur_data; Owner: augur --- - -ALTER TABLE ONLY augur_data.dei_badging - ADD CONSTRAINT user_repo_user_id_fkey FOREIGN KEY (repo_id) REFERENCES augur_data.repo(repo_id); - - -- -- Name: client_applications client_application_user_id_fkey; Type: FK CONSTRAINT; Schema: augur_operations; Owner: augur -- diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index 876ad704a..7cdea7d9b 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -5849,100 +5849,4 @@ paths: type: string enum: [ "Missing argument: refresh_token" ] tags: - - login - - # DEI Endpoints - - /dei/repo/add: - post: - description: Add and start repo for DEI Badging - operationId: DEI Badging Tracking - parameters: - - description: Client [API_Key] - in: header - required: true - name: Authorization - schema: - type: string - type: string - - description: The source badging ID - in: query - required: true - name: id - schema: - type: string - type: string - - description: The badging level - in: query - required: true - name: level - schema: - type: string - type: string - - description: The repo URL to track - in: query - required: true - name: url - schema: - type: string - type: string - responses: - "200": - description: OK - schema: - type: object - properties: - status: - type: string - enum: [ Success, Repo already exists, Error adding repo ] - example: Success - "400": - description: Missing Argument - schema: - type: object - properties: - status: - type: string - enum: [ "Missing argument" ] - tags: - - DEI Badging - /dei/report: - post: - description: Request the report for the given badging project. On success, the - report PDF will be returned in a Binary response. On failure, a JSON - response with a status will be returned. - operationId: DEI Badging Report - parameters: - - description: Client [API_Key] - in: header - required: true - name: Authorization - schema: - type: string - type: string - - description: The source badging ID - in: query - required: true - name: id - schema: - type: string - type: string - responses: - "200": - description: OK - schema: - type: object - properties: - status: - type: string - enum: [ Invalid ID ] - "400": - description: Missing Argument - schema: - type: object - properties: - status: - type: string - enum: [ "Missing argument" ] - tags: - - DEI Badging + - login