diff --git a/src/bpp/admin/filters.py b/src/bpp/admin/filters.py index 0964b0cec..058845a5b 100644 --- a/src/bpp/admin/filters.py +++ b/src/bpp/admin/filters.py @@ -5,10 +5,10 @@ from django.contrib.contenttypes.models import ContentType from django.db.models import Count, F, IntegerField, Max, Q from django.db.models.functions import Cast +from pbn_client.const import ACTIVE, DELETED from bpp.models import BppUser, Uczelnia, Wydawnictwo_Zwarte from bpp.models.struktura import Jednostka -from pbn_api.const import ACTIVE, DELETED class SimpleIntegerFilter(SimpleListFilter): diff --git a/src/bpp/newsfragments/pbn-pobierz-mongodb-on-error.feature.rst b/src/bpp/newsfragments/pbn-pobierz-mongodb-on-error.feature.rst new file mode 100644 index 000000000..c8b79ed26 --- /dev/null +++ b/src/bpp/newsfragments/pbn-pobierz-mongodb-on-error.feature.rst @@ -0,0 +1,7 @@ +``pobierz_mongodb`` przyjmuje teraz parametr ``on_error``: ``"raise"`` (domyślne, +dotychczasowe fail-fast — błąd zapisu jednego rekordu przerywa cały import) lub +``"skip"`` (deleguje do pakietowego ``download_to_model`` — zły rekord jest +logowany i liczony, a import reszty listy kończy się; zwraca +``DownloadResult(processed, errored)``). Przydatne przy masowych synchronizacjach, +gdzie pojedynczy zepsuty rekord nie powinien przerywać całego przebiegu. +Zachowanie domyślne bez zmian. diff --git a/src/bpp/newsfragments/pbn-slim-shims.doc.rst b/src/bpp/newsfragments/pbn-slim-shims.doc.rst new file mode 100644 index 000000000..420528ddb --- /dev/null +++ b/src/bpp/newsfragments/pbn-slim-shims.doc.rst @@ -0,0 +1,6 @@ +Usunięto trzy zbędne shimy zgodności (``pbn_api.const``, ``pbn_api.utils``, +``pbn_api.client.transport``) — czyste re-eksporty z pakietów PBN. Kod importuje +teraz stałe, helpery słownikowe i transport wprost z ``pbn_client`` (spójnie z +resztą kodu po ekstrakcji). Warstwa zgodności ``pbn_api.exceptions`` oraz baza +modeli ``pbn_api.models.base`` pozostają (mają zawartość BPP-specific / stanowią +punkt rozszerzeń). diff --git a/src/bpp/tests/test_autocomplete/test_autocomplete_publications.py b/src/bpp/tests/test_autocomplete/test_autocomplete_publications.py index 58acda2cc..a89f92f90 100644 --- a/src/bpp/tests/test_autocomplete/test_autocomplete_publications.py +++ b/src/bpp/tests/test_autocomplete/test_autocomplete_publications.py @@ -10,6 +10,7 @@ import pytest from model_bakery import baker +from pbn_client.const import PBN_GET_JOURNAL_BY_ID from bpp.const import PBN_UID_LEN from bpp.models.konferencja import Konferencja @@ -29,7 +30,6 @@ pbn_publisher_json, ) from pbn_api.client import PBN_GET_PUBLICATION_BY_ID_URL, PBN_SEARCH_PUBLICATIONS_URL -from pbn_api.const import PBN_GET_JOURNAL_BY_ID from pbn_api.models import Journal, Publication, Publisher diff --git a/src/importer_publikacji/providers/pbn.py b/src/importer_publikacji/providers/pbn.py index 0a0094120..1cc12acb1 100644 --- a/src/importer_publikacji/providers/pbn.py +++ b/src/importer_publikacji/providers/pbn.py @@ -41,9 +41,9 @@ def _get_pbn_client(uczelnia): """ from django.conf import settings from pbn_client.conf import settings as pbn_defaults + from pbn_client.transport import RequestsTransport from pbn_api.client import PBNClient - from pbn_api.client.transport import RequestsTransport from pbn_api.reporting import rollbar_reporter if not uczelnia or not all( diff --git a/src/pbn_api/client/publication_sync.py b/src/pbn_api/client/publication_sync.py index 741def759..cefde7e41 100644 --- a/src/pbn_api/client/publication_sync.py +++ b/src/pbn_api/client/publication_sync.py @@ -9,15 +9,16 @@ from django.core.mail import mail_admins from django.db import transaction from django.db.models import Model +from pbn_client.const import ( + PBN_POST_PUBLICATION_NO_STATEMENTS_URL, + PBN_POST_PUBLICATIONS_URL, +) +from pbn_client.statements import StatementsMixin from pbn_api.adapters.wydawnictwo import ( OplataZaWydawnictwoPBNAdapter, WydawnictwoPBNAdapter, ) -from pbn_api.const import ( - PBN_POST_PUBLICATION_NO_STATEMENTS_URL, - PBN_POST_PUBLICATIONS_URL, -) from pbn_api.exceptions import ( CannotDeleteStatementsException, CannotUploadPublicationFee, @@ -35,7 +36,6 @@ ) from pbn_api.models.pbn_odpowiedzi_niepozadane import PBNOdpowiedziNiepozadane from pbn_api.models.sentdata import SentData -from pbn_client.statements import StatementsMixin logger = logging.getLogger(__name__) diff --git a/src/pbn_api/client/transport.py b/src/pbn_api/client/transport.py deleted file mode 100644 index 05f18b2ad..000000000 --- a/src/pbn_api/client/transport.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Shim kompatybilnościowy — transport przeniesiony do ``pbn_client.transport``. - -Patrz: docs/superpowers/specs/2026-06-02-pbn-client-split-design.md -""" - -from pbn_client.transport import * # noqa: F401,F403 diff --git a/src/pbn_api/const.py b/src/pbn_api/const.py deleted file mode 100644 index 0fc1849cc..000000000 --- a/src/pbn_api/const.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Shim kompatybilnościowy — stałe przeniesione do ``pbn_client.const``. - -Patrz: docs/superpowers/specs/2026-06-02-pbn-client-split-design.md -""" - -from pbn_client.const import * # noqa: F401,F403 diff --git a/src/pbn_api/management/commands/pbn_test_wysylka_interaktywna.py b/src/pbn_api/management/commands/pbn_test_wysylka_interaktywna.py index 655bdb6c4..7cef76aed 100644 --- a/src/pbn_api/management/commands/pbn_test_wysylka_interaktywna.py +++ b/src/pbn_api/management/commands/pbn_test_wysylka_interaktywna.py @@ -25,17 +25,17 @@ from django.core.management.base import CommandError from pbn_client import decode_publication_object_id - -from bpp.models import Wydawnictwo_Ciagle, Wydawnictwo_Zwarte -from bpp.util import zaloguj_polkniety_wyjatek -from pbn_api.adapters.wydawnictwo import WydawnictwoPBNAdapter -from pbn_api.const import ( +from pbn_client.const import ( PBN_DELETE_PUBLICATION_STATEMENT, PBN_GET_INSTITUTION_STATEMENTS, PBN_POST_INSTITUTION_STATEMENTS_URL, PBN_POST_PUBLICATION_NO_STATEMENTS_URL, PBN_POST_PUBLICATIONS_URL, ) + +from bpp.models import Wydawnictwo_Ciagle, Wydawnictwo_Zwarte +from bpp.util import zaloguj_polkniety_wyjatek +from pbn_api.adapters.wydawnictwo import WydawnictwoPBNAdapter from pbn_api.exceptions import ( AccessDeniedException, DaneLokalneWymagajaAktualizacjiException, diff --git a/src/pbn_api/models/sentdata.py b/src/pbn_api/models/sentdata.py index 724f00f70..1c706eed0 100644 --- a/src/pbn_api/models/sentdata.py +++ b/src/pbn_api/models/sentdata.py @@ -4,10 +4,10 @@ from django.db import models from django.db.models import JSONField from django.utils import timezone +from pbn_client.dict_utils import compare_dicts from bpp import const from bpp.models import LinkDoPBNMixin -from pbn_api.utils import compare_dicts class SentDataManager(models.Manager): diff --git a/src/pbn_api/tests/test_bpp_admin_helpers.py b/src/pbn_api/tests/test_bpp_admin_helpers.py index 41a68a7f8..4cdc648b6 100644 --- a/src/pbn_api/tests/test_bpp_admin_helpers.py +++ b/src/pbn_api/tests/test_bpp_admin_helpers.py @@ -1,6 +1,12 @@ import pytest from django.contrib.messages import get_messages from model_bakery import baker +from pbn_client.const import ( + PBN_GET_INSTITUTION_PUBLICATIONS_V2, + PBN_POST_INSTITUTION_STATEMENTS_URL, + PBN_POST_PUBLICATION_NO_STATEMENTS_URL, + PBN_POST_PUBLICATIONS_URL, +) from bpp.admin.helpers.pbn_api.gui import sprobuj_wyslac_do_pbn_gui from bpp.models import Charakter_Formalny, Wydawnictwo_Ciagle @@ -15,12 +21,6 @@ PBN_GET_INSTITUTION_STATEMENTS, PBN_GET_PUBLICATION_BY_ID_URL, ) -from pbn_api.const import ( - PBN_GET_INSTITUTION_PUBLICATIONS_V2, - PBN_POST_INSTITUTION_STATEMENTS_URL, - PBN_POST_PUBLICATION_NO_STATEMENTS_URL, - PBN_POST_PUBLICATIONS_URL, -) from pbn_api.exceptions import AccessDeniedException, PBNValidationError from pbn_api.models import Publication, SentData from pbn_api.tests.utils import middleware diff --git a/src/pbn_api/tests/test_client_disciplines.py b/src/pbn_api/tests/test_client_disciplines.py index de3292a23..e6d186f53 100644 --- a/src/pbn_api/tests/test_client_disciplines.py +++ b/src/pbn_api/tests/test_client_disciplines.py @@ -9,10 +9,10 @@ from pathlib import Path import pytest +from pbn_client.const import PBN_GET_DISCIPLINES_URL from bpp.decorators import json from bpp.models import Dyscyplina_Naukowa -from pbn_api.const import PBN_GET_DISCIPLINES_URL from pbn_api.models import TlumaczDyscyplin from pbn_api.models.discipline import Discipline diff --git a/src/pbn_api/tests/test_client_extended.py b/src/pbn_api/tests/test_client_extended.py index 9d17cdca0..e59edf516 100644 --- a/src/pbn_api/tests/test_client_extended.py +++ b/src/pbn_api/tests/test_client_extended.py @@ -69,7 +69,7 @@ def test_pbn_client_transport_initialization(): def test_pbn_client_transport_default_base_url(): """Test PBNClientTransport uses default URL when None provided""" - from pbn_api.const import DEFAULT_BASE_URL + from pbn_client.const import DEFAULT_BASE_URL transport = PBNClientTransport("app_id", "app_token", None) diff --git a/src/pbn_api/tests/test_client_helpers.py b/src/pbn_api/tests/test_client_helpers.py index 99dd94d6d..50e2eb3bf 100644 --- a/src/pbn_api/tests/test_client_helpers.py +++ b/src/pbn_api/tests/test_client_helpers.py @@ -9,6 +9,11 @@ import pytest from django.contrib.messages import get_messages from model_bakery import baker +from pbn_client.const import ( + PBN_GET_INSTITUTION_PUBLICATIONS_V2, + PBN_POST_INSTITUTION_STATEMENTS_URL, + PBN_POST_PUBLICATION_NO_STATEMENTS_URL, +) from bpp.admin.helpers.pbn_api.gui import sprobuj_wyslac_do_pbn_gui from fixtures.pbn_api import ( @@ -21,11 +26,6 @@ PBN_GET_INSTITUTION_STATEMENTS, PBN_GET_PUBLICATION_BY_ID_URL, ) -from pbn_api.const import ( - PBN_GET_INSTITUTION_PUBLICATIONS_V2, - PBN_POST_INSTITUTION_STATEMENTS_URL, - PBN_POST_PUBLICATION_NO_STATEMENTS_URL, -) from pbn_api.models import Institution, Publication from pbn_api.tests.utils import middleware diff --git a/src/pbn_api/tests/test_client_sync.py b/src/pbn_api/tests/test_client_sync.py index b123c3ee5..6157f1078 100644 --- a/src/pbn_api/tests/test_client_sync.py +++ b/src/pbn_api/tests/test_client_sync.py @@ -24,6 +24,13 @@ from unittest.mock import MagicMock, patch import pytest +from pbn_client.const import ( + PBN_GET_INSTITUTION_PUBLICATIONS_V2, + PBN_POST_INSTITUTION_STATEMENTS_URL, + PBN_POST_PUBLICATION_NO_STATEMENTS_URL, + PBN_POST_PUBLICATIONS_URL, +) +from pbn_client.exceptions import PBNValidationError from fixtures.pbn_api import ( MOCK_RETURNED_INSTITUTION_PUBLICATION_V2_DATA, @@ -36,19 +43,12 @@ PBN_GET_INSTITUTION_STATEMENTS, PBN_GET_PUBLICATION_BY_ID_URL, ) -from pbn_api.const import ( - PBN_GET_INSTITUTION_PUBLICATIONS_V2, - PBN_POST_INSTITUTION_STATEMENTS_URL, - PBN_POST_PUBLICATION_NO_STATEMENTS_URL, - PBN_POST_PUBLICATIONS_URL, -) from pbn_api.exceptions import ( HttpException, PKZeroExportDisabled, StatementsResendFailedException, ) from pbn_api.models import Publication, SentData -from pbn_client.exceptions import PBNValidationError def _patch_intended_statements(monkeypatch, statements): diff --git a/src/pbn_api/tests/test_client_upload.py b/src/pbn_api/tests/test_client_upload.py index de0d852a2..d1392bf02 100644 --- a/src/pbn_api/tests/test_client_upload.py +++ b/src/pbn_api/tests/test_client_upload.py @@ -8,16 +8,16 @@ import pytest from model_bakery import baker +from pbn_client.const import ( + PBN_POST_PUBLICATION_NO_STATEMENTS_URL, + PBN_POST_PUBLICATIONS_URL, +) from pbn_api.adapters.wydawnictwo import WydawnictwoPBNAdapter from pbn_api.client import ( PBN_DELETE_PUBLICATION_STATEMENT, PBN_GET_INSTITUTION_STATEMENTS, ) -from pbn_api.const import ( - PBN_POST_PUBLICATION_NO_STATEMENTS_URL, - PBN_POST_PUBLICATIONS_URL, -) from pbn_api.exceptions import SameDataUploadedRecently, StatementsMissing from pbn_api.models import Publication, SentData @@ -123,9 +123,9 @@ def test_PBNClient_test_upload_publication_bez_statements_idzie_do_repo( # Body wysłane do /v1/repositorium/publications: lista, BEZ statements, # po konwersji pól (givenNames → firstName). - sent_body = pbn_client.transport.input_values[PBN_POST_PUBLICATION_NO_STATEMENTS_URL][ - "body" - ] + sent_body = pbn_client.transport.input_values[ + PBN_POST_PUBLICATION_NO_STATEMENTS_URL + ]["body"] assert isinstance(sent_body, list) and len(sent_body) == 1 assert "statements" not in sent_body[0] assert "firstName" in sent_body[0]["authors"][0] @@ -162,13 +162,14 @@ def test_PBNClient_post_publication_no_statements( Uczelnia z ``pbn_wysylaj_bez_oswiadczen=True`` pozwala na wysyłkę prac bez oświadczeń (inaczej adapter rzuca StatementsMissing w pbn_get_json). """ - from fixtures.pbn_api import MOCK_RETURNED_MONGODB_DATA - from pbn_api.client import PBN_GET_PUBLICATION_BY_ID_URL - from pbn_api.const import ( + from pbn_client.const import ( PBN_GET_INSTITUTION_PUBLICATIONS_V2, PBN_GET_INSTITUTION_STATEMENTS, ) + from fixtures.pbn_api import MOCK_RETURNED_MONGODB_DATA + from pbn_api.client import PBN_GET_PUBLICATION_BY_ID_URL + uczelnia.pbn_wysylaj_bez_oswiadczen = True uczelnia.save() diff --git a/src/pbn_api/tests/test_discipline_uuid_unique.py b/src/pbn_api/tests/test_discipline_uuid_unique.py index e485258aa..ae1ba0b0c 100644 --- a/src/pbn_api/tests/test_discipline_uuid_unique.py +++ b/src/pbn_api/tests/test_discipline_uuid_unique.py @@ -17,7 +17,7 @@ from bpp.decorators import json from bpp.models import Dyscyplina_Naukowa -from pbn_api.const import PBN_GET_DISCIPLINES_URL +from pbn_client.const import PBN_GET_DISCIPLINES_URL from pbn_api.models import TlumaczDyscyplin from pbn_api.models.discipline import Discipline, DisciplineGroup diff --git a/src/pbn_api/tests/test_pbn_test_wysylka_interaktywna.py b/src/pbn_api/tests/test_pbn_test_wysylka_interaktywna.py index 7c50e38fc..9135ed82e 100644 --- a/src/pbn_api/tests/test_pbn_test_wysylka_interaktywna.py +++ b/src/pbn_api/tests/test_pbn_test_wysylka_interaktywna.py @@ -10,15 +10,15 @@ import pytest from django.core.management import call_command from django.core.management.base import CommandError - -from fixtures.pbn_api import pbn_pageable_json -from pbn_api.const import ( +from pbn_client.const import ( PBN_DELETE_PUBLICATION_STATEMENT, PBN_GET_INSTITUTION_STATEMENTS, PBN_POST_INSTITUTION_STATEMENTS_URL, PBN_POST_PUBLICATION_NO_STATEMENTS_URL, PBN_POST_PUBLICATIONS_URL, ) + +from fixtures.pbn_api import pbn_pageable_json from pbn_api.exceptions import HttpException from pbn_api.management.commands import pbn_test_wysylka_interaktywna as cmd_mod diff --git a/src/pbn_api/tests/test_utils.py b/src/pbn_api/tests/test_utils.py index f19ce3f4c..9ece40f72 100644 --- a/src/pbn_api/tests/test_utils.py +++ b/src/pbn_api/tests/test_utils.py @@ -1,4 +1,4 @@ -from pbn_api.utils import rename_dict_key +from pbn_client.dict_utils import rename_dict_key def test_rename_dict_key_simple(): diff --git a/src/pbn_api/utils.py b/src/pbn_api/utils.py deleted file mode 100644 index e2d815ba0..000000000 --- a/src/pbn_api/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Shim kompatybilnościowy — helpery przeniesione do ``pbn_client.dict_utils``. - -Patrz: docs/superpowers/specs/2026-06-02-pbn-client-split-design.md -""" - -from pbn_client.dict_utils import * # noqa: F401,F403 diff --git a/src/pbn_export_queue/views/detail_views.py b/src/pbn_export_queue/views/detail_views.py index cbaf0411c..e6911b5e4 100644 --- a/src/pbn_export_queue/views/detail_views.py +++ b/src/pbn_export_queue/views/detail_views.py @@ -150,8 +150,9 @@ def _resolve_api_endpoint(self, sent_data): if sent_data.api_url: return sent_data.api_url + from pbn_client.const import PBN_POST_PUBLICATION_NO_STATEMENTS_URL + from bpp.models import Uczelnia - from pbn_api.const import PBN_POST_PUBLICATION_NO_STATEMENTS_URL uczelnia = Uczelnia.objects.get_for_request(self.request) if uczelnia and uczelnia.pbn_api_root: diff --git a/src/pbn_integrator/importer/__init__.py b/src/pbn_integrator/importer/__init__.py index 782758e08..d1cd46ac9 100644 --- a/src/pbn_integrator/importer/__init__.py +++ b/src/pbn_integrator/importer/__init__.py @@ -6,11 +6,11 @@ import logging +from pbn_client.const import DELETED from tqdm import tqdm from bpp.models import Dyscyplina_Naukowa, Jednostka, Rekord, Rodzaj_Zrodla from pbn_api.client import PBNClient -from pbn_api.const import DELETED from pbn_api.models import Publication # Re-export publication import functions diff --git a/src/pbn_integrator/importer/publishers.py b/src/pbn_integrator/importer/publishers.py index 8cfcf8a7c..10d824c3b 100644 --- a/src/pbn_integrator/importer/publishers.py +++ b/src/pbn_integrator/importer/publishers.py @@ -4,11 +4,11 @@ from django.core.management import call_command from django.db import transaction +from pbn_client.const import DELETED from bpp import const from bpp.models import Wydawca from bpp.models.wydawca import Poziom_Wydawcy -from pbn_api.const import DELETED from pbn_api.models import Publisher from pbn_integrator.utils import zapisz_mongodb diff --git a/src/pbn_integrator/tests/test_pobierz_mongodb_on_error.py b/src/pbn_integrator/tests/test_pobierz_mongodb_on_error.py new file mode 100644 index 000000000..b21df3174 --- /dev/null +++ b/src/pbn_integrator/tests/test_pobierz_mongodb_on_error.py @@ -0,0 +1,58 @@ +"""Knob ``on_error`` w ``pobierz_mongodb``: fail-fast (default) vs skip-and-continue. + +Błąd dotyczy zapisu POJEDYNCZEGO rekordu do lokalnego lustra BPP (nie PBN). +``"raise"`` (default) zachowuje historyczne fail-fast; ``"skip"`` deleguje do +pakietowego ``download_to_model`` (log + licznik + kontynuacja). +""" + +from unittest.mock import MagicMock + +import pytest + + +def _fun_failing_on(bad_id): + """Save-fun rzucający na elemencie o danym ``id``, resztę „zapisujący”.""" + saved = [] + + def fun(elem, klass, client=None, **extra): + if elem["id"] == bad_id: + raise ValueError(f"boom {bad_id}") + saved.append(elem["id"]) + + fun.saved = saved + return fun + + +def test_on_error_raise_jest_domyslny_fail_fast(): + """Default = fail-fast: pierwszy błąd przerywa batch, reszta nietknięta.""" + from pbn_integrator.utils.mongodb_ops import pobierz_mongodb + + elems = [{"id": "a"}, {"id": "b"}, {"id": "c"}] + fun = _fun_failing_on("b") + with pytest.raises(ValueError, match="boom b"): + pobierz_mongodb(elems, MagicMock(), fun=fun, disable_progress_bar=True) + # „c” (po zepsutym „b”) NIE zostało zapisane — batch przerwany. + assert fun.saved == ["a"] + + +def test_on_error_skip_kontynuuje_i_liczy_bledy(): + """``skip`` = kontynuuj mimo błędu; zwraca DownloadResult z licznikami.""" + from pbn_integrator.utils.mongodb_ops import pobierz_mongodb + + elems = [{"id": "a"}, {"id": "b"}, {"id": "c"}] + fun = _fun_failing_on("b") + result = pobierz_mongodb( + elems, MagicMock(), fun=fun, disable_progress_bar=True, on_error="skip" + ) + # „a” i „c” zapisane mimo błędu na „b”. + assert fun.saved == ["a", "c"] + assert result.processed == 2 + assert result.errored == 1 + + +def test_on_error_nieznana_wartosc_to_valueerror(): + """Literówka w ``on_error`` fail-uje głośno, nie po cichu wybiera tryb.""" + from pbn_integrator.utils.mongodb_ops import pobierz_mongodb + + with pytest.raises(ValueError, match="on_error"): + pobierz_mongodb([], MagicMock(), fun=lambda *a, **k: None, on_error="whatever") diff --git a/src/pbn_integrator/utils/institutions.py b/src/pbn_integrator/utils/institutions.py index 455ad0688..35caf70e4 100644 --- a/src/pbn_integrator/utils/institutions.py +++ b/src/pbn_integrator/utils/institutions.py @@ -4,8 +4,9 @@ from typing import TYPE_CHECKING +from pbn_client.const import ACTIVE, DELETED + from bpp.models import Jednostka, Uczelnia -from pbn_api.const import ACTIVE, DELETED from pbn_api.models import Institution from pbn_integrator.utils.mongodb_ops import pobierz_mongodb, zapisz_mongodb from pbn_integrator.utils.threaded_page_getter import ( diff --git a/src/pbn_integrator/utils/journals.py b/src/pbn_integrator/utils/journals.py index 2d5a74fad..610fd8980 100644 --- a/src/pbn_integrator/utils/journals.py +++ b/src/pbn_integrator/utils/journals.py @@ -7,10 +7,10 @@ from typing import TYPE_CHECKING from django.db.models import F, Func, IntegerField, Q +from pbn_client.const import ACTIVE, DELETED from bpp.models import Zrodlo from bpp.util import pbar -from pbn_api.const import ACTIVE, DELETED from pbn_api.models import Journal from pbn_integrator.utils.threaded_page_getter import ( ThreadedMongoDBSaver, diff --git a/src/pbn_integrator/utils/mongodb_ops.py b/src/pbn_integrator/utils/mongodb_ops.py index 773bd9142..8d96a99b4 100644 --- a/src/pbn_integrator/utils/mongodb_ops.py +++ b/src/pbn_integrator/utils/mongodb_ops.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING import rollbar +from django_pbn_client.download import download_to_model from django_pbn_client.persistence import ( download_pbn_objects, get_or_download, @@ -293,6 +294,7 @@ def pobierz_mongodb( client=None, disable_progress_bar=False, callback=None, + on_error="raise", ): """Fetch and save elements from PBN API. @@ -304,24 +306,50 @@ def pobierz_mongodb( client: PBN client. disable_progress_bar: Whether to disable progress bar. callback: Optional callback for progress tracking. + on_error: Zachowanie przy błędzie zapisu POJEDYNCZEGO rekordu do + lokalnego lustra BPP (``IntegrityError``, zły kształt danych itp.): + + - ``"raise"`` (default) — fail-fast: pierwszy błąd propaguje i + przerywa cały batch (zachowanie historyczne). Zwraca ``None``. + - ``"skip"`` — skip-and-log-and-continue: zły rekord jest logowany + (pełny traceback) i liczony, import reszty listy kończy się. + Deleguje do pakietowego ``download_to_model`` i zwraca + ``DownloadResult(processed, errored)``. Przydatne przy masowych + synchronizacjach (tysiące rekordów), gdzie jeden zepsuty rekord + nie powinien wywalać całego przebiegu. """ if fun is None: fun = zapisz_mongodb - def progress(elements, total, label): + def progress(elements, total, _label): + # Używamy ``pbar_label`` (nie ``_label`` z delegata) — ``download_to_model`` + # nie forwarduje etykiety, więc trzymamy ją stałą w obu trybach. return pbar( elements, total, - label, + pbar_label, disable_progress_bar=disable_progress_bar, callback=callback, ) - return download_pbn_objects( - elems, - klass, - label=pbar_label, - save=fun, - client=client, - progress=progress, - ) + if on_error == "raise": + return download_pbn_objects( + elems, + klass, + label=pbar_label, + save=fun, + client=client, + progress=progress, + ) + if on_error == "skip": + # ``elems`` jest już zbudowanym paginatorem/iteratorem — fasada oczekuje + # fabryki (zero-arg), więc oddajemy gotowy zasób. Domyślne + # ``concurrency=None`` → ścieżka sekwencyjna (bez ponownego requestu). + return download_to_model( + lambda: elems, + klass, + save=fun, + client=client, + progress=progress, + ) + raise ValueError(f"on_error musi być 'raise' albo 'skip', otrzymano {on_error!r}") diff --git a/src/pbn_integrator/utils/odswiez_tabele_publikacji.py b/src/pbn_integrator/utils/odswiez_tabele_publikacji.py index e5106aaea..6a5ba2080 100644 --- a/src/pbn_integrator/utils/odswiez_tabele_publikacji.py +++ b/src/pbn_integrator/utils/odswiez_tabele_publikacji.py @@ -1,4 +1,5 @@ -from pbn_api.const import DELETED +from pbn_client.const import DELETED + from pbn_api.exceptions import BrakIDPracyPoStroniePBN diff --git a/src/pbn_integrator/utils/offline_data.py b/src/pbn_integrator/utils/offline_data.py index eaf8fea5d..15a88b11a 100644 --- a/src/pbn_integrator/utils/offline_data.py +++ b/src/pbn_integrator/utils/offline_data.py @@ -7,9 +7,9 @@ from typing import TYPE_CHECKING from django.db import transaction +from pbn_client.const import ACTIVE, DELETED from bpp.util import pbar -from pbn_api.const import ACTIVE, DELETED from pbn_api.models import Publication, Scientist from pbn_integrator.utils.mongodb_ops import pobierz_mongodb from pbn_integrator.utils.multiprocessing_utils import ( diff --git a/src/pbn_integrator/utils/pobierz_skasowane_prace.py b/src/pbn_integrator/utils/pobierz_skasowane_prace.py index 7c93a3fed..cdbeca747 100644 --- a/src/pbn_integrator/utils/pobierz_skasowane_prace.py +++ b/src/pbn_integrator/utils/pobierz_skasowane_prace.py @@ -42,7 +42,7 @@ def pobierz_skasowane_prace(client): from multiprocessing import Pool - from pbn_api.const import DELETED + from pbn_client.const import DELETED from bpp.util import pbar diff --git a/src/pbn_integrator/utils/publications.py b/src/pbn_integrator/utils/publications.py index f2b0ba29e..3679d0b50 100644 --- a/src/pbn_integrator/utils/publications.py +++ b/src/pbn_integrator/utils/publications.py @@ -8,12 +8,12 @@ from typing import TYPE_CHECKING import rollbar +from pbn_client.const import ACTIVE from tqdm import tqdm from bpp.const import PBN_MIN_ROK from bpp.models import Rekord from bpp.util import pbar -from pbn_api.const import ACTIVE from pbn_api.exceptions import ( BrakIDPracyPoStroniePBN, HttpException, diff --git a/src/pbn_integrator/utils/scientists.py b/src/pbn_integrator/utils/scientists.py index 48bb49511..c72857265 100644 --- a/src/pbn_integrator/utils/scientists.py +++ b/src/pbn_integrator/utils/scientists.py @@ -11,10 +11,10 @@ import rollbar from django.db import IntegrityError, transaction from django.db.models import Q +from pbn_client.const import DELETED from bpp.models import Autor, Autor_Dyscyplina, Tytul, Uczelnia from bpp.util import pbar -from pbn_api.const import DELETED from pbn_api.models import Scientist from pbn_integrator.utils.constants import CPU_COUNT from pbn_integrator.utils.django_imports import _ensure_django_imports diff --git a/src/przemapuj_zrodla_pbn/views.py b/src/przemapuj_zrodla_pbn/views.py index 2a66dc9dc..e915de75a 100644 --- a/src/przemapuj_zrodla_pbn/views.py +++ b/src/przemapuj_zrodla_pbn/views.py @@ -7,11 +7,11 @@ from django.db import models, transaction from django.db.models import Count, Q from django.shortcuts import get_object_or_404, redirect, render +from pbn_client.const import ACTIVE, DELETED from bpp.models import Rekord, Rodzaj_Zrodla, Uczelnia, Wydawnictwo_Ciagle, Zrodlo from bpp.permissions import wprowadzanie_danych_wymagane from bpp.util import zaloguj_polkniety_wyjatek -from pbn_api.const import ACTIVE, DELETED from pbn_api.models import Journal from pbn_export_queue.models import PBN_Export_Queue