[ENG-9046] shtrove cardsearch feeds

TODO.md

@@ -4,13 +4,11 @@ ways to better this mess
 ## better shtrove api experience
 
 - better web-browsing experience
-    - when `Accept` header accepts html, use html regardless of query-params
-        - when query param `acceptMediatype` requests another mediatype, display on page in copy/pastable way
-        - exception: when given `withFileName`, download without html wrapping
-        - exception: `/trove/browse` should still give hypertext with clickable links
     - include more explanatory docs (and better fill out those explanations)
-    - more helpful (less erratic) visual design
+    - even more helpful (less erratic) visual design
     - in each html rendering of an api response, include a `<form>` for adding/editing/viewing query params
+    - in browsable html, replace json literals with rdf rendered like the rest of the page
+    - (perf) add bare-minimal IndexcardDeriver (iris, types, namelikes); use for search-result display
 - better tsv/csv experience
     - set default columns for `index-value-search` (and/or broadly improve `fields` handling)
 - better turtle experience

api/middleware.py

@@ -27,7 +27,7 @@ def process_view(self, request, view_func, view_args, view_kwargs):
 
         if settings.HIDE_DEPRECATED_VIEWS and deprecation_level == DeprecationLevel.HIDDEN:
             return HttpResponse(
-                f'This path ({request.path}) has been removed. If you have built something that relies on it, please email us at share-support@osf.io',
+                f'This path ({request.path}) has been removed. If you have built something that relies on it, please email us at {settings.SHARE_SUPPORT_EMAIL}',
                 status=410,
             )
 

api/views/feeds.py

@@ -1,3 +1,4 @@
+import datetime
 from xml.sax.saxutils import unescape
 import json
 import logging
@@ -10,7 +11,6 @@
 from share.search import index_strategy
 from share.search.exceptions import IndexStrategyError
 from share.util.xml import strip_illegal_xml_chars
-from share.util.fromisoformat import fromisoformat
 
 
 logger = logging.getLogger(__name__)
@@ -108,10 +108,10 @@ def item_author_name(self, item):
         return prepare_string('{}{}'.format(author_name, ' et al.' if len(authors) > 1 else ''))
 
     def item_pubdate(self, item):
-        return fromisoformat(item.get('date_published') or item.get('date_created'))
+        return datetime.datetime.fromisoformat(item.get('date_published') or item.get('date_created'))
 
     def item_updateddate(self, item):
-        return fromisoformat(item.get(self._order))
+        return datetime.datetime.fromisoformat(item.get(self._order))
 
     def item_categories(self, item):
         categories = item.get('subjects', [])

project/settings.py

@@ -445,6 +445,7 @@ def route_urgent_task(name, args, kwargs, options, task=None, **kw):
 PUBLIC_SENTRY_DSN = os.environ.get('PUBLIC_SENTRY_DSN')
 
 SHARE_WEB_URL = os.environ.get('SHARE_WEB_URL', 'http://localhost:8003').rstrip('/') + '/'
+SHARE_SUPPORT_EMAIL = os.environ.get('SHARE_SUPPORT_EMAIL', 'share-support@cos.io')
 SHARE_USER_AGENT = os.environ.get('SHARE_USER_AGENT', 'SHAREbot/{} (+{})'.format(VERSION, SHARE_WEB_URL))
 SHARE_ADMIN_USERNAME = os.environ.get('SHARE_ADMIN_USERNAME', 'admin')
 SHARE_ADMIN_PASSWORD = os.environ.get('SHARE_ADMIN_PASSWORD')

share/models/index_backfill.py

@@ -185,5 +185,4 @@ def task__schedule_index_backfill(self, index_backfill_pk):
     except Exception as error:
         _index_backfill.pls_mark_error(error)
         raise error
-    else:
-        _index_backfill.pls_note_scheduling_has_finished()
+    _index_backfill.pls_note_scheduling_has_finished()

share/oaipmh/indexcard_repository.py

@@ -1,15 +1,16 @@
+import datetime
 import uuid
 
 from django.core.exceptions import ValidationError as DjangoValidationError
+from django.conf import settings
 from django.db.models import OuterRef, Subquery, F
 
 from share.oaipmh import errors as oai_errors
 from share.oaipmh.verbs import OAIVerb
 from share.oaipmh.response_renderer import OAIRenderer
-from share.oaipmh.util import format_datetime
-from share.util.fromisoformat import fromisoformat
 from share import models as share_db
 from trove import models as trove_db
+from trove.util.datetime import datetime_isoformat_z as format_datetime
 from trove.vocab.namespaces import OAI_DC
 
 
@@ -18,7 +19,7 @@ class OaiPmhRepository:
     REPOSITORY_IDENTIFIER = 'share.osf.io'
     IDENTIFER_DELIMITER = ':'
     GRANULARITY = 'YYYY-MM-DD'
-    ADMIN_EMAILS = ['share-support@osf.io']
+    ADMIN_EMAILS = [settings.SHARE_SUPPORT_EMAIL]
 
     # TODO better way of structuring this than a bunch of dictionaries?
     # this dictionary's keys are `metadataPrefix` values
@@ -206,7 +207,7 @@ def _get_indexcard_page_queryset(self, kwargs, catch=True, last_id=None):
         )
         if 'from' in kwargs:
             try:
-                _from = fromisoformat(kwargs['from'])
+                _from = datetime.datetime.fromisoformat(kwargs['from'])
             except ValueError:
                 if not catch:
                     raise
@@ -217,7 +218,7 @@ def _get_indexcard_page_queryset(self, kwargs, catch=True, last_id=None):
                 )
         if 'until' in kwargs:
             try:
-                _until = fromisoformat(kwargs['until'])
+                _until = datetime.datetime.fromisoformat(kwargs['until'])
             except ValueError:
                 if not catch:
                     raise
@@ -291,12 +292,12 @@ def _get_resumption_token(self, kwargs, last_id):
         _until = None
         if 'from' in kwargs:
             try:
-                _from = fromisoformat(kwargs['from'])
+                _from = datetime.datetime.fromisoformat(kwargs['from'])
             except ValueError:
                 self.errors.append(oai_errors.BadArgument('Invalid value for', 'from'))
         if 'until' in kwargs:
             try:
-                _until = fromisoformat(kwargs['until'])
+                _until = datetime.datetime.fromisoformat(kwargs['until'])
             except ValueError:
                 self.errors.append(oai_errors.BadArgument('Invalid value for', 'until'))
         _set_spec = kwargs.get('set', '')

share/oaipmh/response_renderer.py

@@ -4,7 +4,8 @@
 
 from django.urls import reverse
 
-from share.oaipmh.util import format_datetime, SubEl, ns, nsmap
+from share.oaipmh.util import SubEl, ns, nsmap
+from trove.util.datetime import datetime_isoformat_z as format_datetime
 
 
 class OAIRenderer:

share/oaipmh/util.py

@@ -1,23 +1,11 @@
-import datetime
 from typing import Any
 
 from lxml import etree
 from primitive_metadata import primitive_rdf
 
-from share.util.fromisoformat import fromisoformat
 from trove.vocab.namespaces import OAI, OAI_DC
 
 
-def format_datetime(dt: datetime.datetime | primitive_rdf.Literal | str) -> str:
-    """OAI-PMH has specific time format requirements -- comply.
-    """
-    if isinstance(dt, primitive_rdf.Literal):
-        dt = dt.unicode_value
-    if isinstance(dt, str):
-        dt = fromisoformat(dt)
-    return dt.strftime('%Y-%m-%dT%H:%M:%SZ')
-
-
 XML_NAMESPACES = {
     'dc': 'http://purl.org/dc/elements/1.1/',
     'oai': str(OAI),

share/util/xml.py

@@ -15,5 +15,5 @@
 )
 
 
-def strip_illegal_xml_chars(string):
+def strip_illegal_xml_chars(string: str) -> str:
     return RE_XML_ILLEGAL.sub('', string)

templates/admin/login.html

@@ -0,0 +1,3 @@
+{% extends "admin/login.html" %}
+
+{% block content %}{{ block.super }}<a href="{% url 'osf_login' %}">login with osf</a>{% endblock %}

templates/allauth/login_errored_cancelled.html

@@ -3,9 +3,6 @@
     {% load static %}
     <head>
         <title>Login Failed</title>
-        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
-        <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
-        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
     </head>
     <body class="container">
         <div class="jumbo" style="padding-top: 25px">

tests/share/search/index_strategy/_common_trovesearch_tests.py

@@ -117,10 +117,10 @@ def test_cardsearch_pagination(self):
             }))
         self._index_indexcards(_cards)
         # gather all pages results:
-        _querystring: str = f'page[size]={_page_size}'
+        _querystring: str | None = f'page[size]={_page_size}'
         _result_iris: set[str] = set()
         _page_count = 0
-        while True:
+        while _querystring is not None:
             _cardsearch_handle = self.index_strategy.pls_handle_cardsearch(
                 CardsearchParams.from_querystring(_querystring),
             )
@@ -133,9 +133,11 @@ def test_cardsearch_pagination(self):
             _result_iris.update(_page_iris)
             _page_count += 1
             _next_cursor = _cardsearch_handle.cursor.next_cursor()
-            if _next_cursor is None:
-                break
-            _querystring = urlencode({'page[cursor]': _next_cursor.as_queryparam_value()})
+            _querystring = (
+                urlencode({'page[cursor]': _next_cursor.as_queryparam_value()})
+                if _next_cursor is not None
+                else None  # done
+            )
         self.assertEqual(_page_count, math.ceil(_total_count / _page_size))
         self.assertEqual(_result_iris, _expected_iris)
 

tests/share/search/index_strategy/_with_real_services.py

@@ -48,12 +48,6 @@ def tearDown(self):
             connections['default']._test_serialized_contents
         )
 
-    def enterContext(self, context_manager):
-        # TestCase.enterContext added in python3.11 -- implementing here until then
-        result = context_manager.__enter__()
-        self.addCleanup(lambda: context_manager.__exit__(None, None, None))
-        return result
-
     @contextlib.contextmanager
     def _daemon_up(self):
         _daemon_control = IndexerDaemonControl(celery_app)

tests/share/test_oaipmh_trove.py

@@ -8,8 +8,8 @@
 import pytest
 
 from share import models as share_db
-from share.oaipmh.util import format_datetime
 from trove import models as trove_db
+from trove.util.datetime import datetime_isoformat_z as format_datetime
 from trove.vocab.namespaces import OAI_DC
 
 from tests import factories
@@ -232,11 +232,9 @@ def _assert_full_list(self, verb, params, request_method, expected_count, page_s
         pages = 0
         count = 0
         token = None
-        while True:
-            if token:
-                parsed = oai_request({'verb': verb, 'resumptionToken': token}, request_method)
-            else:
-                parsed = oai_request({'verb': verb, 'metadataPrefix': 'oai_dc', **params}, request_method)
+        next_params: dict[str, str] | None = {'verb': verb, 'metadataPrefix': 'oai_dc', **params}
+        while next_params is not None:
+            parsed = oai_request(next_params, request_method)
             page = parsed.xpath('//oai:header/oai:identifier', namespaces=NAMESPACES)
             pages += 1
             count += len(page)
@@ -245,9 +243,10 @@ def _assert_full_list(self, verb, params, request_method, expected_count, page_s
             token = token[0].text
             if token:
                 assert len(page) == page_size
+                next_params = {'verb': verb, 'resumptionToken': token}
             else:
                 assert len(page) <= page_size
-                break
+                next_params = None  # done
 
         assert count == expected_count
         assert pages == math.ceil(expected_count / page_size)

tests/trove/_input_output_tests.py

@@ -28,12 +28,12 @@ def assert_outputs_equal(self, expected_output: typing.Any, actual_output: typin
         self.assertEqual(expected_output, actual_output)
 
     # (optional override, for when logic is more complicated)
-    def run_input_output_test(self, given_input, expected_output):
+    def run_input_output_test(self, given_input: typing.Any, expected_output: typing.Any) -> None:
         _actual_output = self.compute_output(given_input)
         self.assert_outputs_equal(expected_output, _actual_output)
 
     # (optional override, for when logic is more complicated)
-    def missing_case(self, name: str, given_input):
+    def missing_case(self, name: str, given_input: typing.Any) -> typing.Never:
         _cls = self.__class__
         _actual_output = self.compute_output(given_input)
         raise NotImplementedError('\n'.join((
@@ -43,16 +43,10 @@ def missing_case(self, name: str, given_input):
             pprint.pformat(_actual_output),
         )))
 
-    def enterContext(self, context_manager):
-        # TestCase.enterContext added in python3.11 -- implementing here until then
-        result = context_manager.__enter__()
-        self.addCleanup(lambda: context_manager.__exit__(None, None, None))
-        return result
-
     ###
     # private details
 
-    def __init_subclass__(cls, **kwargs):
+    def __init_subclass__(cls, **kwargs: typing.Any) -> None:
         super().__init_subclass__(**kwargs)
         # HACK: assign `test_*` method only on concrete subclasses,
         # so the test runner doesn't try instantiating a base class

tests/trove/digestive_tract/test_expel.py

@@ -40,12 +40,6 @@ def setUp(self):
     def _replacement_notify_indexcard_update(self, indexcards, **kwargs):
         self.notified_indexcard_ids.update(_card.id for _card in indexcards)
 
-    def enterContext(self, context_manager):
-        # TestCase.enterContext added in python3.11 -- implementing here until then
-        result = context_manager.__enter__()
-        self.addCleanup(lambda: context_manager.__exit__(None, None, None))
-        return result
-
     def test_setup(self):
         self.indexcard_1.refresh_from_db()
         self.indexcard_2.refresh_from_db()

tests/trove/render/_base.py

@@ -1,4 +1,5 @@
 import json
+import typing
 
 from primitive_metadata import (
     gather,
@@ -7,7 +8,7 @@
 
 from trove.trovesearch.trovesearch_gathering import trovesearch_by_indexstrategy
 from trove.render._base import BaseRenderer
-from trove.render._rendering import ProtoRendering
+from trove.render.rendering import ProtoRendering
 from trove.vocab.namespaces import RDF
 from tests.trove._input_output_tests import BasicInputOutputTestCase
 from ._inputs import UNRENDERED_RDF, UNRENDERED_SEARCH_RDF, RdfCase
@@ -56,7 +57,7 @@ def compute_output(self, given_input: RdfCase):
         )
         return _renderer.render_document()
 
-    def assert_outputs_equal(self, expected_output, actual_output) -> None:
+    def assert_outputs_equal(self, expected_output: typing.Any, actual_output: typing.Any) -> None:
         if expected_output is None:
             print(repr(actual_output))
             raise NotImplementedError
@@ -66,9 +67,9 @@ def assert_outputs_equal(self, expected_output, actual_output) -> None:
             self._get_rendered_output(actual_output),
         )
 
-    def _get_rendered_output(self, rendering: ProtoRendering):
+    def _get_rendered_output(self, rendering: ProtoRendering) -> str:
         # for now, they always iter strings (update if/when bytes are in play)
-        return ''.join(rendering.iter_content())  # type: ignore[arg-type]
+        return ''.join(map(str, rendering.iter_content()))
 
 
 class TrovesearchRendererTests(TroveRendererTests):