diff --git a/README.md b/README.md index 7d0d21d..aef0673 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,8 @@ Most of this documentation is thanks to the team over at LinuxServer.io ## Versions +- **04.03.26:** - Added session cookie caching, updated search request building pattern +- **04.02.26:** - Redid search normalization, more search queries should now be supported - **08.15.25:** - Created arm64 variant and fixed bug that caused search to error when capital letters were used in the query. - **07.07.25:** - Updated Dockerfile so that both Prowlarr's `master` and `develop` (which changes to .net8.0) branches can be built. - **08.29.23:** - Change User-Agent to bypass AudioBookBay's block. diff --git a/custom_indexers/AudioBookBay.cs b/custom_indexers/AudioBookBay.cs index 749e425..3bfdd14 100644 --- a/custom_indexers/AudioBookBay.cs +++ b/custom_indexers/AudioBookBay.cs @@ -22,6 +22,8 @@ namespace NzbDrone.Core.Indexers.Definitions; public class AudioBookBay : TorrentIndexerBase { + private const string PhpSessionCookie = "PHPSESSID"; + public override string Name => "AudioBook Bay"; public override string[] IndexerUrls => new[] { @@ -69,7 +71,7 @@ public AudioBookBay(IIndexerHttpClient httpClient, IEventAggregator eventAggrega public override IIndexerRequestGenerator GetRequestGenerator() { - return new AudioBookBayRequestGenerator(Settings, Capabilities); + return new AudioBookBayRequestGenerator(Settings); } public override IParseIndexerResponse GetParser() @@ -108,6 +110,61 @@ public override async Task Download(Uri link) return await base.Download(new Uri(magnet)); } + protected override async Task FetchPage(IndexerRequest request, IParseIndexerResponse parser) + { + var response = await FetchIndexerResponse(request); + + if (ShouldRetryWithFreshSession(request, response)) + { + var hadSessionCookie = Cookies != null && Cookies.ContainsKey(PhpSessionCookie); + _logger.Debug("ABB search response contained no result rows for {0} using {1}cached session cookie. Refreshing session cookie and retrying once.", + request.Url.FullUri, + hadSessionCookie ? "an existing " : "no "); + + var refreshedSession = await TryRefreshSessionCookie(); + if (refreshedSession) + { + ModifyRequest(request); + response = await FetchIndexerResponse(request); + + if (ShouldRetryWithFreshSession(request, response)) + { + _logger.Debug("ABB search retry still contained no result rows for {0} after refreshing the session cookie.", request.Url.FullUri); + } + else + { + _logger.Debug("ABB search retry succeeded for {0} after refreshing the session cookie.", request.Url.FullUri); + } + } + else + { + _logger.Debug("ABB session refresh did not produce a usable {0} cookie for {1}.", PhpSessionCookie, request.Url.FullUri); + } + } + + try + { + var releases = parser.ParseResponse(response).ToList(); + + if (releases.Count == 0) + { + _logger.Trace("No releases found. Response: {0}", response.Content); + } + + return new IndexerQueryResult + { + Releases = releases, + Response = response.HttpResponse + }; + } + catch (Exception ex) + { + ex.WithData(response.HttpResponse, 128 * 1024); + _logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content); + throw; + } + } + private IndexerCapabilities SetCapabilities() { var caps = new IndexerCapabilities @@ -179,17 +236,82 @@ private IndexerCapabilities SetCapabilities() return caps; } + + private static bool IsSearchRequestWithTerm(HttpUri url) + { + return url?.Query.IsNotNullOrWhiteSpace() == true && Regex.IsMatch(url.Query, @"(?:^|&)s=[^&]+", RegexOptions.Compiled); + } + + private static bool IsFirstSearchPage(HttpUri url) + { + return url?.Path.IsNullOrWhiteSpace() != false || url.Path.Equals("/", StringComparison.OrdinalIgnoreCase); + } + + private bool ShouldRetryWithFreshSession(IndexerRequest request, IndexerResponse response) + { + return IsSearchRequestWithTerm(request.Url) && + IsFirstSearchPage(request.Url) && + !AudioBookBayParser.HasResultRows(response.Content); + } + + private async Task TryRefreshSessionCookie() + { + try + { + var bootstrapRequest = new HttpRequestBuilder(new UriBuilder(Settings.BaseUrl) { Path = "/" }.Uri.AbsoluteUri) + .Accept(HttpAccept.Html) + .Build(); + + bootstrapRequest.RateLimit = RateLimit; + bootstrapRequest.Encoding ??= Encoding; + bootstrapRequest.SuppressHttpError = true; + + if (_configService.LogIndexerResponse) + { + bootstrapRequest.LogResponseContent = true; + } + + _logger.Debug("Refreshing ABB session cookie via {0}", bootstrapRequest.Url.FullUri); + + var bootstrapResponse = await RetryStrategy + .ExecuteAsync(static async (state, _) => await state.HttpClient.ExecuteProxiedAsync(state.Request, state.Definition), + (HttpClient: _httpClient, Request: bootstrapRequest, Definition)); + + var cookies = bootstrapResponse.GetCookies(); + if (cookies == null || !cookies.Any()) + { + _logger.Debug("ABB session bootstrap response did not include any cookies."); + return false; + } + + UpdateCookies(cookies, DateTime.Now.AddDays(30)); + _logger.Trace("ABB session bootstrap returned cookies: {0}", string.Join(", ", cookies.Keys.OrderBy(x => x))); + + return cookies.ContainsKey(PhpSessionCookie); + } + catch (Exception ex) + { + _logger.Debug(ex, "ABB session bootstrap request failed."); + return false; + } + } } public class AudioBookBayRequestGenerator : IIndexerRequestGenerator { + private static readonly Regex StandardizeDashesRegex = new(@"[\p{Pd}\u2212]+", RegexOptions.Compiled); + private static readonly Regex StandardizeSlashRegex = new(@"[\u2044\u2215]+", RegexOptions.Compiled); + private static readonly Regex StandardizeSingleQuotesRegex = new(@"[\u0060\u00B4\u2018\u2019\u201B\u02BC\uFF07]", RegexOptions.Compiled); + private static readonly Regex StandardizeDoubleQuotesRegex = new(@"[""\u201C\u201D\u201E\u201F\u00AB\u00BB\uFF02]+", RegexOptions.Compiled); + private static readonly Regex NormalizeSeparatorsRegex = new(@"[/:&\\]+", RegexOptions.Compiled); + private static readonly Regex StripDisallowedCharactersRegex = new(@"[^\p{L}\p{N}\s\-\._\(\)@/'\[\]\+%\*#""]+", RegexOptions.Compiled); + private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled); + private readonly NoAuthTorrentBaseSettings _settings; - private readonly IndexerCapabilities _capabilities; - public AudioBookBayRequestGenerator(NoAuthTorrentBaseSettings settings, IndexerCapabilities capabilities) + public AudioBookBayRequestGenerator(NoAuthTorrentBaseSettings settings) { _settings = settings; - _capabilities = capabilities; } public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) @@ -211,7 +333,7 @@ public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCr { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); + pageableRequests.Add(GetPagedRequests(searchCriteria.SearchTerm ?? string.Empty)); return pageableRequests; } @@ -220,7 +342,7 @@ public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchC { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); + pageableRequests.Add(GetPagedRequests(searchCriteria.SearchTerm ?? string.Empty)); return pageableRequests; } @@ -231,12 +353,12 @@ private IEnumerable GetPagedRequests(string term) var parameters = new NameValueCollection(); - term = Regex.Replace(term, @"[\W]+", " ").Trim().ToLower(); + term = NormalizeSearchTerm(term); if (term.IsNotNullOrWhiteSpace()) { parameters.Set("s", term); - parameters.Set("tt", "1"); + parameters.Set("cat", "undefined,undefined"); } if (parameters.Count > 0) @@ -249,6 +371,22 @@ private IEnumerable GetPagedRequests(string term) yield return new IndexerRequest(new UriBuilder(searchUrl) { Path = "/page/3/" }.Uri.AbsoluteUri, HttpAccept.Html); } + private static string NormalizeSearchTerm(string term) + { + term ??= string.Empty; + + term = StandardizeDashesRegex.Replace(term, "-"); + term = StandardizeSlashRegex.Replace(term, "/"); + term = StandardizeSingleQuotesRegex.Replace(term, "'"); + term = StandardizeDoubleQuotesRegex.Replace(term, "\""); + term = term.ToLowerInvariant(); + term = NormalizeSeparatorsRegex.Replace(term, " "); + term = StripDisallowedCharactersRegex.Replace(term, " "); + term = CollapseWhitespaceRegex.Replace(term, " ").Trim(); + + return term; + } + public Func> GetCookies { get; set; } public Action, DateTime?> CookiesUpdater { get; set; } } @@ -268,6 +406,12 @@ public IList ParseResponse(IndexerResponse indexerResponse) { var releaseInfos = new List(); + var cookies = indexerResponse.HttpResponse.GetCookies(); + if (cookies != null && cookies.Any() && CookiesUpdater != null) + { + CookiesUpdater(cookies, DateTime.Now.AddDays(30)); + } + var doc = ParseHtmlDocument(indexerResponse.Content); var rows = doc.QuerySelectorAll("div.post:has(div[class=\"postTitle\"])"); @@ -329,6 +473,12 @@ public IList ParseResponse(IndexerResponse indexerResponse) return releaseInfos; } + internal static bool HasResultRows(string response) + { + var doc = ParseHtmlDocument(response); + return doc.QuerySelector("div.post:has(div[class=\"postTitle\"])") != null; + } + private static IHtmlDocument ParseHtmlDocument(string response) { var parser = new HtmlParser();