Skip to content

redo search normalization and cleanup request generation #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AllergicSquare wants to merge 4 commits into
base: main
Choose a base branch
from
Open

redo search normalization and cleanup request generation #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e283a00
redo search normalization and cleanup request generation
AllergicSquare Apr 2, 2026
54ff264
update version history accordingly
AllergicSquare Apr 2, 2026
9295181
add session cookie caching, change search request building pattern sl…
AllergicSquare Apr 3, 2026
13a8a00
Update README with latest version changes
AllergicSquare Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,8 @@ Most of this documentation is thanks to the team over at LinuxServer.io

## Versions

- **04.03.26:** - Added session cookie caching, updated search request building pattern
- **04.02.26:** - Redid search normalization, more search queries should now be supported
- **08.15.25:** - Created arm64 variant and fixed bug that caused search to error when capital letters were used in the query.
- **07.07.25:** - Updated Dockerfile so that both Prowlarr's `master` and `develop` (which changes to .net8.0) branches can be built.
- **08.29.23:** - Change User-Agent to bypass AudioBookBay's block.
Expand Down
166 changes: 158 additions & 8 deletions custom_indexers/AudioBookBay.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ namespace NzbDrone.Core.Indexers.Definitions;

public class AudioBookBay : TorrentIndexerBase<NoAuthTorrentBaseSettings>
{
private const string PhpSessionCookie = "PHPSESSID";

public override string Name => "AudioBook Bay";
public override string[] IndexerUrls => new[]
{
Expand Down Expand Up @@ -69,7 +71,7 @@ public AudioBookBay(IIndexerHttpClient httpClient, IEventAggregator eventAggrega

public override IIndexerRequestGenerator GetRequestGenerator()
{
return new AudioBookBayRequestGenerator(Settings, Capabilities);
return new AudioBookBayRequestGenerator(Settings);
}

public override IParseIndexerResponse GetParser()
Expand Down Expand Up @@ -108,6 +110,61 @@ public override async Task<IndexerDownloadResponse> Download(Uri link)
return await base.Download(new Uri(magnet));
}

protected override async Task<IndexerQueryResult> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
{
var response = await FetchIndexerResponse(request);

if (ShouldRetryWithFreshSession(request, response))
{
var hadSessionCookie = Cookies != null && Cookies.ContainsKey(PhpSessionCookie);
_logger.Debug("ABB search response contained no result rows for {0} using {1}cached session cookie. Refreshing session cookie and retrying once.",
request.Url.FullUri,
hadSessionCookie ? "an existing " : "no ");

var refreshedSession = await TryRefreshSessionCookie();
if (refreshedSession)
{
ModifyRequest(request);
response = await FetchIndexerResponse(request);

if (ShouldRetryWithFreshSession(request, response))
{
_logger.Debug("ABB search retry still contained no result rows for {0} after refreshing the session cookie.", request.Url.FullUri);
}
else
{
_logger.Debug("ABB search retry succeeded for {0} after refreshing the session cookie.", request.Url.FullUri);
}
}
else
{
_logger.Debug("ABB session refresh did not produce a usable {0} cookie for {1}.", PhpSessionCookie, request.Url.FullUri);
}
}

try
{
var releases = parser.ParseResponse(response).ToList();

if (releases.Count == 0)
{
_logger.Trace("No releases found. Response: {0}", response.Content);
}

return new IndexerQueryResult
{
Releases = releases,
Response = response.HttpResponse
};
}
catch (Exception ex)
{
ex.WithData(response.HttpResponse, 128 * 1024);
_logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content);
throw;
}
}

private IndexerCapabilities SetCapabilities()
{
var caps = new IndexerCapabilities
Expand Down Expand Up @@ -179,17 +236,82 @@ private IndexerCapabilities SetCapabilities()

return caps;
}

private static bool IsSearchRequestWithTerm(HttpUri url)
{
return url?.Query.IsNotNullOrWhiteSpace() == true && Regex.IsMatch(url.Query, @"(?:^|&)s=[^&]+", RegexOptions.Compiled);
}

private static bool IsFirstSearchPage(HttpUri url)
{
return url?.Path.IsNullOrWhiteSpace() != false || url.Path.Equals("/", StringComparison.OrdinalIgnoreCase);
}

private bool ShouldRetryWithFreshSession(IndexerRequest request, IndexerResponse response)
{
return IsSearchRequestWithTerm(request.Url) &&
IsFirstSearchPage(request.Url) &&
!AudioBookBayParser.HasResultRows(response.Content);
}

private async Task<bool> TryRefreshSessionCookie()
{
try
{
var bootstrapRequest = new HttpRequestBuilder(new UriBuilder(Settings.BaseUrl) { Path = "/" }.Uri.AbsoluteUri)
.Accept(HttpAccept.Html)
.Build();

bootstrapRequest.RateLimit = RateLimit;
bootstrapRequest.Encoding ??= Encoding;
bootstrapRequest.SuppressHttpError = true;

if (_configService.LogIndexerResponse)
{
bootstrapRequest.LogResponseContent = true;
}

_logger.Debug("Refreshing ABB session cookie via {0}", bootstrapRequest.Url.FullUri);

var bootstrapResponse = await RetryStrategy
.ExecuteAsync(static async (state, _) => await state.HttpClient.ExecuteProxiedAsync(state.Request, state.Definition),
(HttpClient: _httpClient, Request: bootstrapRequest, Definition));

var cookies = bootstrapResponse.GetCookies();
if (cookies == null || !cookies.Any())
{
_logger.Debug("ABB session bootstrap response did not include any cookies.");
return false;
}

UpdateCookies(cookies, DateTime.Now.AddDays(30));
_logger.Trace("ABB session bootstrap returned cookies: {0}", string.Join(", ", cookies.Keys.OrderBy(x => x)));

return cookies.ContainsKey(PhpSessionCookie);
}
catch (Exception ex)
{
_logger.Debug(ex, "ABB session bootstrap request failed.");
return false;
}
}
}

public class AudioBookBayRequestGenerator : IIndexerRequestGenerator
{
private static readonly Regex StandardizeDashesRegex = new(@"[\p{Pd}\u2212]+", RegexOptions.Compiled);
private static readonly Regex StandardizeSlashRegex = new(@"[\u2044\u2215]+", RegexOptions.Compiled);
private static readonly Regex StandardizeSingleQuotesRegex = new(@"[\u0060\u00B4\u2018\u2019\u201B\u02BC\uFF07]", RegexOptions.Compiled);
private static readonly Regex StandardizeDoubleQuotesRegex = new(@"[""\u201C\u201D\u201E\u201F\u00AB\u00BB\uFF02]+", RegexOptions.Compiled);
private static readonly Regex NormalizeSeparatorsRegex = new(@"[/:&\\]+", RegexOptions.Compiled);
private static readonly Regex StripDisallowedCharactersRegex = new(@"[^\p{L}\p{N}\s\-\._\(\)@/'\[\]\+%\*#""]+", RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);

private readonly NoAuthTorrentBaseSettings _settings;
private readonly IndexerCapabilities _capabilities;

public AudioBookBayRequestGenerator(NoAuthTorrentBaseSettings settings, IndexerCapabilities capabilities)
public AudioBookBayRequestGenerator(NoAuthTorrentBaseSettings settings)
{
_settings = settings;
_capabilities = capabilities;
}

public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
Expand All @@ -211,7 +333,7 @@ public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCr
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
pageableRequests.Add(GetPagedRequests(searchCriteria.SearchTerm ?? string.Empty));

return pageableRequests;
}
Expand All @@ -220,7 +342,7 @@ public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchC
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
pageableRequests.Add(GetPagedRequests(searchCriteria.SearchTerm ?? string.Empty));

return pageableRequests;
}
Expand All @@ -231,12 +353,12 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string term)

var parameters = new NameValueCollection();

term = Regex.Replace(term, @"[\W]+", " ").Trim().ToLower();
term = NormalizeSearchTerm(term);

if (term.IsNotNullOrWhiteSpace())
{
parameters.Set("s", term);
parameters.Set("tt", "1");
parameters.Set("cat", "undefined,undefined");
}

if (parameters.Count > 0)
Expand All @@ -249,6 +371,22 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string term)
yield return new IndexerRequest(new UriBuilder(searchUrl) { Path = "/page/3/" }.Uri.AbsoluteUri, HttpAccept.Html);
}

private static string NormalizeSearchTerm(string term)
{
term ??= string.Empty;

term = StandardizeDashesRegex.Replace(term, "-");
term = StandardizeSlashRegex.Replace(term, "/");
term = StandardizeSingleQuotesRegex.Replace(term, "'");
term = StandardizeDoubleQuotesRegex.Replace(term, "\"");
term = term.ToLowerInvariant();
term = NormalizeSeparatorsRegex.Replace(term, " ");
term = StripDisallowedCharactersRegex.Replace(term, " ");
term = CollapseWhitespaceRegex.Replace(term, " ").Trim();

return term;
}

public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
Expand All @@ -268,6 +406,12 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var releaseInfos = new List<ReleaseInfo>();

var cookies = indexerResponse.HttpResponse.GetCookies();
if (cookies != null && cookies.Any() && CookiesUpdater != null)
{
CookiesUpdater(cookies, DateTime.Now.AddDays(30));
}

var doc = ParseHtmlDocument(indexerResponse.Content);

var rows = doc.QuerySelectorAll("div.post:has(div[class=\"postTitle\"])");
Expand Down Expand Up @@ -329,6 +473,12 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
return releaseInfos;
}

internal static bool HasResultRows(string response)
{
var doc = ParseHtmlDocument(response);
return doc.QuerySelector("div.post:has(div[class=\"postTitle\"])") != null;
}

private static IHtmlDocument ParseHtmlDocument(string response)
{
var parser = new HtmlParser();
Expand Down