Skip to content

HttpWebRequest and WebResponse To HttpClient #1018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maikebing wants to merge 11 commits into
base: MAJOR.migrate-to-httpclient
Choose a base branch
from
Open

HttpWebRequest and WebResponse To HttpClient #1018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c22f08a
First steps to remove System.Net.HttpWebRequest & WebResponse from pu…
louis-z Apr 21, 2024
e71969c
httpclient
maikebing Dec 3, 2025
888ca7c
fix for public api
maikebing Dec 3, 2025
8900fd8
fix
maikebing Dec 3, 2025
092d572
fix multiple critical bugs:
maikebing Dec 3, 2025
5d56ad3
ProcessGitLabRequestResult
maikebing Dec 3, 2025
39ff74d
Update NGitLab/Impl/HttpRequestor.GitLabRequest.cs
maikebing Dec 3, 2025
e6a1476
Update NGitLab/Impl/HttpRequestor.GitLabRequest.cs
maikebing Dec 3, 2025
8afd20f
fix
maikebing Dec 3, 2025
523ca49
modify test
maikebing Dec 3, 2025
d262690
fix
maikebing Dec 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion NGitLab.Tests/Docker/GitLabTestContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static async Task<GitLabTestContext> CreateAsync()

public IGitLabClient Client { get; }

public WebRequest LastRequest => _customRequestOptions.AllRequests[_customRequestOptions.AllRequests.Count - 1];
public HttpRequestMessage LastRequest => _customRequestOptions.AllRequests[_customRequestOptions.AllRequests.Count - 1];

private static bool IsUnique(string str)
{
Expand Down
234 changes: 107 additions & 127 deletions NGitLab.Tests/Docker/GitLabTestContextRequestOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -17,102 +20,48 @@ namespace NGitLab.Tests.Docker;
/// </summary>
internal sealed class GitLabTestContextRequestOptions : RequestOptions
{
private readonly List<WebRequest> _allRequests = [];
private readonly List<HttpRequestMessage> _allRequests = [];
private static readonly SemaphoreSlim s_semaphoreSlim = new(1, 1);

private readonly ConcurrentDictionary<WebRequest, LoggableRequestStream> _pendingRequest = new();
private readonly ConcurrentDictionary<HttpRequestMessage, LoggableRequestStream> _pendingRequest = new();

public IReadOnlyList<WebRequest> AllRequests => _allRequests;
public IReadOnlyList<HttpRequestMessage> AllRequests => _allRequests;

public GitLabTestContextRequestOptions()
: base(retryCount: 0, retryInterval: TimeSpan.FromSeconds(1), isIncremental: true)
{
UserAgent = "NGitLab.Tests/1.0.0";
}

public override WebResponse GetResponse(HttpWebRequest request)
public override Task ProcessGitLabRequestResult(GitLabRequestResult result)
{
var request = result.Request;
lock (_allRequests)
{
_allRequests.Add(request);
}

WebResponse response = null;

// GitLab is unstable, so let's make sure we don't overload it with many concurrent requests
s_semaphoreSlim.Wait();
HttpResponseMessage response = result.Response;
try
{
try
if (result.Exception is HttpRequestException exception)
{
response = base.GetResponse(request);
}
catch (WebException exception)
{
response = exception.Response;
if (response is HttpWebResponse webResponse)
{
response = new LoggableHttpWebResponse(webResponse);
throw new WebException(exception.Message, exception, exception.Status, response);
}

throw;
}
finally
{
response = LogRequest(request, response);
//response = exception.Response;
//if (response is HttpResponseMessage webResponse)
//{
// response = new LoggableHttpWebResponse(webResponse);
// result.Exception = new WebException(exception.Message, exception, exception.Status, response);
//}
}
}
finally
{
s_semaphoreSlim.Release();
result.Response = LogRequest(request, response);
}

return response;
return Task.CompletedTask;
}

public override async Task<WebResponse> GetResponseAsync(HttpWebRequest request, CancellationToken cancellationToken)
{
lock (_allRequests)
{
_allRequests.Add(request);
}

WebResponse response = null;

// GitLab is unstable, so let's make sure we don't overload it with many concurrent requests
await s_semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
try
{
response = await base.GetResponseAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (WebException exception)
{
response = exception.Response;
if (response is HttpWebResponse webResponse)
{
response = new LoggableHttpWebResponse(webResponse);
throw new WebException(exception.Message, exception, exception.Status, response);
}

throw;
}
finally
{
response = LogRequest(request, response);
}
}
finally
{
s_semaphoreSlim.Release();
}

return response;
}

private WebResponse LogRequest(HttpWebRequest request, WebResponse response)
private HttpResponseMessage LogRequest( HttpRequestMessage request, HttpResponseMessage response)
{
byte[] requestContent = null;
if (_pendingRequest.TryRemove(request, out var requestStream))
Expand All @@ -130,7 +79,7 @@ private WebResponse LogRequest(HttpWebRequest request, WebResponse response)
{
sb.AppendLine();

if (string.Equals(request.ContentType, "application/json", StringComparison.OrdinalIgnoreCase))
if (string.Equals(request.Content.Headers.ContentType.MediaType, "application/json", StringComparison.OrdinalIgnoreCase))
{
sb.AppendLine(Encoding.UTF8.GetString(requestContent));
}
Expand All @@ -146,29 +95,26 @@ private WebResponse LogRequest(HttpWebRequest request, WebResponse response)
{
sb.AppendLine("----------");

if (response.ResponseUri != request.RequestUri)
{
sb.Append(request.RequestUri).AppendLine();
}


if (response is HttpWebResponse webResponse)
if (response is HttpResponseMessage webResponse)
{
sb.Append((int)webResponse.StatusCode).Append(' ').AppendLine(webResponse.StatusCode.ToString());
LogHeaders(sb, response.Headers);
if (string.Equals(webResponse.ContentType, "application/json", StringComparison.OrdinalIgnoreCase))
LogHeaders(sb, response.Headers);
if (string.Equals(webResponse.Content.Headers.ContentType.MediaType, "application/json", StringComparison.OrdinalIgnoreCase))
{
// This response allows multiple reads, so NGitLab can also read the response
// AllowResponseBuffering does not seem to work for WebException.Response
response = new LoggableHttpWebResponse(webResponse);
sb.AppendLine();
using var responseStream = response.GetResponseStream();
using var responseStream = response.Content.ReadAsStream();
using var sr = new StreamReader(responseStream);
var responseText = sr.ReadToEnd();
sb.AppendLine(responseText);
}
else
{
sb.Append("Binary data: ").Append(response.ContentLength).AppendLine(" bytes");
sb.Append("Binary data: ").Append(response.Content).AppendLine(" bytes");
}
}
}
Expand All @@ -178,22 +124,56 @@ private WebResponse LogRequest(HttpWebRequest request, WebResponse response)
return response;
}

internal override Stream GetRequestStream(HttpWebRequest request)
internal override Stream GetRequestStream(HttpRequestMessage request)
{
var stream = new LoggableRequestStream(request.GetRequestStream());
var stream = new LoggableRequestStream(request.Content.ReadAsStream());
_pendingRequest.AddOrUpdate(request, stream, (_, _) => stream);
return stream;
}
private static void LogHeaders(StringBuilder sb, HttpRequestHeaders _headers)
{
var headers = _headers.ToList();
for (var i = 0; i < headers.Count; i++)
{
var headerName = headers[i]!.Key ?? null;
if (headerName == null)
continue;

private static void LogHeaders(StringBuilder sb, WebHeaderCollection headers)
var headerValues = _headers.GetValues(headerName);
if (headerValues == null)
continue;

foreach (var headerValue in headerValues)
{
sb.Append(headerName).Append(": ");
if (string.Equals(headerName, "Private-Token", StringComparison.OrdinalIgnoreCase))
{
sb.AppendLine("******");
}
else if (string.Equals(headerName, "Authorization", StringComparison.OrdinalIgnoreCase))
{
const string BearerTokenPrefix = "Bearer ";
if (headerValue.StartsWith(BearerTokenPrefix, StringComparison.Ordinal))
sb.Append(BearerTokenPrefix);
sb.AppendLine("******");
}
else
{
sb.AppendLine(headerValue);
}
}
}
}
private static void LogHeaders(StringBuilder sb, HttpResponseHeaders _headers)
{
var headers = _headers.ToList();
for (var i = 0; i < headers.Count; i++)
{
var headerName = headers.GetKey(i);
var headerName = headers[i]!.Key ?? null;
if (headerName == null)
continue;

var headerValues = headers.GetValues(i);
var headerValues = _headers.GetValues(headerName);
if (headerValues == null)
continue;

Expand All @@ -219,42 +199,42 @@ private static void LogHeaders(StringBuilder sb, WebHeaderCollection headers)
}
}

private sealed class LoggableHttpWebResponse : HttpWebResponse
private sealed class LoggableHttpWebResponse : HttpResponseMessage
{
private readonly HttpWebResponse _innerWebResponse;
private byte[] _stream;
private readonly HttpResponseMessage _innerWebResponse;
// private byte[] _stream;

[Obsolete("We have to use it")]
public LoggableHttpWebResponse(HttpWebResponse innerWebResponse)
public LoggableHttpWebResponse(HttpResponseMessage innerWebResponse)
{
_innerWebResponse = innerWebResponse;
}

public override long ContentLength => _innerWebResponse.ContentLength;
//public override long ContentLength => _innerWebResponse.Cont;

public override string ContentType => _innerWebResponse.ContentType;
//public override string ContentType => _innerWebResponse.ContentType;

public override CookieCollection Cookies
{
get => _innerWebResponse.Cookies;
set => _innerWebResponse.Cookies = value;
}
//public override CookieCollection Cookies
//{
// get => _innerWebResponse.Cookies;
// set => _innerWebResponse.Cookies = value;
//}

public override WebHeaderCollection Headers => _innerWebResponse.Headers;
//public override WebHeaderCollection Headers => _innerWebResponse.Headers;

public override bool IsFromCache => _innerWebResponse.IsFromCache;
//public override bool IsFromCache => _innerWebResponse.IsFromCache;

public override bool IsMutuallyAuthenticated => _innerWebResponse.IsMutuallyAuthenticated;
//public override bool IsMutuallyAuthenticated => _innerWebResponse.IsMutuallyAuthenticated;

public override string Method => _innerWebResponse.Method;
//public override string Method => _innerWebResponse.Method;

public override Uri ResponseUri => _innerWebResponse.ResponseUri;
//public override Uri ResponseUri => _innerWebResponse.ResponseUri;

public override HttpStatusCode StatusCode => _innerWebResponse.StatusCode;
//public override HttpStatusCode StatusCode => _innerWebResponse.StatusCode;

public override string StatusDescription => _innerWebResponse.StatusDescription;
//public override string StatusDescription => _innerWebResponse.StatusDescription;

public override bool SupportsHeaders => _innerWebResponse.SupportsHeaders;
//public override bool SupportsHeaders => _innerWebResponse.SupportsHeaders;

public override bool Equals(object obj)
{
Expand All @@ -266,10 +246,10 @@ public override int GetHashCode()
return _innerWebResponse.GetHashCode();
}

public override void Close()
{
_innerWebResponse.Close();
}
//public override void Close()
//{
// _innerWebResponse.Close();
//}

protected override void Dispose(bool disposing)
{
Expand All @@ -286,25 +266,25 @@ public override string ToString()
return _innerWebResponse.ToString();
}

public override object InitializeLifetimeService()
{
return _innerWebResponse.InitializeLifetimeService();
}

public override Stream GetResponseStream()
{
if (_stream == null)
{
using var ms = new MemoryStream();
using var responseStream = _innerWebResponse.GetResponseStream();
responseStream.CopyTo(ms);

_stream = ms.ToArray();
}

var result = new MemoryStream(_stream);
return result;
}
//public override object InitializeLifetimeService()
//{
// return _innerWebResponse.InitializeLifetimeService();
//}

//public override Stream GetResponseStream()
//{
// if (_stream == null)
// {
// using var ms = new MemoryStream();
// using var responseStream = _innerWebResponse.GetResponseStream();
// responseStream.CopyTo(ms);

// _stream = ms.ToArray();
// }

// var result = new MemoryStream(_stream);
// return result;
//}
}

private sealed class LoggableRequestStream : Stream
Expand Down
Loading
Loading