diff --git a/README.md b/README.md index 5dddd7379..1eaa312bb 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,81 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http ### Using `httpx` as a library `httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder +## Common Recipes + +Below are practical one-liners for common use cases leveraging httpx's composable primitives. These recipes are validated in `runner/wellknown_recipes_test.go`. + +Use `-mdc` with DSL helpers such as `contains(content_type, ...)` and `contains(body, ...)` to match response metadata and body content. (`-mr`/`-ms` match the full raw response; use `-mdc` for structured field matching.) + +### Well-known files + +**security.txt** +Probe for a valid [RFC 9116](https://www.rfc-editor.org/rfc/rfc9116.html) security.txt file at the standard paths (`/.well-known/security.txt`, `/security.txt`): + +```bash +echo target.com | httpx -path '/.well-known/security.txt,/security.txt' -mc 200 -mdc 'contains(content_type, "text/plain") && contains(body, "Contact:") && contains_any(body, "mailto:", "https://")' +``` + +- `-path` tests custom path(s) +- `-mc 200` matches HTTP 200 +- `-mdc` matches using DSL expressions on response fields such as `content_type` and `body` + +**robots.txt** + +```bash +echo target.com | httpx -path '/robots.txt' -mc 200 -mdc 'contains(content_type, "text/plain")' +``` + +**sitemap.xml** + +```bash +echo target.com | httpx -path '/sitemap.xml' -mc 200 -mdc 'contains_any(content_type, "application/xml", "text/xml") && contains(body, "`, + }, + "/humans.txt": { + statusCode: 200, + contentType: "text/plain", + body: "/* TEAM */\nDeveloper: Example Dev\n", + }, + "/ads.txt": { + statusCode: 200, + contentType: "text/plain", + body: "google.com, pub-0000000000000000, DIRECT, f08c47fec0942fa0\n", + }, + "/.well-known/openid-configuration": { + statusCode: 200, + contentType: "application/json", + body: `{"issuer":"https://example.com","authorization_endpoint":"https://example.com/auth"}`, + }, + "/.well-known/apple-app-site-association": { + statusCode: 200, + contentType: "application/json", + body: `{"applinks":{"apps":[],"details":[]}}`, + }, + "/.well-known/apple-app-site-association.json": { + statusCode: 200, + contentType: "application/json", + body: `{"applinks":{"apps":[],"details":[]}}`, + }, + "/.well-known/assetlinks.json": { + statusCode: 200, + contentType: "application/json", + body: `[{"relation":["delegate_permission/common.handle_all_urls"],"target":{"namespace":"android_app","package_name":"com.example.app"}}]`, + }, + "/crossdomain.xml": { + statusCode: 200, + contentType: "text/xml", + body: ``, + }, + "/.well-known/change-password": { + statusCode: 200, + contentType: "text/html", + body: "Change password", + }, +} + +// soft404Fixture is an HTML error page that should not match strict well-known recipes. +var soft404Fixture = wellKnownFixture{ + statusCode: 200, + contentType: "text/html; charset=utf-8", + body: "Not Found

404

", +} diff --git a/runner/wellknown_recipes_test.go b/runner/wellknown_recipes_test.go new file mode 100644 index 000000000..327e29103 --- /dev/null +++ b/runner/wellknown_recipes_test.go @@ -0,0 +1,123 @@ +package runner + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/projectdiscovery/httpx/common/stringz" + sliceutil "github.com/projectdiscovery/utils/slice" + "github.com/stretchr/testify/require" +) + +func TestWellKnownRecipes(t *testing.T) { + for _, recipe := range WellKnownRecipes() { + t.Run(recipe.Name, func(t *testing.T) { + matched := false + for path, fixture := range wellKnownFixtures { + if !recipeMatchesPath(recipe, path) { + continue + } + result := resultFromWellKnownFixture(path, fixture) + if recipeMatches(recipe, result) { + matched = true + break + } + } + require.True(t, matched, "recipe %q should match at least one fixture", recipe.Name) + }) + } +} + +func TestWellKnownRecipeSecurityTxtRejectsSoft404(t *testing.T) { + recipe := WellKnownRecipes()[0] + result := resultFromWellKnownFixture("/.well-known/security.txt", soft404Fixture) + require.False(t, recipeMatches(recipe, result), "security.txt recipe should not match HTML soft-404 pages") +} + +func TestWellKnownRecipeSecurityTxtRejectsMissingContact(t *testing.T) { + recipe := WellKnownRecipes()[0] + result := resultFromWellKnownFixture("/.well-known/security.txt", wellKnownFixture{ + statusCode: http.StatusOK, + contentType: "text/plain", + body: "Preferred-Languages: en\n", + }) + require.False(t, recipeMatches(recipe, result), "security.txt recipe should require a Contact field") +} + +func TestWellKnownRecipeAdsTxtRejectsInvalidBody(t *testing.T) { + recipe := WellKnownRecipes()[4] + result := resultFromWellKnownFixture("/ads.txt", wellKnownFixture{ + statusCode: http.StatusOK, + contentType: "text/plain", + body: "example.com, DIRECT\n", + }) + require.False(t, recipeMatches(recipe, result), "ads.txt recipe should require an authorized digital seller entry") +} + +func TestWellKnownRecipesHTTPProbe(t *testing.T) { + ts := newWellKnownTestServer(t) + defer ts.Close() + + recipe := WellKnownRecipes()[len(WellKnownRecipes())-1] + for _, path := range strings.Split(recipe.Paths, ",") { + t.Run(path, func(t *testing.T) { + resp, err := http.Get(ts.URL + path) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusOK, resp.StatusCode) + }) + } +} + +func recipeMatchesPath(recipe WellKnownRecipe, path string) bool { + for _, recipePath := range strings.Split(recipe.Paths, ",") { + if recipePath == path { + return true + } + } + return false +} + +func recipeMatches(recipe WellKnownRecipe, result Result) bool { + if recipe.MatchStatusCode != "" { + codes, err := stringz.StringToSliceInt(recipe.MatchStatusCode) + if err != nil || !sliceutil.Contains(codes, result.StatusCode) { + return false + } + } + if recipe.MatchCondition != "" && !evalDslExpr(result, recipe.MatchCondition) { + return false + } + return true +} + +func resultFromWellKnownFixture(path string, fixture wellKnownFixture) Result { + url := "http://example.com" + path + contentType := fixture.contentType + if idx := strings.Index(contentType, ";"); idx >= 0 { + contentType = strings.TrimSpace(contentType[:idx]) + } + return Result{ + StatusCode: fixture.statusCode, + ContentType: contentType, + ResponseBody: fixture.body, + URL: url, + str: url, + } +} + +func newWellKnownTestServer(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fixture, ok := wellKnownFixtures[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", fixture.contentType) + w.WriteHeader(fixture.statusCode) + _, _ = w.Write([]byte(fixture.body)) + })) +}