From dc41ee8df6ebee5341640de8b8127f728e6de05e Mon Sep 17 00:00:00 2001 From: rochala Date: Wed, 26 Aug 2026 19:04:51 +0200 Subject: [PATCH] Pass coursier credentials to private repositories Users with artifacts on a private Artifactory got 401s from cellar while their sbt/Mill builds resolved fine. Two causes in coursier-interface: - 1.0.28 never read ~/.config/coursier/credentials.properties or COURSIER_CREDENTIALS (coursier/interface#460). Fixed upstream in 1.0.29-M4, so bump to it. - Even on 1.0.29-M4, ApiHelper.fetch gives the credential-bearing cache only to Resolve; Artifacts() keeps its default cache, so POMs authenticate but JAR downloads still fail with 401. Repository-level credentials travel with every artifact request, so read the same sources coursier does and attach them to each --repository whose host matches. Verified against a basic-auth Maven server: POM and JAR both download with only credentials.properties present. Also disable per-class test forking for lib.test: one JVM per test class, each booting a tasty-query Context, saturated the machine and made tests fail under load. Co-Authored-By: Claude Fable 5 --- README.md | 5 ++ build.mill | 5 +- cli/src/cellar/cli/CellarApp.scala | 50 +++++++------ lib/src/cellar/CoursierCredentials.scala | 72 +++++++++++++++++++ .../src/cellar/CoursierCredentialsTest.scala | 56 +++++++++++++++ 5 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 lib/src/cellar/CoursierCredentials.scala create mode 100644 lib/test/src/cellar/CoursierCredentialsTest.scala diff --git a/README.md b/README.md index 423830c..141a6be 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,11 @@ cellar get-external org.typelevel:cats-core_3:latest cats.Monad | `-v`, `--verbose` | all except `telemetry` | Log progress and warnings to stderr | | `--debug` | all except `telemetry` | Log detailed diagnostics and stack traces to stderr | +Private repositories: credentials from `~/.config/coursier/credentials.properties` (or +`COURSIER_CONFIG_DIR`) and the `COURSIER_CREDENTIALS` environment variable are read the same way +the coursier CLI, sbt and Mill read them, and applied to every `--repository` whose host matches. +See the [coursier credentials docs](https://get-coursier.io/docs/other-credentials). + Diagnostics are written to stderr, never stdout, so `--verbose` and `--debug` are safe to use when piping output into a prompt. Set `CELLAR_LOG=verbose` or `CELLAR_LOG=debug` to get the same effect without a flag — useful for debugging an installed binary. An explicit flag wins over the diff --git a/build.mill b/build.mill index 23637ac..5eb805c 100644 --- a/build.mill +++ b/build.mill @@ -61,7 +61,7 @@ object lib extends CellarPublishModule { def mvnDeps = Seq( mvn"org.typelevel::cats-effect:3.7.1", mvn"co.fs2::fs2-io:3.13.0", - mvn"io.get-coursier:interface:1.0.28", + mvn"io.get-coursier:interface:1.0.29-M4", mvn"ch.epfl.scala::tasty-query:1.8.0", mvn"org.scala-lang:scala3-tasty-inspector_3:3.8.4", mvn"com.github.pureconfig::pureconfig-core:0.17.10", @@ -71,6 +71,9 @@ object lib extends CellarPublishModule { object test extends ScalaTests with TestModule.Munit { + // Every test class boots its own tasty-query Context; one forked JVM per class saturates + // the whole machine and starts failing under load, so run them in a single JVM. + def testParallelism = false def mvnDeps = super.mvnDeps() ++ Seq( mvn"org.scalameta::munit:1.3.5", mvn"org.typelevel::munit-cats-effect:2.2.0" diff --git a/cli/src/cellar/cli/CellarApp.scala b/cli/src/cellar/cli/CellarApp.scala index a3fa75d..bd4acfd 100644 --- a/cli/src/cellar/cli/CellarApp.scala +++ b/cli/src/cellar/cli/CellarApp.scala @@ -149,12 +149,16 @@ object CellarApp extends ProfilingIOApp: private val javaHomeOpt: Opts[Option[Path]] = Opts.option[Path]("java-home", "Use a specific JDK for JRE classpath").orNone - private val extraReposOpt: Opts[List[Repository]] = + private val extraReposOpt: Opts[IO[List[Repository]]] = Opts.options[String]("repository", "Extra Maven repository URL (repeatable)", short = "r", metavar = "url") .orEmpty .mapValidated(_.traverse { raw => RepositoryUrl.parse(raw).leftMap(reason => s"Invalid --repository: $reason").toValidatedNel }) + .map { repos => + if repos.isEmpty then IO.pure(Nil) + else CoursierCredentials.load().map(creds => repos.map(CoursierCredentials.applyTo(_, creds))) + } private val limitOpt: Opts[Int] = Opts @@ -194,10 +198,14 @@ object CellarApp extends ProfilingIOApp: private val groupInheritedOpt: Opts[Boolean] = Opts.flag("group-inherited", "Group members by declaring type with section headers").orFalse - private def parseAndResolve(raw: String, extraRepos: List[Repository]): IO[Either[String, MavenCoordinate]] = + private def parseAndResolve( + raw: String, + extraReposIO: IO[List[Repository]] + ): IO[Either[String, (MavenCoordinate, List[Repository])]] = MavenCoordinate.parse(raw) match case Left(err) => IO.pure(Left(err)) - case Right(coord) => coord.resolveLatest(extraRepos).map(Right(_)) + case Right(coord) => + extraReposIO.flatMap(extraRepos => coord.resolveLatest(extraRepos).map(c => Right((c, extraRepos)))) private val getSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("get", "Fetch symbol info from the current project") { @@ -232,11 +240,11 @@ object CellarApp extends ProfilingIOApp: private val getExternalSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("get-external", "Fetch symbol info from a Maven coordinate") { (coordArg, symbolArg, memberLimitOpt, hideInheritedOpt, groupInheritedOpt, javaHomeOpt, extraReposOpt, loggerOpt).mapN { - (rawCoord, fqn, limit, hideInherited, groupInherited, javaHome, extraRepos, logger) => + (rawCoord, fqn, limit, hideInherited, groupInherited, javaHome, extraReposIO, logger) => traced("get-external") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => + case Right((coord, extraRepos)) => GetHandler.run(coord, fqn, javaHome, extraRepos, limit, hideInherited, groupInherited, logger) } } @@ -245,11 +253,11 @@ object CellarApp extends ProfilingIOApp: private val getSourceSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("get-source", "Fetch the source code of a named symbol") { - (coordArg, symbolArg, javaHomeOpt, extraReposOpt, loggerOpt).mapN { (rawCoord, fqn, javaHome, extraRepos, logger) => + (coordArg, symbolArg, javaHomeOpt, extraReposOpt, loggerOpt).mapN { (rawCoord, fqn, javaHome, extraReposIO, logger) => traced("get-source") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => GetSourceHandler.run(coord, fqn, javaHome, extraRepos, logger) + case Right((coord, extraRepos)) => GetSourceHandler.run(coord, fqn, javaHome, extraRepos, logger) } } } @@ -257,11 +265,11 @@ object CellarApp extends ProfilingIOApp: private val listExternalSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("list-external", "List symbols from a Maven coordinate") { - (coordArg, symbolArg, limitOpt, javaHomeOpt, extraReposOpt, loggerOpt).mapN { (rawCoord, fqn, limit, javaHome, extraRepos, logger) => + (coordArg, symbolArg, limitOpt, javaHomeOpt, extraReposOpt, loggerOpt).mapN { (rawCoord, fqn, limit, javaHome, extraReposIO, logger) => traced("list-external") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => ListHandler.run(coord, fqn, limit, javaHome, extraRepos, logger) + case Right((coord, extraRepos)) => ListHandler.run(coord, fqn, limit, javaHome, extraRepos, logger) } } } @@ -270,11 +278,11 @@ object CellarApp extends ProfilingIOApp: private val searchExternalSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("search-external", "Substring search for symbol names from a Maven coordinate") { (coordArg, Opts.argument[String]("query"), limitOpt, javaHomeOpt, extraReposOpt, loggerOpt).mapN { - (rawCoord, query, limit, javaHome, extraRepos, logger) => + (rawCoord, query, limit, javaHome, extraReposIO, logger) => traced("search-external") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => SearchHandler.run(coord, query, limit, javaHome, extraRepos, logger) + case Right((coord, extraRepos)) => SearchHandler.run(coord, query, limit, javaHome, extraRepos, logger) } } } @@ -282,11 +290,11 @@ object CellarApp extends ProfilingIOApp: private val depsSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("deps", "Print the transitive dependency list") { - (coordArg, extraReposOpt, loggerOpt).mapN { (rawCoord, extraRepos, logger) => + (coordArg, extraReposOpt, loggerOpt).mapN { (rawCoord, extraReposIO, logger) => traced("deps") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => DepsHandler.run(coord, extraRepositories = extraRepos, logger = logger) + case Right((coord, extraRepos)) => DepsHandler.run(coord, extraRepositories = extraRepos, logger = logger) } } } @@ -294,11 +302,11 @@ object CellarApp extends ProfilingIOApp: private val metaSubcmd: Opts[IO[ExitCode]] = Opts.subcommand("meta", "Print POM metadata (name, description, license, SCM, developers)") { - (coordArg, extraReposOpt, loggerOpt).mapN { (rawCoord, extraRepos, logger) => + (coordArg, extraReposOpt, loggerOpt).mapN { (rawCoord, extraReposIO, logger) => traced("meta") { - parseAndResolve(rawCoord, extraRepos).flatMap { + parseAndResolve(rawCoord, extraReposIO).flatMap { case Left(err) => IO.blocking(System.err.println(err)).as(ExitCode.Error) - case Right(coord) => MetaHandler.run(coord, extraRepositories = extraRepos, logger = logger) + case Right((coord, extraRepos)) => MetaHandler.run(coord, extraRepositories = extraRepos, logger = logger) } } } diff --git a/lib/src/cellar/CoursierCredentials.scala b/lib/src/cellar/CoursierCredentials.scala new file mode 100644 index 0000000..dd3b4d1 --- /dev/null +++ b/lib/src/cellar/CoursierCredentials.scala @@ -0,0 +1,72 @@ +package cellar + +import cats.effect.IO +import coursierapi.{Credentials, MavenRepository, Repository} +import fs2.io.file.{Files, Path} + +import java.io.StringReader +import java.net.URI +import java.util.Properties +import scala.jdk.CollectionConverters.* + +/** Reads the credentials coursier itself would use (`COURSIER_CREDENTIALS`, then + * `credentials.properties` in the coursier config dir) and pins them onto each repository. + * + * coursier-interface already attaches these to its cache, but only the metadata (POM) requests + * see them: the artifact download path is built with a separate default cache and never retries + * a 401 with credentials. Repository-level credentials travel with every artifact request, so + * they are the only way to reach the JAR download. + */ +object CoursierCredentials: + def load( + env: Map[String, String] = sys.env, + home: Path = Path(sys.props("user.home")) + ): IO[List[Credentials]] = + env.get("COURSIER_CREDENTIALS").filter(_.trim.nonEmpty) match + case Some(value) if value.startsWith("/") => fromFile(Path(value)) + case Some(value) if value.startsWith("file:") => fromFile(Path(URI.create(value).getPath)) + case Some(value) => IO.pure(parseInline(value)) + case None => + val configDir = env.get("COURSIER_CONFIG_DIR").map(Path(_)).getOrElse { + val xdg = env.get("XDG_CONFIG_HOME").map(Path(_)).getOrElse(home / ".config") + if sys.props("os.name").toLowerCase.contains("mac") then home / "Library" / "Application Support" / "Coursier" + else xdg / "coursier" + } + fromFile(configDir / "credentials.properties") + + def applyTo(repo: Repository, credentials: List[Credentials]): Repository = repo match + case mvn: MavenRepository if mvn.getCredentials == null => + val uri = URI.create(mvn.getBase) + credentials + .find(c => c.getHost == uri.getHost && (!c.isHttpsOnly || uri.getScheme == "https")) + .fold(repo)(mvn.withCredentials) + case other => other + + def parseInline(content: String): List[Credentials] = + content.linesIterator.map(_.trim).filter(_.nonEmpty).toList.flatMap { + case s"$host($realm) $user:$password" => Some(Credentials.of(host, user, password, realm)) + case s"$host $user:$password" => Some(Credentials.of(host, user, password)) + case _ => None + } + + def parseProperties(props: Properties): List[Credentials] = + props.stringPropertyNames.asScala.toList.filter(_.endsWith(".username")).sorted.flatMap { userKey => + val prefix = userKey.stripSuffix(".username") + for + host <- Option(props.getProperty(s"$prefix.host")) + password <- Option(props.getProperty(s"$prefix.password")) + yield + val base = Credentials.of(host, props.getProperty(userKey), password, props.getProperty(s"$prefix.realm")) + Option(props.getProperty(s"$prefix.https-only")).map(_.toBoolean).fold(base.withHttpsOnly(true))(base.withHttpsOnly) + } + + private def fromFile(path: Path): IO[List[Credentials]] = + Files[IO].isRegularFile(path).flatMap { + case false => IO.pure(Nil) + case true => + Files[IO].readUtf8(path).compile.string.map { content => + val props = new Properties + props.load(new StringReader(content)) + parseProperties(props) + }.handleError(_ => Nil) + } diff --git a/lib/test/src/cellar/CoursierCredentialsTest.scala b/lib/test/src/cellar/CoursierCredentialsTest.scala new file mode 100644 index 0000000..cc75419 --- /dev/null +++ b/lib/test/src/cellar/CoursierCredentialsTest.scala @@ -0,0 +1,56 @@ +package cellar + +import coursierapi.MavenRepository +import fs2.io.file.Files + +import cats.effect.IO +import cats.effect.unsafe.implicits.global + +import java.util.Properties + +class CoursierCredentialsTest extends munit.FunSuite: + + private def props(pairs: (String, String)*): Properties = + val p = new Properties + pairs.foreach((k, v) => p.setProperty(k, v)) + p + + test("parseProperties reads a coursier credentials.properties entry"): + val creds = CoursierCredentials.parseProperties( + props("art.host" -> "artifactory.example.com", "art.username" -> "alice", "art.password" -> "s3cret", "art.realm" -> "Artifactory Realm") + ) + assertEquals(creds.map(c => (c.getHost, c.getUser, c.getPassword, c.getRealm, c.isHttpsOnly)), + List(("artifactory.example.com", "alice", "s3cret", "Artifactory Realm", true))) + + test("parseProperties honours https-only=false and skips entries without host"): + val creds = CoursierCredentials.parseProperties( + props("a.host" -> "h", "a.username" -> "u", "a.password" -> "p", "a.https-only" -> "false", "b.username" -> "u", "b.password" -> "p") + ) + assertEquals(creds.map(c => (c.getHost, c.isHttpsOnly)), List(("h", false))) + + test("parseInline reads COURSIER_CREDENTIALS lines with and without realm"): + val creds = CoursierCredentials.parseInline(" h1(My Realm) u1:p1\n\nh2 u2:p2\n") + assertEquals(creds.map(c => (c.getHost, c.getUser, c.getPassword, Option(c.getRealm))), + List(("h1", "u1", "p1", Some("My Realm")), ("h2", "u2", "p2", None))) + + test("applyTo attaches credentials only to the repository whose host matches"): + val creds = CoursierCredentials.parseInline("artifactory.example.com alice:s3cret") + val hit = CoursierCredentials.applyTo(MavenRepository.of("https://artifactory.example.com/maven"), creds) + val miss = CoursierCredentials.applyTo(MavenRepository.of("https://repo1.maven.org/maven2"), creds) + assertEquals(hit.asInstanceOf[MavenRepository].getCredentials.getUser, "alice") + assertEquals(miss.asInstanceOf[MavenRepository].getCredentials, null) + + test("applyTo skips https-only credentials for an http repository"): + val creds = CoursierCredentials.parseProperties(props("a.host" -> "h", "a.username" -> "u", "a.password" -> "p")) + val repo = CoursierCredentials.applyTo(MavenRepository.of("http://h/maven"), creds) + assertEquals(repo.asInstanceOf[MavenRepository].getCredentials, null) + + test("load prefers COURSIER_CREDENTIALS and falls back to the config dir file"): + val dir = Files[IO].createTempDirectory.unsafeRunSync() + fs2.Stream.emit("a.host=filehost\na.username=u\na.password=p\n").through(Files[IO].writeUtf8(dir / "credentials.properties")).compile.drain.unsafeRunSync() + val fromFile = CoursierCredentials.load(Map("COURSIER_CONFIG_DIR" -> dir.toString), dir).unsafeRunSync() + val fromEnv = CoursierCredentials.load(Map("COURSIER_CREDENTIALS" -> "envhost u:p", "COURSIER_CONFIG_DIR" -> dir.toString), dir).unsafeRunSync() + val missing = CoursierCredentials.load(Map.empty, dir).unsafeRunSync() + assertEquals(fromFile.map(_.getHost), List("filehost")) + assertEquals(fromEnv.map(_.getHost), List("envhost")) + assertEquals(missing, Nil)