diff --git a/.gitignore b/.gitignore index 2995c77b..ad6baec0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ node_modules/ .build/ tests/tool-build/new/myapp/ tests/tool-build/new-lib/mylib/ +tests/tool-build/template-single/myapp/ +tests/tool-build/template-single/myapp-pinned/ +tests/tool-build/template-multi/myapp/ tests/tool-build/*/repo/ tests/tool-build/*/jo.lock tests/tool-build/*/*/jo.lock diff --git a/docs/usage/commands/new.md b/docs/usage/commands/new.md index 8dc464bf..9f39f385 100644 --- a/docs/usage/commands/new.md +++ b/docs/usage/commands/new.md @@ -1,18 +1,71 @@ # jo new -Create a new project with a standard scaffold. +Create a new project with a standard scaffold, or from a third-party +template repo. ## Usage ``` jo new [--lib] +jo new --template +jo new --template --list ``` ## Options -| Option | Description | -|---------|-----------------------------------------| -| `--lib` | Create a library project. Default: app. | +| Option | Description | +|---------------|----------------------------------------------------------| +| `--lib` | Create a library project. Default: app. Not combinable with `--template`. | +| `--template` | Scaffold from a template ref instead of the built-in app/lib scaffold. | +| `--list` | List the templates declared by `--template`'s repo, instead of scaffolding. Requires `--template`, no ``. | + +## Template Scaffold + +`jo new --template ` scaffolds from a third-party GitHub repo +instead of the built-in scaffolds. The repo must have a `jo-templates.jsonl` +manifest at its root declaring one or more named templates — this is what +makes a repo a valid Jo template repo. + +Ref grammar: `[gh:]owner/repo[#gitref][:name]`. `gh:` is optional and +implied when omitted. `gitref` (branch, tag, or commit) defaults to the +repo's default branch. `name` selects a template when the repo declares +more than one. + +``` +jo new --template acme/jo-templates my-app # single-template repo +jo new --template acme/jo-templates:web-app my-app # multi-template repo, pick one +jo new --template acme/jo-templates#v2:web-app my-app # pin a ref +jo new --template acme/jo-templates --list # list available templates +``` + +There is no variable substitution, prompts, or setup hooks — files are +copied as-is. + +### `jo-templates.jsonl` + +One JSON object per line, at the repo root: + +```jsonl +{"name": "web-app", "path": "templates/web-app", "description": "Minimal web app with a REST endpoint"} +{"name": "cli", "path": "templates/cli", "description": "CLI scaffold with arg parsing"} +``` + +| Field | Required | Notes | +|---------------|----------|-----------------------------------------------------------------------| +| `name` | yes | Matches `[a-zA-Z0-9][a-zA-Z0-9_-]*`. Must be unique within the file. | +| `path` | yes | Relative POSIX path from the repo root. No leading `/`, no `..` segments. Everything under it is copied verbatim into the new project. | +| `description` | no | Free text, shown by `--list`. Not used for resolution. | + +A single-template repo still needs this file, just one line, typically: + +```jsonl +{"name": "default", "path": ".", "description": "Minimal starter"} +``` + +With `path: "."`, the entire repo root — including `jo-templates.jsonl` +itself — gets copied into the scaffolded project, so keep any repo-only +files (CI config, contributor docs) in a subdirectory instead if you don't +want them showing up in generated projects. ## App Scaffold diff --git a/stack-lang/cli/Main.scala b/stack-lang/cli/Main.scala index 57fa71f4..b093eb07 100644 --- a/stack-lang/cli/Main.scala +++ b/stack-lang/cli/Main.scala @@ -256,6 +256,8 @@ object Main: println("""Usage: | jo Run program (defaults to 'eval') | jo new Create a new project + | jo new --template Create a new project from a template repo (e.g. 'gh:owner/repo') + | jo new --template --list List templates declared by a template repo | jo clean [module] Remove build artifacts (default: all modules) | jo build [module] Build a module (default module if omitted) | jo check [module] Type-check and compile to sast, skip executable diff --git a/stack-lang/tool/HttpPackageProvider.scala b/stack-lang/tool/HttpPackageProvider.scala index 5afe745d..0e98dc61 100644 --- a/stack-lang/tool/HttpPackageProvider.scala +++ b/stack-lang/tool/HttpPackageProvider.scala @@ -4,7 +4,6 @@ import java.net.URI import java.net.http.{HttpClient, HttpRequest, HttpResponse} import java.nio.file.{Files, Path} import java.util.zip.ZipFile -import scala.collection.mutable import tool.toml.TomlParser @@ -187,7 +186,7 @@ case class HttpPackageProvider( private object ReleaseJson: def parse(line: String): Either[String, ReleaseRecord] = - JsonParser.parseObj(line.trim).flatMap: obj => + Json.parseObj(line.trim).flatMap: obj => for versionStr <- requireStr(obj, "version") url <- requireStr(obj, "url") @@ -213,101 +212,3 @@ private object ReleaseJson: case Some(s: String) => Right(s) case Some(_) => Left(s"'$key' must be a string") case None => Left(s"missing required field '$key'") - - -// ---- Minimal recursive-descent JSON parser ----------------------------------- - -private object JsonParser: - def parseObj(input: String): Either[String, Map[String, Any]] = - parseObject(input, 0) match - case Right((obj, _)) => Right(obj) - case Left(msg) => Left(msg) - - private def ws(s: String, i: Int): Int = - var j = i - while j < s.length && s(j).isWhitespace do j += 1 - j - - private def parseObject(s: String, i0: Int): Either[String, (Map[String, Any], Int)] = - var i = ws(s, i0) - if i >= s.length || s(i) != '{' then return Left(s"expected '{' at $i") - i = ws(s, i + 1) - - val fields = collection.mutable.LinkedHashMap.empty[String, Any] - var first = true - - while i < s.length && s(i) != '}' do - if !first then - if s(i) != ',' then return Left(s"expected ',' at $i") - i = ws(s, i + 1) - first = false - - parseString(s, i) match - case Left(msg) => return Left(msg) - case Right((key, j)) => - i = ws(s, j) - if i >= s.length || s(i) != ':' then return Left(s"expected ':' at $i") - i = ws(s, i + 1) - parseValue(s, i) match - case Left(msg) => return Left(msg) - case Right((v, j2)) => - fields(key) = v - i = ws(s, j2) - - if i >= s.length then Left("unterminated object") - else Right((fields.toMap, i + 1)) - - private def parseValue(s: String, i: Int): Either[String, (Any, Int)] = - if i >= s.length then Left("unexpected end of input") - else s(i) match - case '"' => parseString(s, i) - case '{' => parseObject(s, i) - case '[' => parseArray(s, i) - case 't' if s.startsWith("true", i) => Right((true, i + 4)) - case 'f' if s.startsWith("false", i) => Right((false, i + 5)) - case 'n' if s.startsWith("null", i) => Right((null, i + 4)) - case c if c.isDigit || c == '-' => parseNumber(s, i) - case c => Left(s"unexpected char '$c' at $i") - - private def parseString(s: String, i0: Int): Either[String, (String, Int)] = - if i0 >= s.length || s(i0) != '"' then return Left(s"expected '\"' at $i0") - val sb = new StringBuilder - var i = i0 + 1 - while i < s.length && s(i) != '"' do - if s(i) == '\\' && i + 1 < s.length then - s(i + 1) match - case '"' => sb += '"'; i += 2 - case '\\' => sb += '\\'; i += 2 - case '/' => sb += '/'; i += 2 - case 'n' => sb += '\n'; i += 2 - case 'r' => sb += '\r'; i += 2 - case 't' => sb += '\t'; i += 2 - case c => sb += c; i += 2 - else - sb += s(i) - i += 1 - if i >= s.length then Left("unterminated string") - else Right((sb.toString, i + 1)) - - private def parseArray(s: String, i0: Int): Either[String, (List[Any], Int)] = - var i = ws(s, i0 + 1) - val items = new mutable.ArrayBuffer[Any] - var first = true - while i < s.length && s(i) != ']' do - if !first then - if s(i) != ',' then return Left(s"expected ',' at $i") - i = ws(s, i + 1) - first = false - parseValue(s, i) match - case Left(msg) => return Left(msg) - case Right((v, j)) => - items += v - i = ws(s, j) - if i >= s.length then Left("unterminated array") - else Right((items.toList, i + 1)) - - private def parseNumber(s: String, i0: Int): Either[String, (String, Int)] = - var i = i0 - if i < s.length && s(i) == '-' then i += 1 - while i < s.length && s(i).isDigit do i += 1 - Right((s.substring(i0, i), i)) diff --git a/stack-lang/tool/Json.scala b/stack-lang/tool/Json.scala new file mode 100644 index 00000000..37cc6d9d --- /dev/null +++ b/stack-lang/tool/Json.scala @@ -0,0 +1,155 @@ +package tool + +import scala.collection.mutable + +/** Minimal recursive-descent JSON parser for single-object lines (JSONL). + * + * Deliberately strict rather than permissive: input comes from a registry + * index or a third-party `jo-templates.jsonl` file, both untrusted, so + * parser leniency (accepting `\q`, silently truncating trailing garbage, + * merging duplicate keys) would make malformed or adversarial input harder + * to catch rather than easier. Numbers are represented as `Double` in the + * `Any` result, never as `String` — otherwise a caller checking `case s: + * String` for a required field would wrongly accept `{"name": 123}`. + */ +object Json: + def parseObj(input: String): Either[String, Map[String, Any]] = + parseObject(input, 0).flatMap: (obj, j) => + val end = ws(input, j) + if end != input.length then Left(s"unexpected trailing data at $end") + else Right(obj) + + private def ws(s: String, i: Int): Int = + var j = i + while j < s.length && s(j).isWhitespace do j += 1 + j + + private def parseObject(s: String, i0: Int): Either[String, (Map[String, Any], Int)] = + var i = ws(s, i0) + if i >= s.length || s(i) != '{' then return Left(s"expected '{' at $i") + i = ws(s, i + 1) + + val fields = collection.mutable.LinkedHashMap.empty[String, Any] + var first = true + + while i < s.length && s(i) != '}' do + if !first then + if s(i) != ',' then return Left(s"expected ',' at $i") + i = ws(s, i + 1) + first = false + + parseString(s, i) match + case Left(msg) => return Left(msg) + case Right((key, j)) => + if fields.contains(key) then return Left(s"duplicate key '$key' at $i") + i = ws(s, j) + if i >= s.length || s(i) != ':' then return Left(s"expected ':' at $i") + i = ws(s, i + 1) + parseValue(s, i) match + case Left(msg) => return Left(msg) + case Right((v, j2)) => + fields(key) = v + i = ws(s, j2) + + if i >= s.length then Left("unterminated object") + else Right((fields.toMap, i + 1)) + + private def parseValue(s: String, i: Int): Either[String, (Any, Int)] = + if i >= s.length then Left("unexpected end of input") + else s(i) match + case '"' => parseString(s, i) + case '{' => parseObject(s, i) + case '[' => parseArray(s, i) + case 't' if s.startsWith("true", i) => Right((true, i + 4)) + case 'f' if s.startsWith("false", i) => Right((false, i + 5)) + case 'n' if s.startsWith("null", i) => Right((null, i + 4)) + case c if c.isDigit || c == '-' => parseNumber(s, i) + case c => Left(s"unexpected char '$c' at $i") + + private def parseString(s: String, i0: Int): Either[String, (String, Int)] = + if i0 >= s.length || s(i0) != '"' then return Left(s"expected '\"' at $i0") + val sb = new StringBuilder + var i = i0 + 1 + + while i < s.length && s(i) != '"' do + val c = s(i) + + if c == '\\' then + if i + 1 >= s.length then return Left(s"unterminated escape at $i") + + s(i + 1) match + case '"' => sb += '"'; i += 2 + case '\\' => sb += '\\'; i += 2 + case '/' => sb += '/'; i += 2 + case 'b' => sb += '\b'; i += 2 + case 'f' => sb += '\f'; i += 2 + case 'n' => sb += '\n'; i += 2 + case 'r' => sb += '\r'; i += 2 + case 't' => sb += '\t'; i += 2 + + case 'u' => + if i + 6 > s.length then return Left(s"incomplete unicode escape at $i") + val hex = s.substring(i + 2, i + 6) + val code = + try Integer.parseInt(hex, 16) + catch case _: NumberFormatException => return Left(s"invalid unicode escape '\\u$hex' at $i") + sb += code.toChar + i += 6 + + case other => return Left(s"invalid escape '\\$other' at $i") + + else if c < ' ' then + return Left(s"unescaped control character at $i") + + else + sb += c + i += 1 + + if i >= s.length then Left("unterminated string") + else Right((sb.toString, i + 1)) + + private def parseArray(s: String, i0: Int): Either[String, (List[Any], Int)] = + var i = ws(s, i0 + 1) + val items = new mutable.ArrayBuffer[Any] + var first = true + while i < s.length && s(i) != ']' do + if !first then + if s(i) != ',' then return Left(s"expected ',' at $i") + i = ws(s, i + 1) + first = false + parseValue(s, i) match + case Left(msg) => return Left(msg) + case Right((v, j)) => + items += v + i = ws(s, j) + if i >= s.length then Left("unterminated array") + else Right((items.toList, i + 1)) + + /** Full JSON number grammar: `-`? int frac? exp?, with `int` being either + * `0` or a non-zero digit followed by more digits (no leading zeros). A + * lone `-` or any other malformed shape is rejected here rather than + * silently accepted as a truncated "number". + */ + private def parseNumber(s: String, i0: Int): Either[String, (Double, Int)] = + var i = i0 + if i < s.length && s(i) == '-' then i += 1 + + if i >= s.length || !s(i).isDigit then return Left(s"invalid number at $i0") + + if s(i) == '0' then i += 1 + else while i < s.length && s(i).isDigit do i += 1 + + if i < s.length && s(i) == '.' then + i += 1 + if i >= s.length || !s(i).isDigit then return Left(s"invalid number at $i0") + while i < s.length && s(i).isDigit do i += 1 + + if i < s.length && (s(i) == 'e' || s(i) == 'E') then + i += 1 + if i < s.length && (s(i) == '+' || s(i) == '-') then i += 1 + if i >= s.length || !s(i).isDigit then return Left(s"invalid number at $i0") + while i < s.length && s(i).isDigit do i += 1 + + val text = s.substring(i0, i) + try Right((text.toDouble, i)) + catch case _: NumberFormatException => Left(s"invalid number '$text' at $i0") diff --git a/stack-lang/tool/New.scala b/stack-lang/tool/New.scala index e59c7f38..a5da4117 100644 --- a/stack-lang/tool/New.scala +++ b/stack-lang/tool/New.scala @@ -2,29 +2,132 @@ package tool import java.nio.file.{Files, Path, Paths} -/** Scaffolds a new Jo project in a fresh directory. */ +import tool.template.{TemplateProvider, TemplateRef} + +/** Scaffolds a new Jo project in a fresh directory, either from a built-in + * scaffold (app / `--lib`) or from a third-party template repo + * (`--template`, see templates.md). + */ object New: - private val libOpt = CommandLine.BooleanSetting("--lib", "create a library project") + private val libOpt = CommandLine.BooleanSetting("--lib", "create a library project") + private val templateOpt = CommandLine.OptionStringSetting("--template", "scaffold from a template ref, e.g. 'gh:owner/repo:name'") + private val listOpt = CommandLine.BooleanSetting("--list", "list templates declared by --template's repo instead of scaffolding") - case class Args(name: String, isLib: Boolean) + enum Args: + case Scaffold(name: String, isLib: Boolean, template: Option[TemplateRef]) + case ListTemplates(ref: TemplateRef) def run(args: Array[String]): Unit = - parseArgs(args) match - case Result.Ok(parsed) => - scaffold(parsed.name, parsed.isLib, Paths.get("").toAbsolutePath) match - case Result.Ok(msg) => print(msg) - case Result.Err(msg) => System.err.println(msg); sys.exit(1) - case Result.Err(msg) => System.err.println(msg); sys.exit(1) + val baseDir = Paths.get("").toAbsolutePath + + val result = parseArgs(args).flatMap: + case Args.Scaffold(name, isLib, None) => + scaffold(name, isLib, baseDir) + + case Args.Scaffold(name, isLib, Some(ref)) => + scaffoldFromTemplate(name, ref, baseDir, TemplateProvider.forHost) + + case Args.ListTemplates(ref) => + listTemplates(ref, TemplateProvider.forHost) + + result match + case Result.Ok(msg) => print(msg) + case Result.Err(msg) => System.err.println(s"${Ansi.red("error:")} $msg"); sys.exit(1) def parseArgs(args: Array[String]): Result[Args] = - CommandLine.parse(args, List(libOpt, CommandLine.verboseOpt)).flatMap: parsed => - parsed.positional match - case name :: Nil => - Result.Ok(Args(name, parsed.value(libOpt))) - case Nil => - Result.Err("error: 'jo new' requires a project name") - case arg :: _ => - Result.Err(s"error: unexpected argument '$arg'") + CommandLine.parse(args, List(libOpt, templateOpt, listOpt, CommandLine.verboseOpt)).flatMap: parsed => + val isLib = parsed.value(libOpt) + val list = parsed.value(listOpt) + val templateRaw = parsed.value(templateOpt) + + (templateRaw, isLib, list) match + case (None, _, true) => + Result.Err("'--list' requires '--template'") + + case (Some(_), true, _) => + Result.Err("'--template' cannot be combined with '--lib'") + + case (None, _, false) => + requireName(parsed.positional).map(name => Args.Scaffold(name, isLib, None)) + + case (Some(raw), false, true) => + TemplateRef.parse(raw).flatMap: ref => + if parsed.positional.nonEmpty then + Result.Err("'--list' does not take a project name") + else + Result.Ok(Args.ListTemplates(ref)) + + case (Some(raw), false, false) => + TemplateRef.parse(raw).flatMap: ref => + requireName(parsed.positional).map(name => Args.Scaffold(name, isLib, Some(ref))) + + private def requireName(positional: List[String]): Result[String] = + positional match + case name :: Nil => validateName(name) + case Nil => Result.Err("'jo new' requires a project name") + case arg :: _ => Result.Err(s"unexpected argument '$arg'") + + /** `name` becomes `baseDir.resolve(name)` with no other check beyond + * "does it already exist" — `Path.resolve` on an absolute `name` ignores + * `baseDir` entirely, and a relative `name` containing `..` escapes it. + * Rejecting any path separator (and `.`/`..` themselves) keeps the + * scaffold target always a direct child of `baseDir`, for both this + * path and the built-in (non-template) scaffold. + */ + private def validateName(name: String): Result[String] = + if name.isEmpty then + Result.Err("'jo new' requires a project name") + else if name == "." || name == ".." then + Result.Err(s"invalid project name '$name'") + else if name.contains('/') || name.contains('\\') then + Result.Err(s"invalid project name '$name' (must not contain a path separator)") + else + Result.Ok(name) + + /** Scaffolds `name` from a third-party template. + * + * `resolveProvider` is injectable (defaults to `TemplateProvider.forHost` + * in [[run]]) so tests can substitute a `LocalTemplateProvider` fixture + * without going through host parsing or the network at all. Template-name + * resolution happens inside `provider.fetch` itself, against the same + * fetch that pulls the files — not here — so a manifest can never be + * validated against a different revision than what actually gets copied. + */ + def scaffoldFromTemplate( + name: String, + ref: TemplateRef, + baseDir: Path, + resolveProvider: String => Result[TemplateProvider], + ): Result[String] = + val dir = baseDir.resolve(name) + + if Files.exists(dir) then + return Result.Err(s"directory '$name' already exists") + + for + provider <- resolveProvider(ref.host) + _ <- provider.fetch(ref.identifier, ref.gitref, ref.name, dir) + yield + s"""${Ansi.green("Created")} ${Ansi.blue("'" + name + "'")} ${Ansi.dim(s"from ${ref.canonical}")} + | + |${Ansi.dim("You can now:")} + | ${Ansi.blue("cd")} $name + |""".stripMargin + + /** Lists the templates declared by the repo at `ref`, without scaffolding anything. */ + def listTemplates(ref: TemplateRef, resolveProvider: String => Result[TemplateProvider]): Result[String] = + for + provider <- resolveProvider(ref.host) + entries <- provider.manifest(ref.identifier, ref.gitref) + yield + if entries.isEmpty then + s"${ref.identifier} declares no templates\n" + else + entries + .map: entry => + val desc = entry.description.map(d => s" - $d").getOrElse("") + s" ${entry.name}$desc" + .mkString(s"Templates in ${ref.identifier}:\n", "\n", "\n") def scaffold(name: String, isLib: Boolean, baseDir: Path): Result[String] = val dir = baseDir.resolve(name) @@ -32,7 +135,7 @@ object New: val joConstraint = s"${v.major}.${v.minor}" if Files.exists(dir) then - return Result.Err(s"error: directory '$name' already exists") + return Result.Err(s"directory '$name' already exists") Files.createDirectories(dir.resolve("src")) Files.createDirectories(dir.resolve("tests")) diff --git a/stack-lang/tool/Test.scala b/stack-lang/tool/Test.scala index 9eb40a7d..709f3415 100644 --- a/stack-lang/tool/Test.scala +++ b/stack-lang/tool/Test.scala @@ -2,6 +2,7 @@ package tool import java.nio.file.{Files, FileSystems, Path, Paths} import java.io.{ByteArrayOutputStream, PrintStream} +import java.util.zip.{ZipEntry, ZipOutputStream} import scala.collection.mutable import scala.jdk.CollectionConverters.* @@ -9,6 +10,7 @@ import scala.util.Using import tool.toml.{TomlError, TomlParser} +import tool.template.{GithubTemplateProvider, LocalTemplateProvider, TemplateArchive, TemplateEntry, TemplateManifest, TemplateProvider, TemplateRef} /** Runs all file-based regression tests for the build tool. * @@ -68,10 +70,34 @@ import tool.toml.{TomlError, TomlParser} failed :::= runResolverTests() println() + println("=== Json ===") + failed :::= runJsonTests() + println() + println("=== ReleaseJson ===") failed :::= runReleaseJsonTests() println() + println("=== New (name validation) ===") + failed :::= runNewNameValidationTests() + println() + + println("=== TemplateRef ===") + failed :::= runTemplateRefTests() + println() + + println("=== TemplateManifest ===") + failed :::= runTemplateManifestTests() + println() + + println("=== TemplateArchive ===") + failed :::= runTemplateArchiveTests() + println() + + println("=== GithubTemplateProvider (HTTP) ===") + failed :::= runGithubTemplateProviderTests() + println() + if failed.isEmpty then println("All tool tests passed.") else println(s"FAILED: ${failed.reverse.mkString(" ")}") @@ -169,6 +195,72 @@ private def runResolverTests(): List[Path] = if failed then List(Paths.get("JoResolver")) else Nil +// ---- Json suite ---------------------------------------------------------------- + +/** Input is untrusted (a registry index, or a third-party + * `jo-templates.jsonl`), so the parser is deliberately strict rather than + * permissive — each check here corresponds to a specific leniency bug that + * was found and fixed: trailing garbage, unknown escapes, undecoded + * `\uXXXX`, unescaped control characters, invalid numbers like a bare `-`, + * silently-merged duplicate keys, and numbers being indistinguishable from + * strings in the result. + */ +private def runJsonTests(): List[Path] = + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def isErr(r: Either[String, ?]): Boolean = r.isLeft + + check("a well-formed object parses"): + Json.parseObj("""{"a": "x", "b": true, "c": null, "d": [1, 2.5]}""") == + Right(Map("a" -> "x", "b" -> true, "c" -> null, "d" -> List(1.0, 2.5))) + + check("insignificant trailing whitespace is accepted"): + Json.parseObj("{\"a\":\"x\"} \n") == Right(Map("a" -> "x")) + + check("trailing non-whitespace garbage is rejected, not silently ignored"): + isErr(Json.parseObj("""{"name":"x"} garbage""")) + + check("an unknown escape sequence like '\\q' is rejected"): + isErr(Json.parseObj("""{"a":"\q"}""")) + + check("'\\uXXXX' escapes are decoded, not left as literal characters"): + // Built via concatenation, not a literal "A" in source: Scala (like + // Java) pre-processes \uXXXX unicode escapes in raw source text before + // any tokenization runs, so a literal backslash-u sequence can't survive + // as text in a string literal written directly in this file. + val bs = "\\" + val encoded = bs + "u0041" + bs + "u0042" + bs + "u0043" + Json.parseObj(s"""{"a":"$encoded"}""") == Right(Map("a" -> "ABC")) + + check("an unescaped control character inside a string is rejected"): + isErr(Json.parseObj("{\"a\":\"x\ty\"}")) + + check("a bare '-' is rejected, not accepted as a truncated number"): + isErr(Json.parseObj("""{"a":-}""")) + + check("a duplicate object key is rejected, not silently overwritten"): + isErr(Json.parseObj("""{"a":"x","a":"y"}""")) + + check("numbers are returned as Double, not String, so a string-typed field check can reject them"): + Json.parseObj("""{"a":1}""") match + case Right(obj) => obj.get("a").exists(_.isInstanceOf[Double]) && !obj.get("a").exists(_.isInstanceOf[String]) + case Left(_) => false + + if failed then List(Paths.get("Json")) else Nil + // ---- ReleaseJson suite ------------------------------------------------------- /** Guards the registry JSONL wire format against the shape the registry daemon actually @@ -218,6 +310,431 @@ private def runReleaseJsonTests(): List[Path] = if failed then List(Paths.get("ReleaseJson")) else Nil +// ---- New name-validation suite --------------------------------------------------- + +/** `requireName` is the one place both `New.scaffold` (built-in) and + * `New.scaffoldFromTemplate` get their target directory name from, so + * testing it through `parseArgs` covers the containment fix for both. + */ +private def runNewNameValidationTests(): List[Path] = + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def isErr(r: Result[?], contains: String): Boolean = r match + case Result.Err(msg) => msg.contains(contains) + case _ => false + + check("a plain name is accepted"): + New.parseArgs(Array("myapp")) == Result.Ok(New.Args.Scaffold("myapp", false, None)) + + check("'..' escaping baseDir via a relative name is rejected"): + isErr(New.parseArgs(Array("../evil")), "invalid project name") + + check("an absolute path as name is rejected (Path.resolve would otherwise ignore baseDir entirely)"): + isErr(New.parseArgs(Array("/etc/evil")), "invalid project name") + + check("a bare '..' is rejected"): + isErr(New.parseArgs(Array("..")), "invalid project name") + + check("a bare '.' is rejected"): + isErr(New.parseArgs(Array(".")), "invalid project name") + + check("an empty name is rejected"): + isErr(New.parseArgs(Array("")), "requires a project name") + + check("the same containment check applies on the --template path"): + isErr(New.parseArgs(Array("--template", "gh:acme/repo", "../evil")), "invalid project name") + + if failed then List(Paths.get("New")) else Nil + +// ---- TemplateRef suite --------------------------------------------------------- + +private def runTemplateRefTests(): List[Path] = + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def isErr(r: Result[?], contains: String): Boolean = r match + case Result.Err(msg) => msg.contains(contains) + case _ => false + + check("bare identifier defaults to host 'gh', no name, gitref HEAD"): + TemplateRef.parse("acme/repo") == Result.Ok(TemplateRef("gh", "acme/repo", None, "HEAD")) + + check("':name' selects a template"): + TemplateRef.parse("acme/repo:web-app") == Result.Ok(TemplateRef("gh", "acme/repo", Some("web-app"), "HEAD")) + + check("'#gitref' sits between identifier and ':name'"): + TemplateRef.parse("acme/repo#v2:web-app") == Result.Ok(TemplateRef("gh", "acme/repo", Some("web-app"), "v2")) + + check("'#gitref' alone, no name"): + TemplateRef.parse("acme/repo#v1") == Result.Ok(TemplateRef("gh", "acme/repo", None, "v1")) + + check("explicit 'gh:' host prefix, '#gitref', and ':name' all together"): + TemplateRef.parse("gh:acme/repo#v2:name") == Result.Ok(TemplateRef("gh", "acme/repo", Some("name"), "v2")) + + check("canonical: unpinned, default host — no noise from either default"): + TemplateRef("gh", "acme/repo", None, "HEAD").canonical == "acme/repo" + + check("canonical: a pinned gitref is never dropped, unlike the old success message"): + TemplateRef("gh", "acme/repo", None, "v2").canonical == "acme/repo#v2" + + check("canonical: pinned gitref plus a name"): + TemplateRef("gh", "acme/repo", Some("web-app"), "v2").canonical == "acme/repo#v2:web-app" + + check("unsupported host is a hard error, not a hostname fallback"): + isErr(TemplateRef.parse("gitlab:acme/repo"), "unsupported template host 'gitlab'") + + check("empty ref is an error"): + isErr(TemplateRef.parse(""), "empty template ref") + + check("host prefix with nothing after it is an error"): + isErr(TemplateRef.parse("gh:"), "missing identifier") + + check("empty name after ':' is an error"): + isErr(TemplateRef.parse("acme/repo:"), "empty template name") + + check("empty gitref after '#' is an error"): + isErr(TemplateRef.parse("acme/repo#"), "empty git ref") + + if failed then List(Paths.get("TemplateRef")) else Nil + +// ---- TemplateManifest suite ----------------------------------------------------- + +private def runTemplateManifestTests(): List[Path] = + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def isErr(r: Result[?], contains: String): Boolean = r match + case Result.Err(msg) => msg.contains(contains) + case _ => false + + val valid = + """{"name": "web-app", "path": "templates/web-app", "description": "Web app"} + |{"name": "cli", "path": "templates/cli"} + |""".stripMargin + + val parsedValid = List( + TemplateEntry("web-app", "templates/web-app", Some("Web app")), + TemplateEntry("cli", "templates/cli", None), + ) + + check("parses a valid manifest"): + TemplateManifest.parse(valid) == Result.Ok(parsedValid) + + check("blank lines are skipped"): + TemplateManifest.parse("\n" + valid + "\n") == Result.Ok(parsedValid) + + check("malformed JSON reports the 1-based line number"): + isErr(TemplateManifest.parse("not json"), "malformed line 1 in jo-templates.jsonl") + + check("a missing required field is reported as malformed"): + isErr(TemplateManifest.parse("""{"name": "web-app"}"""), "malformed line 1") + + check("duplicate name is an error"): + val dup = + """{"name": "web-app", "path": "a"} + |{"name": "web-app", "path": "b"}""".stripMargin + isErr(TemplateManifest.parse(dup), "duplicate template name 'web-app'") + + check("invalid template name is an error"): + isErr(TemplateManifest.parse("""{"name": "-bad", "path": "a"}"""), "invalid template name") + + check("absolute path is an error"): + isErr(TemplateManifest.parse("""{"name": "web-app", "path": "/etc/passwd"}"""), "invalid path") + + check("'..' segment in path is an error"): + isErr(TemplateManifest.parse("""{"name": "web-app", "path": "a/../../b"}"""), "invalid path") + + check("an empty path is an error ('.' is the only spelling for the repo root)"): + isErr(TemplateManifest.parse("""{"name": "web-app", "path": ""}"""), "invalid path") + + check("'.' as a path is accepted (the repo root)"): + TemplateManifest.parse("""{"name": "web-app", "path": "."}""") == + Result.Ok(List(TemplateEntry("web-app", ".", None))) + + check("a trailing '/' in a path is accepted (harmless, common directory notation)"): + TemplateManifest.parse("""{"name": "web-app", "path": "templates/web-app/"}""") == + Result.Ok(List(TemplateEntry("web-app", "templates/web-app/", None))) + + check("a doubled '/' in a path is an error"): + isErr(TemplateManifest.parse("""{"name": "web-app", "path": "templates//web-app"}"""), "invalid path") + + check("a present but wrong-typed description is an error, not silently treated as absent"): + isErr(TemplateManifest.parse("""{"name": "web-app", "path": ".", "description": 123}"""), "malformed line 1") + + check("fields beyond name/path/description are allowed and ignored"): + TemplateManifest.parse("""{"name": "web-app", "path": ".", "minJoVersion": "1.5"}""") == + Result.Ok(List(TemplateEntry("web-app", ".", None))) + + check("resolve: explicit name found"): + TemplateManifest.resolve(parsedValid, Some("cli"), "acme/repo") == Result.Ok(parsedValid(1)) + + check("resolve: explicit name not found lists available names"): + isErr(TemplateManifest.resolve(parsedValid, Some("nope"), "acme/repo"), "Available: web-app, cli") + + check("resolve: no name, zero entries is an error"): + isErr(TemplateManifest.resolve(Nil, None, "acme/repo"), "declares no templates") + + check("resolve: no name, exactly one entry resolves to it"): + TemplateManifest.resolve(List(parsedValid.head), None, "acme/repo") == Result.Ok(parsedValid.head) + + check("resolve: no name, multiple entries is an ambiguity error, not a guess"): + isErr(TemplateManifest.resolve(parsedValid, None, "acme/repo"), "declares multiple templates, pick one: web-app, cli") + + if failed then List(Paths.get("TemplateManifest")) else Nil + +// ---- TemplateArchive suite ------------------------------------------------------- + +private def runTemplateArchiveTests(): List[Path] = + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def buildZip(entries: Map[String, String]): Path = + val zipFile = Files.createTempFile("jo-template-test-", ".zip") + val out = ZipOutputStream(Files.newOutputStream(zipFile)) + try + for (name, content) <- entries do + out.putNextEntry(ZipEntry(name)) + out.write(content.getBytes("UTF-8")) + out.closeEntry() + finally out.close() + zipFile + + val manifest = + """{"name": "web-app", "path": "templates/web-app"} + |{"name": "root", "path": "."} + |{"name": "broken-path", "path": "does/not/exist"} + |{"name": "file-path", "path": "README.md"} + |""".stripMargin + + val repoZip = buildZip(Map( + "repo-main/jo-templates.jsonl" -> manifest, + "repo-main/README.md" -> "hello", + "repo-main/templates/web-app/src/Main.jo" -> "def main = println \"web-app\"\n", + )) + + check("resolves ':name' against the manifest found inside the archive, extracts only that subtree, and leaves no staging directory behind"): + val destParent = Files.createTempDirectory("jo-template-test-dest-") + val dest = destParent.resolve("myapp") + TemplateArchive.extract(repoZip, Some("web-app"), dest, "acme/repo at main") match + case Result.Ok(_) => + Files.readString(dest.resolve("src/Main.jo")) == "def main = println \"web-app\"\n" + && !Files.exists(dest.resolve("README.md")) + && Files.list(destParent).iterator.asScala.toList == List(dest) + case Result.Err(_) => false + + check("a failure writing the destination (e.g. it's non-empty and can't be replaced) leaves it untouched and no staging debris behind"): + val parent = Files.createTempDirectory("jo-template-test-parent-") + val dest = parent.resolve("existing") + Files.createDirectory(dest) + Files.writeString(dest.resolve("keep.txt"), "do not touch") + + val source = Files.createTempDirectory("jo-template-test-source-") + Files.writeString(source.resolve("a.txt"), "hi") + + TemplateArchive.copyTree(source, dest) match + case Result.Err(_) => + Files.list(dest).iterator.asScala.toList.map(_.getFileName.toString) == List("keep.txt") + && Files.list(parent).iterator.asScala.toList == List(dest) + case Result.Ok(_) => false + + check("a manifest entry with 'path: .' copies the whole repo root, including jo-templates.jsonl itself"): + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(repoZip, Some("root"), dest, "acme/repo at main") match + case Result.Ok(_) => Files.exists(dest.resolve("README.md")) && Files.exists(dest.resolve("jo-templates.jsonl")) + case Result.Err(_) => false + + check("no name given, multiple entries, is an ambiguity error, not a guess"): + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(repoZip, None, dest, "acme/repo at main") match + case Result.Err(msg) => msg.contains("declares multiple templates") + case _ => false + + check("a requested name absent from the manifest is an error"): + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(repoZip, Some("nope"), dest, "acme/repo at main") match + case Result.Err(msg) => msg.contains("no template 'nope'") + case _ => false + + check("a manifest path absent from the archive is a 'not found' error"): + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(repoZip, Some("broken-path"), dest, "acme/repo at main") match + case Result.Err(msg) => msg.contains("template path 'does/not/exist' not found") + case _ => false + + check("a manifest path pointing at a file, not a directory, is a 'not found' error"): + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(repoZip, Some("file-path"), dest, "acme/repo at main") match + case Result.Err(msg) => msg.contains("not found") + case _ => false + + check("an archive with no jo-templates.jsonl is not a valid Jo template repo"): + val bareZip = buildZip(Map("repo-main/src/Main.jo" -> "def main = println \"hi\"\n")) + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(bareZip, None, dest, "acme/bare at main") match + case Result.Err(msg) => msg.contains("has no jo-templates.jsonl") + case _ => false + + check("zip-slip entries are rejected before the manifest is even read"): + val evilZip = buildZip(Map("repo-main/../../evil.txt" -> "pwned")) + val dest = Files.createTempDirectory("jo-template-test-dest-") + TemplateArchive.extract(evilZip, None, dest, "acme/evil at main") match + case Result.Err(_) => true + case Result.Ok(_) => false + + if failed then List(Paths.get("TemplateArchive")) else Nil + +// ---- GithubTemplateProvider suite (thin HTTP layer only — resolution and ----- +// ---- extraction logic are already covered above with no network involved) ---- + +private def runGithubTemplateProviderTests(): List[Path] = + import com.sun.net.httpserver.{HttpExchange, HttpServer} + import java.net.InetSocketAddress + + var failed = false + + def check(label: String)(body: => Boolean): Unit = + val ok = + try body + catch + case e: Exception => + println(s" threw: ${e.getMessage}") + false + + if ok then println(s" ok: $label") + else + println(s"FAIL: $label") + failed = true + + def respond(exchange: HttpExchange, status: Int, bytes: Array[Byte]): Unit = + exchange.sendResponseHeaders(status, bytes.length) + val os = exchange.getResponseBody + try os.write(bytes) finally os.close() + + def buildZipBytes(entries: Map[String, String]): Array[Byte] = + val buf = ByteArrayOutputStream() + val out = ZipOutputStream(buf) + try + for (name, content) <- entries do + out.putNextEntry(ZipEntry(name)) + out.write(content.getBytes("UTF-8")) + out.closeEntry() + finally out.close() + buf.toByteArray + + val zipBytes = buildZipBytes(Map( + "repo-main/jo-templates.jsonl" -> """{"name": "default", "path": "."}""", + "repo-main/src/Main.jo" -> "def main = println \"ok\"\n", + )) + val manifestBytes = """{"name": "default", "path": "."}""".getBytes("UTF-8") + + val server = HttpServer.create(InetSocketAddress("127.0.0.1", 0), 0) + server.createContext("/acme/repo/HEAD/jo-templates.jsonl", (ex: HttpExchange) => respond(ex, 200, manifestBytes)) + server.createContext("/acme/repo/zip/HEAD", (ex: HttpExchange) => respond(ex, 200, zipBytes)) + server.createContext("/acme/missing/HEAD/jo-templates.jsonl", (ex: HttpExchange) => respond(ex, 404, Array.emptyByteArray)) + server.createContext("/acme/missing/zip/HEAD", (ex: HttpExchange) => respond(ex, 404, Array.emptyByteArray)) + // A context path is matched against the *decoded* request path — if a gitref + // containing '/' or '#' were embedded in the request URL unencoded (or + // encoded incorrectly), these two would never be hit, and the requests + // would 404 or get truncated at the '#' instead. + server.createContext("/acme/slashref/release/1.0/jo-templates.jsonl", (ex: HttpExchange) => respond(ex, 200, manifestBytes)) + server.createContext("/acme/hashref/v1#odd/jo-templates.jsonl", (ex: HttpExchange) => respond(ex, 200, manifestBytes)) + server.start() + + try + val baseUrl = s"http://127.0.0.1:${server.getAddress.getPort}" + val provider = GithubTemplateProvider(baseUrl, baseUrl) + + check("manifest: 200 response parses correctly"): + provider.manifest("acme/repo", "HEAD") == Result.Ok(List(TemplateEntry("default", ".", None))) + + check("manifest: 404 is reported as 'not a valid Jo template repo'"): + provider.manifest("acme/missing", "HEAD") match + case Result.Err(msg) => msg.contains("acme/missing has no jo-templates.jsonl") + case _ => false + + check("fetch: 200 response downloads the zip once and resolves the manifest from inside it"): + val dest = Files.createTempDirectory("jo-template-http-test-") + provider.fetch("acme/repo", "HEAD", None, dest) match + case Result.Ok(_) => Files.exists(dest.resolve("src/Main.jo")) + case Result.Err(_) => false + + check("fetch: 404 is reported with the identifier and ref"): + val dest = Files.createTempDirectory("jo-template-http-test-") + provider.fetch("acme/missing", "HEAD", None, dest) match + case Result.Err(msg) => msg.contains("acme/missing") && msg.contains("HEAD") + case _ => false + + check("manifest: connection failure is a fetch error, not a crash"): + val deadProvider = GithubTemplateProvider("http://127.0.0.1:1", "http://127.0.0.1:1") + deadProvider.manifest("acme/repo", "HEAD") match + case Result.Err(msg) => msg.contains("failed to fetch") + case _ => false + + check("manifest: a '/' in gitref (a slash-containing branch name) reaches the right URL"): + provider.manifest("acme/slashref", "release/1.0") == Result.Ok(List(TemplateEntry("default", ".", None))) + + check("manifest: a '#' in gitref is percent-encoded, not misread as a URL fragment"): + provider.manifest("acme/hashref", "v1#odd") == Result.Ok(List(TemplateEntry("default", ".", None))) + + check("manifest: an identifier with characters outside the safe charset is rejected before any request"): + provider.manifest("ac me/repo", "HEAD") match + case Result.Err(msg) => msg.contains("invalid GitHub identifier") + case _ => false + + finally + server.stop(0) + + if failed then List(Paths.get("GithubTemplateProvider")) else Nil + private def matchesFilter(path: Path, filters: List[String]): Boolean = if filters.isEmpty then true else @@ -344,8 +861,16 @@ private def runJoCmd(subcmd: String, specDir: Path)(using Logger): Result[String // Commands that don't need a build plan if command == "new" then - return New.parseArgs(cmdArgs).flatMap: parsed => - New.scaffold(parsed.name, parsed.isLib, specDir) + val result = New.parseArgs(cmdArgs).flatMap: + case New.Args.Scaffold(name, isLib, None) => + New.scaffold(name, isLib, specDir) + case New.Args.Scaffold(name, isLib, Some(ref)) => + New.scaffoldFromTemplate(name, ref, specDir, testTemplateProvider(specDir)) + case New.Args.ListTemplates(ref) => + New.listTemplates(ref, testTemplateProvider(specDir)) + return result match + case ok @ Result.Ok(_) => ok + case Result.Err(msg) => Result.Err(s"error: $msg\n") if command == "package" then val parsed = Build.parseProjectArgs(cmdArgs) match @@ -457,6 +982,12 @@ private def resolveSpecDir(specFile: String, specDir: Path): String = val resolved = if specPath.isAbsolute then specPath else specDir.resolve(specPath).normalize() resolved.toString +/** Ignores the requested host — every `jo new --template` scenario test + * fixture is a single local repo, so there's nothing to route between. + */ +private def testTemplateProvider(specDir: Path): String => Result[TemplateProvider] = + _ => Result.Ok(LocalTemplateProvider(specDir.resolve("template-src"))) + private def testPackageProvider(specDir: Path): PackageProvider = val repoSrc = specDir.resolve("repo-src") val repoDir = specDir.resolve("repo") diff --git a/stack-lang/tool/template/GithubTemplateProvider.scala b/stack-lang/tool/template/GithubTemplateProvider.scala new file mode 100644 index 00000000..31e73c13 --- /dev/null +++ b/stack-lang/tool/template/GithubTemplateProvider.scala @@ -0,0 +1,132 @@ +package tool.template + +import java.net.URI +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.nio.file.{Files, Path} + +import tool.Result + +/** [[TemplateProvider]] for GitHub (`gh:owner/repo`). + * + * Uses two unauthenticated, CDN-backed endpoints — neither counts against + * `api.github.com` rate limits, so no token is needed. Base URLs are + * constructor parameters, not hardcoded, so tests can point this at a local + * server instead of the real GitHub hosts. + */ +class GithubTemplateProvider( + rawBaseUrl: String = "https://raw.githubusercontent.com", + archiveBaseUrl: String = "https://codeload.github.com", +) extends TemplateProvider: + private val http = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + + def manifest(identifier: String, gitref: String): Result[List[TemplateEntry]] = + parseIdentifier(identifier).flatMap: (owner, repo) => + val url = uri(rawBaseUrl, s"/$owner/$repo/$gitref/jo-templates.jsonl") + + get(url) match + case FetchResult.Ok(body) => + TemplateManifest.parse(body) + + case FetchResult.NotFound => + Result.Err(s"$identifier has no jo-templates.jsonl — not a valid Jo template repo") + + case FetchResult.Failure(msg) => + Result.Err(msg) + + def fetch(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = + parseIdentifier(identifier).flatMap: (owner, repo) => + val url = uri(archiveBaseUrl, s"/$owner/$repo/zip/$gitref") + val tempZip = Files.createTempFile("jo-template-", ".zip") + + try + getToFile(url, tempZip) match + case FetchResult.Ok(_) => + TemplateArchive.extract(tempZip, name, destDir, s"$identifier at $gitref") + + case FetchResult.NotFound => + Result.Err(s"$identifier has no ref '$gitref', or the repo does not exist") + + case FetchResult.Failure(msg) => + Result.Err(msg) + + finally + Files.deleteIfExists(tempZip) + + // ---- Internals --------------------------------------------------------------- + + private val ownerRepoChars = "^[A-Za-z0-9._-]+$".r + + private def parseIdentifier(identifier: String): Result[(String, String)] = + identifier.split("/", -1) match + case Array(owner, repo) if ownerRepoChars.matches(owner) && ownerRepoChars.matches(repo) => + Result.Ok((owner, repo)) + case _ => + Result.Err(s"invalid GitHub identifier '$identifier' (expected 'owner/repo', letters/digits/'.'/'_'/'-' only)") + + /** Builds a request URI from a trusted base (`rawBaseUrl`/`archiveBaseUrl`, + * either the real GitHub hosts or a test server) and a `path` built from + * untrusted input (`gitref` in particular — a git ref can legally contain + * `#`, which `URI.create` on a plain interpolated string would parse as + * the start of a fragment, silently truncating everything after it from + * the actual request). + * + * The multi-argument `URI` constructor percent-encodes characters that + * aren't valid in a path (`#`, spaces, etc.) while still treating `/` as + * a structural separator — which matters because branch names themselves + * can legally contain `/` (e.g. `release/1.0`) and must stay usable as + * path segments rather than being escaped into `%2F`. + */ + private def uri(baseUrl: String, path: String): URI = + val base = URI.create(baseUrl) + URI(base.getScheme, base.getAuthority, path, null, null) + + private enum FetchResult[+A]: + case Ok(value: A) + case NotFound + case Failure(message: String) + + /** `Throwable.getMessage` is nullable when a exception was raised with no + * message, and embedding a literal "null" in an error string reads like + * a bug, not a message — fall back to the exception's class name. + */ + private def describe(e: Exception): String = + Option(e.getMessage).getOrElse(e.getClass.getSimpleName) + + private def get(url: URI): FetchResult[String] = + try + val req = HttpRequest.newBuilder(url).build() + val res = http.send(req, HttpResponse.BodyHandlers.ofString()) + if res.statusCode() == 200 then FetchResult.Ok(res.body()) + else if res.statusCode() == 404 then FetchResult.NotFound + else FetchResult.Failure(s"HTTP ${res.statusCode()}: $url") + catch + case e: InterruptedException => + Thread.currentThread().interrupt() + FetchResult.Failure(s"failed to fetch $url: ${describe(e)}") + + case e: Exception => + FetchResult.Failure(s"failed to fetch $url: ${describe(e)}") + + private def getToFile(url: URI, dest: Path): FetchResult[Unit] = + try + val req = HttpRequest.newBuilder(url).build() + val res = http.send(req, HttpResponse.BodyHandlers.ofFile(dest)) + if res.statusCode() == 200 then FetchResult.Ok(()) + else + Files.deleteIfExists(dest) + if res.statusCode() == 404 then FetchResult.NotFound + else FetchResult.Failure(s"HTTP ${res.statusCode()}: $url") + catch + case e: InterruptedException => + Thread.currentThread().interrupt() + Files.deleteIfExists(dest) + FetchResult.Failure(s"failed to fetch $url: ${describe(e)}") + + case e: Exception => + Files.deleteIfExists(dest) + FetchResult.Failure(s"failed to fetch $url: ${describe(e)}") + +object GithubTemplateProvider: + val default: GithubTemplateProvider = GithubTemplateProvider() diff --git a/stack-lang/tool/template/LocalTemplateProvider.scala b/stack-lang/tool/template/LocalTemplateProvider.scala new file mode 100644 index 00000000..ae75c9ca --- /dev/null +++ b/stack-lang/tool/template/LocalTemplateProvider.scala @@ -0,0 +1,42 @@ +package tool.template + +import java.nio.file.{Files, Path} + +import tool.Result + +/** Filesystem-backed [[TemplateProvider]]: reads `jo-templates.jsonl` and + * template directories straight off `root`, no network, no zip. + * + * Used by scenario tests to exercise the full `jo new --template` pipeline + * (ref parsing through manifest resolution to files landing on disk) + * without a server — mirrors how `LocalPackageProvider` stands in for + * `HttpPackageProvider` in the existing package-manager tests. `identifier` + * and `gitref` only appear in error messages; the fixture content always + * comes from `root`, so — unlike a network provider — there's no + * independent-fetches-can-diverge risk here to begin with, but `fetch` + * still resolves `name` against its own `manifest` call to keep the same + * shape as `GithubTemplateProvider`. + */ +case class LocalTemplateProvider(root: Path) extends TemplateProvider: + def manifest(identifier: String, gitref: String): Result[List[TemplateEntry]] = + val manifestFile = root.resolve("jo-templates.jsonl") + + if !Files.exists(manifestFile) then + Result.Err(s"$identifier has no jo-templates.jsonl — not a valid Jo template repo") + else + TemplateManifest.parse(Files.readString(manifestFile)) + + def fetch(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = + for + entries <- manifest(identifier, gitref) + entry <- TemplateManifest.resolve(entries, name, identifier) + _ <- copyResolved(entry.path, destDir, identifier, gitref) + yield () + + private def copyResolved(path: String, destDir: Path, identifier: String, gitref: String): Result[Unit] = + val source = if path == "." then root else root.resolve(path).normalize() + + if !source.startsWith(root) || !Files.isDirectory(source) then + Result.Err(s"template path '$path' not found in $identifier at $gitref") + else + TemplateArchive.copyTree(source, destDir) diff --git a/stack-lang/tool/template/TemplateArchive.scala b/stack-lang/tool/template/TemplateArchive.scala new file mode 100644 index 00000000..6bd5e2cd --- /dev/null +++ b/stack-lang/tool/template/TemplateArchive.scala @@ -0,0 +1,121 @@ +package tool.template + +import java.nio.file.{Files, Path, StandardCopyOption} +import scala.jdk.CollectionConverters.* +import scala.util.Using + +import tool.{JoyArchive, Result} + +/** Resolves and extracts a template out of a downloaded repo archive. + * + * Kept independent of how the zip was obtained (no HTTP here) so it can be + * unit-tested directly against a hand-built zip fixture. + */ +object TemplateArchive: + /** Unzips `zipFile`, reads its `jo-templates.jsonl`, resolves `name` + * against the manifest found there, and copies the resolved template + * into `destDir`. + * + * The manifest is read from the *same* extracted archive as the files it + * describes — never fetched separately. That matters because `gitref` + * can be a mutable ref (branch, or the default `HEAD`): a caller that + * fetched a manifest and an archive as two independent requests could + * observe two different commits if the branch moved in between, silently + * validating one revision and extracting another. A single archive fetch + * makes that impossible — manifest and files can't diverge if there's + * only ever one download. + * + * Archives are expected to contain exactly one top-level directory (as + * GitHub/GitLab/Gitea archive endpoints always produce) — its name isn't + * predictable ahead of time, so it's discovered rather than assumed. + * Reuses `JoyArchive.unpack` for the actual unzipping, including its + * zip-slip guard. + */ + def extract(zipFile: Path, name: Option[String], destDir: Path, label: String): Result[Unit] = + val tempDir = Files.createTempDirectory("jo-template-") + + try + JoyArchive.unpack(zipFile, tempDir) + + topLevelDir(tempDir) match + case None => + Result.Err(s"unexpected archive structure for $label (expected exactly one top-level directory)") + + case Some(root) => + val manifestFile = root.resolve("jo-templates.jsonl") + + if !Files.exists(manifestFile) then + Result.Err(s"$label has no jo-templates.jsonl — not a valid Jo template repo") + else + for + entries <- TemplateManifest.parse(Files.readString(manifestFile)) + entry <- TemplateManifest.resolve(entries, name, label) + _ <- copyResolved(root, entry.path, destDir, label) + yield () + + catch + case e: Exception => Result.Err(describe(e)) + + finally + deleteRecursively(tempDir) + + private def describe(e: Exception): String = + Option(e.getMessage).getOrElse(e.getClass.getSimpleName) + + private def copyResolved(root: Path, path: String, destDir: Path, label: String): Result[Unit] = + val source = if path == "." then root else root.resolve(path).normalize() + + if !source.startsWith(root) || !Files.isDirectory(source) then + Result.Err(s"template path '$path' not found in $label") + else + copyTree(source, destDir) + + private def topLevelDir(root: Path): Option[Path] = + Using.resource(Files.list(root)): stream => + stream.iterator.asScala.toList match + case single :: Nil if Files.isDirectory(single) => Some(single) + case _ => None + + /** Copies the contents of `source` (a trusted local directory — not zip + * entries, so no traversal guard needed here) into `destDir`. Shared with + * `LocalTemplateProvider`, which copies straight from a fixture root + * rather than an extracted archive. + * + * Stages into a temporary sibling of `destDir` and atomically renames it + * into place only once every file has copied successfully. A disk, I/O, + * or permission failure partway through then leaves no `destDir` at all, + * rather than a half-populated one that would fail the collision check + * on a retry. The staging directory sits next to `destDir` (not under + * the system temp dir) specifically so the rename is same-filesystem, + * and therefore atomic. + */ + def copyTree(source: Path, destDir: Path): Result[Unit] = + val staging = Files.createTempDirectory(destDir.getParent, ".jo-new-staging-") + + try + val entries = Using.resource(Files.walk(source)): stream => + stream.iterator.asScala.filterNot(_ == source).toList + + for entry <- entries do + val target = staging.resolve(source.relativize(entry).toString) + + if Files.isDirectory(entry) then + Files.createDirectories(target) + else + Files.createDirectories(target.getParent) + Files.copy(entry, target, StandardCopyOption.REPLACE_EXISTING) + + Files.move(staging, destDir, StandardCopyOption.ATOMIC_MOVE) + Result.unit + + catch + case e: Exception => + deleteRecursively(staging) + Result.Err(s"failed to write '$destDir': ${describe(e)}") + + private def deleteRecursively(dir: Path): Unit = + if Files.exists(dir) then + Using.resource(Files.walk(dir)): stream => + stream + .sorted(java.util.Comparator.reverseOrder()) + .forEach(Files.delete) diff --git a/stack-lang/tool/template/TemplateManifest.scala b/stack-lang/tool/template/TemplateManifest.scala new file mode 100644 index 00000000..cd14e13a --- /dev/null +++ b/stack-lang/tool/template/TemplateManifest.scala @@ -0,0 +1,117 @@ +package tool.template + +import tool.{Json, Result} + +/** One entry declared in a `jo-templates.jsonl` manifest. + * + * `path` is relative to the repo root; it is never typed by the ref + * consumer (see `TemplateRef`) — only the template author, in the manifest. + * + * Fields beyond `name`/`path`/`description` are allowed and ignored, not + * rejected — that's a deliberate forward-compatibility choice (a future Jo + * version could add an optional field without breaking manifests read by + * today's `jo`), matching how the registry's own JSONL wire format + * (`HttpPackageProvider`'s `ReleaseRecord`) already treats unknown keys. + */ +case class TemplateEntry(name: String, path: String, description: Option[String]) + +object TemplateManifest: + private val nameRegex = "^[a-zA-Z0-9][a-zA-Z0-9_-]*$".r + + /** Parses `jo-templates.jsonl`, one JSON object per line. + * + * All validation (malformed JSON, duplicate names, invalid paths) happens + * eagerly here, before any name resolution — a bad manifest is reported + * once, up front. + */ + def parse(text: String): Result[List[TemplateEntry]] = + text.linesIterator.zipWithIndex.foldLeft(Result.Ok(List.empty[TemplateEntry])): (acc, lineAndIdx) => + acc.flatMap: entries => + val (line, i) = lineAndIdx + + if line.trim.isEmpty then + Result.Ok(entries) + + else + parseLine(line.trim) match + case Left(err) => + Result.Err(s"malformed line ${i + 1} in jo-templates.jsonl: $err") + + case Right(entry) => + validate(entry, entries).map(_ => entries :+ entry) + + /** Resolves which entry a ref selects, per the rules in templates.md: + * an explicit `:name` must match exactly; with no `:name`, the manifest + * must declare exactly one entry. Ambiguity is always an error, never + * guessed. + */ + def resolve(entries: List[TemplateEntry], requestedName: Option[String], repoLabel: String): Result[TemplateEntry] = + requestedName match + case Some(name) => + entries.find(_.name == name) match + case Some(entry) => Result.Ok(entry) + case None => + Result.Err(s"no template '$name' in $repoLabel. Available: ${entries.map(_.name).mkString(", ")}") + + case None => + entries match + case Nil => Result.Err(s"jo-templates.jsonl in $repoLabel declares no templates") + case entry :: Nil => Result.Ok(entry) + case many => Result.Err(s"$repoLabel declares multiple templates, pick one: ${many.map(_.name).mkString(", ")}") + + // ---- Internals --------------------------------------------------------------- + + private def parseLine(line: String): Either[String, TemplateEntry] = + Json.parseObj(line).flatMap: obj => + for + name <- requireStr(obj, "name") + path <- requireStr(obj, "path") + description <- optionalStr(obj, "description") + yield TemplateEntry(name, path, description) + + private def requireStr(obj: Map[String, Any], key: String): Either[String, String] = + obj.get(key) match + case Some(s: String) => Right(s) + case Some(_) => Left(s"'$key' must be a string") + case None => Left(s"missing required field '$key'") + + /** Like `requireStr`, but the field may be absent. A *present* value with + * the wrong type is still an error — `{"description": 123}` must not + * quietly become "no description", which is what naively `.collect`-ing + * only the `String` case (and dropping anything else, absent or not) + * used to do. + */ + private def optionalStr(obj: Map[String, Any], key: String): Either[String, Option[String]] = + obj.get(key) match + case Some(s: String) => Right(Some(s)) + case Some(_) => Left(s"'$key' must be a string") + case None => Right(None) + + private def validate(entry: TemplateEntry, existing: List[TemplateEntry]): Result[Unit] = + if existing.exists(_.name == entry.name) then + Result.Err(s"duplicate template name '${entry.name}' in jo-templates.jsonl") + + else if !nameRegex.matches(entry.name) then + Result.Err(s"invalid template name '${entry.name}' in jo-templates.jsonl (must match [a-zA-Z0-9][a-zA-Z0-9_-]*)") + + else if !validPath(entry.path) then + Result.Err(s"invalid path '${entry.path}' for template '${entry.name}' in jo-templates.jsonl " + + "(must be relative, '.' for the repo root, no leading or doubled '/', no '..' segments)") + + else + Result.unit + + /** `.` (explicitly, the repo root) or a non-empty relative path with no + * leading or doubled `/` and no `..` segment. A trailing `/` is left + * alone — it's common, intentional directory notation that resolves + * identically with or without it, unlike the rejected forms: a leading + * `/` signals an author who thinks the path is filesystem-rooted rather + * than repo-relative, `..` is a real containment concern, `""` would be + * an ambiguous second spelling of `.`, and `//` is never intentional. + */ + private def validPath(path: String): Boolean = + path == "." || + (path.nonEmpty + && !path.startsWith("/") + && !path.contains("//") + && !path.split("/").contains("..")) diff --git a/stack-lang/tool/template/TemplateProvider.scala b/stack-lang/tool/template/TemplateProvider.scala new file mode 100644 index 00000000..1e88da50 --- /dev/null +++ b/stack-lang/tool/template/TemplateProvider.scala @@ -0,0 +1,47 @@ +package tool.template + +import java.nio.file.Path + +import tool.Result + +/** Resolves and fetches third-party `jo new --template` sources. + * + * One implementation per host (`GithubTemplateProvider` for `gh`), plus + * `LocalTemplateProvider`, which reads a fixture straight off disk instead + * of going over the network — that's what lets `jo new --template` + * scenarios be tested without a server. + */ +trait TemplateProvider: + /** The manifest entries declared by `jo-templates.jsonl` at `identifier`/`gitref`. + * + * Used only by `--list`, a standalone lookup with nothing to combine it + * against — unlike `fetch` below, there's no risk in this being a fetch + * of its own. + */ + def manifest(identifier: String, gitref: String): Result[List[TemplateEntry]] + + /** Fetches `identifier`/`gitref` exactly once and populates `destDir` with + * the template `name` resolves to. + * + * `name` must be resolved against the manifest found in that same fetch, + * never one obtained separately — see `TemplateArchive.extract` for why + * that would be unsound when `gitref` is mutable. + */ + def fetch(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] + +object TemplateProvider: + private val byHost: Map[String, TemplateProvider] = Map("gh" -> GithubTemplateProvider.default) + + /** The single source of truth for which host codes `TemplateRef.parse` + * accepts — kept in sync with `byHost` by construction (it's this map's + * key set) rather than as a second list maintained by hand. + */ + val supportedHosts: Set[String] = byHost.keySet + + /** Adding a second host is one new implementation plus one entry here — + * `New.scala` never needs to change. + */ + def forHost(host: String): Result[TemplateProvider] = + byHost.get(host) match + case Some(provider) => Result.Ok(provider) + case None => Result.Err(s"unsupported template host '$host' (supported: ${supportedHosts.toList.sorted.mkString(", ")})") diff --git a/stack-lang/tool/template/TemplateRef.scala b/stack-lang/tool/template/TemplateRef.scala new file mode 100644 index 00000000..cd86f245 --- /dev/null +++ b/stack-lang/tool/template/TemplateRef.scala @@ -0,0 +1,78 @@ +package tool.template + +import tool.Result + +/** A parsed `jo new --template` ref: `[host:]identifier[#gitref][:name]`. + * + * `identifier` is not interpreted here — its shape (e.g. GitHub's flat + * `owner/repo`) is host-specific and owned by that host's [[TemplateProvider]]. + * + * `gitref` sits between `identifier` and `name`, not after `name`, because + * it scopes the whole repo fetch (it's part of the URLs a `TemplateProvider` + * hits) while `name` is a manifest lookup resolved only after that fetch + * completes — the same order GitHub's own `tree/{branch}/{path}` URLs use. + */ +case class TemplateRef(host: String, identifier: String, name: Option[String], gitref: String): + /** Renders back in `[host:]identifier[#gitref][:name]` form. `host` and + * `gitref` are omitted when they're just the defaults (`gh`, `HEAD`) — an + * unpinned ref printing `#HEAD` would be noise, not information. But a + * `gitref` that *was* pinned (`#v2`) always shows: a success message + * built from this must not let a pinned source print as if it weren't. + */ + def canonical: String = + val hostPart = if host == TemplateRef.defaultHost then "" else s"$host:" + val gitrefPart = if gitref == TemplateRef.defaultGitref then "" else s"#$gitref" + s"$hostPart$identifier$gitrefPart" + name.map(n => s":$n").getOrElse("") + +object TemplateRef: + private val defaultHost = "gh" + private val defaultGitref = "HEAD" + + /** Parses `[host:]identifier[#gitref][:name]`. + * + * Proceeds outside-in. The host prefix, if any, is only ever recognized + * before the first `/`, so it's stripped first. `:name` is then split off + * from the end: git ref names can't contain `:` (disallowed by + * `git-check-ref-format`), so any colon remaining after the host prefix + * is unambiguously the name separator, regardless of whether a `#gitref` + * precedes it. What's left after that splits on `#` for `gitref`. + */ + def parse(raw: String): Result[TemplateRef] = + if raw.isEmpty then + return Result.Err(s"empty template ref (expected [host:]identifier[#gitref][:name])") + + val slashIdx = raw.indexOf('/') + val searchLimit = if slashIdx >= 0 then slashIdx else raw.length + val hostColonIdx = raw.indexOf(':') match + case idx if idx >= 0 && idx < searchLimit => Some(idx) + case _ => None + + hostColonIdx match + case Some(idx) => + val host = raw.substring(0, idx) + if TemplateProvider.supportedHosts.contains(host) then parseRest(host, raw.substring(idx + 1)) + else Result.Err(s"unsupported template host '$host' (supported: ${TemplateProvider.supportedHosts.toList.sorted.mkString(", ")})") + + case None => + parseRest(defaultHost, raw) + + private def parseRest(host: String, rest: String): Result[TemplateRef] = + val (beforeName, name) = rest.lastIndexOf(':') match + case idx if idx >= 0 => (rest.substring(0, idx), Some(rest.substring(idx + 1))) + case _ => (rest, None) + + if name.exists(_.isEmpty) then + return Result.Err(s"invalid template ref: empty template name after ':'") + + val (identifier, gitref) = beforeName.indexOf('#') match + case idx if idx >= 0 => (beforeName.substring(0, idx), beforeName.substring(idx + 1)) + case _ => (beforeName, defaultGitref) + + if gitref.isEmpty then + Result.Err(s"invalid template ref: empty git ref after '#'") + + else if identifier.isEmpty then + Result.Err(s"invalid template ref: missing identifier") + + else + Result.Ok(TemplateRef(host, identifier, name, gitref)) diff --git a/tests/tool-build/template-missing-manifest/jo.steps b/tests/tool-build/template-missing-manifest/jo.steps new file mode 100644 index 00000000..42d3f756 --- /dev/null +++ b/tests/tool-build/template-missing-manifest/jo.steps @@ -0,0 +1,9 @@ +rm -rf myapp +jo new --template acme/empty myapp +: ' +error: acme/empty has no jo-templates.jsonl — not a valid Jo template repo +' +jo new --template acme/empty --list +: ' +error: acme/empty has no jo-templates.jsonl — not a valid Jo template repo +' diff --git a/tests/tool-build/template-missing-manifest/template-src/.gitkeep b/tests/tool-build/template-missing-manifest/template-src/.gitkeep new file mode 100644 index 00000000..40f8d633 --- /dev/null +++ b/tests/tool-build/template-missing-manifest/template-src/.gitkeep @@ -0,0 +1 @@ +Intentionally empty: no jo-templates.jsonl, to exercise the "not a valid Jo template repo" error. diff --git a/tests/tool-build/template-multi/jo.steps b/tests/tool-build/template-multi/jo.steps new file mode 100644 index 00000000..133d0206 --- /dev/null +++ b/tests/tool-build/template-multi/jo.steps @@ -0,0 +1,38 @@ +rm -rf myapp myapp2 +jo new --template acme/multi myapp +: ' +error: acme/multi declares multiple templates, pick one: web-app, cli +' +jo new --template acme/multi:web-app myapp +: ' +Created 'myapp' from acme/multi:web-app + +You can now: + cd myapp +' +cat myapp/src/Main.jo +: ' +def main = println "web-app" +' +jo new --template acme/multi:nope myapp2 +: ' +error: no template 'nope' in acme/multi. Available: web-app, cli +' +jo new --template acme/multi --list +: ' +Templates in acme/multi: + web-app - Web app starter + cli +' +jo new --template acme/multi --list myapp +: ' +error: '--list' does not take a project name +' +jo new --list myapp +: ' +error: '--list' requires '--template' +' +jo new --template acme/multi --lib myapp +: ' +error: '--template' cannot be combined with '--lib' +' diff --git a/tests/tool-build/template-multi/template-src/jo-templates.jsonl b/tests/tool-build/template-multi/template-src/jo-templates.jsonl new file mode 100644 index 00000000..f4877230 --- /dev/null +++ b/tests/tool-build/template-multi/template-src/jo-templates.jsonl @@ -0,0 +1,2 @@ +{"name": "web-app", "path": "templates/web-app", "description": "Web app starter"} +{"name": "cli", "path": "templates/cli"} diff --git a/tests/tool-build/template-multi/template-src/templates/cli/src/Main.jo b/tests/tool-build/template-multi/template-src/templates/cli/src/Main.jo new file mode 100644 index 00000000..77bae21e --- /dev/null +++ b/tests/tool-build/template-multi/template-src/templates/cli/src/Main.jo @@ -0,0 +1 @@ +def main = println "cli" diff --git a/tests/tool-build/template-multi/template-src/templates/web-app/src/Main.jo b/tests/tool-build/template-multi/template-src/templates/web-app/src/Main.jo new file mode 100644 index 00000000..c353aebf --- /dev/null +++ b/tests/tool-build/template-multi/template-src/templates/web-app/src/Main.jo @@ -0,0 +1 @@ +def main = println "web-app" diff --git a/tests/tool-build/template-single/jo.steps b/tests/tool-build/template-single/jo.steps new file mode 100644 index 00000000..00dc6630 --- /dev/null +++ b/tests/tool-build/template-single/jo.steps @@ -0,0 +1,28 @@ +rm -rf myapp +jo new --template acme/starter myapp +: ' +Created 'myapp' from acme/starter + +You can now: + cd myapp +' +cat myapp/src/Main.jo +: ' +def main = println "Hello from template!" +' +cat myapp/jo-templates.jsonl +: ' +{"name": "default", "path": ".", "description": "Minimal starter"} +' +jo new --template acme/starter myapp +: ' +error: directory 'myapp' already exists +' +rm -rf myapp-pinned +jo new --template acme/starter#v2 myapp-pinned +: ' +Created 'myapp-pinned' from acme/starter#v2 + +You can now: + cd myapp-pinned +' diff --git a/tests/tool-build/template-single/template-src/jo-templates.jsonl b/tests/tool-build/template-single/template-src/jo-templates.jsonl new file mode 100644 index 00000000..482b2ced --- /dev/null +++ b/tests/tool-build/template-single/template-src/jo-templates.jsonl @@ -0,0 +1 @@ +{"name": "default", "path": ".", "description": "Minimal starter"} diff --git a/tests/tool-build/template-single/template-src/src/Main.jo b/tests/tool-build/template-single/template-src/src/Main.jo new file mode 100644 index 00000000..b53ca42f --- /dev/null +++ b/tests/tool-build/template-single/template-src/src/Main.jo @@ -0,0 +1 @@ +def main = println "Hello from template!"