diff --git a/.buildkite/primer-wasm.yaml b/.buildkite/primer-wasm.yaml new file mode 100644 index 000000000..e261466a2 --- /dev/null +++ b/.buildkite/primer-wasm.yaml @@ -0,0 +1,9 @@ +agents: + public: "true" + os: "linux" + +steps: + - label: ":haskell: :linux: Primer Wasm targets" + command: | + nix develop .#wasm --print-build-logs --command make wasm32-update + nix develop .#wasm --print-build-logs --command make wasm32-test-opt diff --git a/Makefile b/Makefile index 2141bf247..df723a504 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,15 @@ $(targets): $(MAKE) -C primer-service $@ $(MAKE) -C primer-benchmark $@ +wasm32-update: + wasm32-wasi-cabal update + +wasm32 = wasm32-build wasm32-build-opt wasm32-configure wasm32-check wasm32-test wasm32-test-opt wasm32-clean + +$(wasm32): + $(MAKE) -C primer $@ + $(MAKE) -C primer-api $@ + weeder: cabal build all --enable-benchmarks --enable-tests weeder @@ -21,4 +30,4 @@ openapi.json: build cabal run -v0 primer-service:exe:primer-openapi > $@ openapi-generator-cli validate --recommend -i $@ -.PHONY: $(targets) weeder +.PHONY: $(targets) $(wasm32-targets) weeder diff --git a/cabal.project b/cabal.project index 1d0d940f1..729f37b45 100644 --- a/cabal.project +++ b/cabal.project @@ -9,7 +9,7 @@ packages: optimization: 0 -allow-newer: hedgehog-classes:hedgehog +allow-newer: hedgehog-classes:hedgehog,hedgehog-classes:pretty-show,hedgehog:pretty-show package * ghc-options: -fwrite-ide-info @@ -24,6 +24,10 @@ package primer-api package primer-service test-options: "--size-cutoff=32768" +if arch(wasm32) + package tasty + flags: -unix + -- We need a newer version of Selda than what's been released to -- Hackage, plus some GHC 9.6 fixes from a community fork. source-repository-package @@ -39,3 +43,34 @@ source-repository-package tag: 54c12169ce8cd46a7b3c698f65cea55e41a13fe6 subdir: selda-sqlite --sha256: 0q8m8asmb83mpa3j3adlrhc446xif7gv6lql20gv05k33lmbjfhg + +-- Wasm workarounds. + +-- Upstream requires `happy` at build time, which doesn't work on Wasm +-- targets. +source-repository-package + type: git + location: https://github.com/hackworthltd/pretty-show + tag: 91d119cb0e3c5f7d866589b25158739580c8fc88 + --sha256: sha256-mu8Eq0Sg6nCF8C2sXB6ebZcLhz8TVZAbNMiorA7RVc8= + +-- Upstream depends on Posix types unavailable in Wasm. +source-repository-package + type: git + location: https://github.com/hackworthltd/semirings + tag: 369f696d9d00fe004b16b0de08888fee7a3d08c3 + --sha256: sha256-kkHCp4Y9IqMXGaDyW5UpsmRjy0ZWZkVSo1nOhpgZUQ0= + +-- Upstream uses custom setup, which breaks on Wasm. +source-repository-package + type: git + location: https://github.com/cdepillabout/pretty-simple + tag: 6fb9b281800ad045925c7344ceb9fd293d86c3b9 + --sha256: sha256-1gsYj/iznEUCeQ1f5Xk7w54h9FLJSNrIR9V3p4eaRYk= + +-- Upstream doesn't want to support Wasm while it's "experimental." +source-repository-package + type: git + location: https://github.com/amesgen/splitmix + tag: 83b906c4bcdc2720546f1779a16eb65e8e12ecba + --sha256: sha256-sR+Ne56SBzVbPfC7AJeQZn20YDfFwBDpRI873cTm1nU= diff --git a/docs/development-guide-toc.md b/docs/development-guide-toc.md index 890ec830a..c1dbde450 100644 --- a/docs/development-guide-toc.md +++ b/docs/development-guide-toc.md @@ -27,3 +27,4 @@ the various packages' Haddocks. * [Primitives](primitives.md) * [Database ops](database.md) * [Benchmarking](benchmarking.md) +* [WebAssembly support](wasm.md) diff --git a/docs/wasm.md b/docs/wasm.md new file mode 100644 index 000000000..dc764f6e6 --- /dev/null +++ b/docs/wasm.md @@ -0,0 +1,43 @@ +# WebAssembly (Wasm) support + +**Note**: WebAssembly support is currently very preliminary. + +For horizontal scalability reasons, we would like to run the `primer` +and `primer-api` packages in the student's browser, rather than on a +backend server. Therefore, we'd like to compile these packages to a +`wasm32-wasi` target and call the (native) Primer API from TypeScript. + +Currently, we can compile these two packages to `wasm32-wasi`, but +with the following caveats: + +1. Neither `haskell.nix` nor `nixpkgs.haskellPackages` support the + `wasm32-wasi` cross-target at the moment, so we can only build Wasm + targets directly via `wasm32-wasi-cabal` and `wasm32-wasi-ghc`. For + interactive development, we provide a special `nix develop` shell + which provides the necessary tools: + + ```sh + nix develop .#wasm + ``` + + Note that the required tools are currently only available for + `x86_64-linux` Nix systems, so the special `wasm` Nix shell only + exists for that platform. + +2. Once you're in the special Wasm shell, it's advisable to use the + special `wasm32` `Makefile` targets. To build the libraries, run: + + ```sh + make wasm32-configure + make wasm32-build + ``` + + To build the tests and run them using the `wasmtime` runtime, run: + + ```sh + make wasm32-test + ``` + + The `wasm32-test`, in particular, needs to run several steps that + you'd otherwise need to run by hand in order to work around + `wasmtime` issues. diff --git a/flake.lock b/flake.lock index ab1df44aa..7fe3cdbbe 100644 --- a/flake.lock +++ b/flake.lock @@ -204,6 +204,24 @@ "inputs": { "systems": "systems" }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, "locked": { "lastModified": 1685518550, "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", @@ -218,9 +236,9 @@ "type": "github" } }, - "flake-utils_2": { + "flake-utils_3": { "inputs": { - "systems": "systems_2" + "systems": "systems_3" }, "locked": { "lastModified": 1685518550, @@ -253,6 +271,25 @@ "type": "github" } }, + "ghc-wasm": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1698055010, + "narHash": "sha256-OGk0mIHtIbGQi2zON+xqyXAsobSdVMgC/7jcFLvWAMo=", + "ref": "refs/heads/master", + "rev": "c0aa3bb7d88bb6ec809210e17658dd1ed64ba66c", + "revCount": 136, + "type": "git", + "url": "https://gitlab.haskell.org/ghc/ghc-wasm-meta" + }, + "original": { + "type": "git", + "url": "https://gitlab.haskell.org/ghc/ghc-wasm-meta" + } + }, "ghc980": { "flake": false, "locked": { @@ -581,7 +618,7 @@ "nix": { "inputs": { "lowdown-src": "lowdown-src", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs_3", "nixpkgs-regression": "nixpkgs-regression" }, "locked": { @@ -601,7 +638,7 @@ }, "nix-darwin": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" }, "locked": { "lastModified": 1696360011, @@ -656,15 +693,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687274257, - "narHash": "sha256-TutzPriQcZ8FghDhEolnHcYU2oHIG5XWF+/SUBNnAOE=", - "path": "/nix/store/22qgs3skscd9bmrxv9xv4q5d4wwm5ppx-source", - "rev": "2c9ecd1f0400076a4d6b2193ad468ff0a7e7fdc5", - "type": "path" + "lastModified": 1697723726, + "narHash": "sha256-SaTWPkI8a5xSHX/rrKzUe+/uVNy6zCGMXgoeMb7T9rg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7c9cc5a6e5d38010801741ac830a3f8fd667a7a0", + "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs-2003": { @@ -864,6 +904,19 @@ } }, "nixpkgs_2": { + "locked": { + "lastModified": 1687274257, + "narHash": "sha256-TutzPriQcZ8FghDhEolnHcYU2oHIG5XWF+/SUBNnAOE=", + "path": "/nix/store/22qgs3skscd9bmrxv9xv4q5d4wwm5ppx-source", + "rev": "2c9ecd1f0400076a4d6b2193ad468ff0a7e7fdc5", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1657693803, "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", @@ -899,7 +952,7 @@ "pre-commit-hooks-nix": { "inputs": { "flake-compat": "flake-compat_3", - "flake-utils": "flake-utils", + "flake-utils": "flake-utils_2", "gitignore": "gitignore", "nixpkgs": [ "hacknix", @@ -924,7 +977,7 @@ "pre-commit-hooks-nix_2": { "inputs": { "flake-compat": "flake-compat_5", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils_3", "gitignore": "gitignore_2", "nixpkgs": [ "nixpkgs" @@ -949,6 +1002,7 @@ "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "ghc-wasm": "ghc-wasm", "hacknix": "hacknix", "haskell-nix": "haskell-nix", "nixpkgs": [ @@ -1003,6 +1057,21 @@ "repo": "default", "type": "github" } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index c5f387603..2541a2bd4 100644 --- a/flake.nix +++ b/flake.nix @@ -19,6 +19,8 @@ nixpkgs.follows = "haskell-nix/nixpkgs-unstable"; hacknix.inputs.nixpkgs.follows = "nixpkgs"; pre-commit-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + + ghc-wasm.url = "git+https://gitlab.haskell.org/ghc/ghc-wasm-meta"; }; outputs = inputs@ { flake-parts, ... }: @@ -305,7 +307,25 @@ }) // primerFlake.apps; - devShells.default = primerFlake.devShell; + devShells = { + default = primerFlake.devShell; + } // (pkgs.lib.optionalAttrs (system == "x86_64-linux")) { + # Unfortunately, this is only available on x86_64-linux. + wasm = pkgs.mkShell { + packages = with inputs.ghc-wasm.packages.${system}; + [ + wasm32-wasi-ghc-9_6 + wasm32-wasi-cabal-9_6 + wasmtime + + pkgs.gnumake + + # We need to run native `tasty-discover` at compile + # time, because we can't do it via `wasmtime`. + (pkgs.haskell-nix.tool ghcVersion "tasty-discover" { }) + ]; + }; + }; # This is a non-standard flake output, but we don't want to # include benchmark runs in `packages`, because we don't diff --git a/primer-api/.gitignore b/primer-api/.gitignore new file mode 100644 index 000000000..3e255afe5 --- /dev/null +++ b/primer-api/.gitignore @@ -0,0 +1 @@ +test/TestsWasm32.hs diff --git a/primer-api/Makefile b/primer-api/Makefile index 3eb72da32..0ff34f4c8 100644 --- a/primer-api/Makefile +++ b/primer-api/Makefile @@ -3,16 +3,37 @@ # Most commands assume you're running this from the top-level `nix # develop` shell. +wasm32-primer-api-test := $(shell wasm32-wasi-cabal list-bin -v0 test:primer-api-test) +wasm32-primer-api-test-opt := $(shell wasm32-wasi-cabal list-bin -O2 -v0 test:primer-api-test) + build: cabal build +wasm32-build: + wasm32-wasi-cabal build + +wasm32-build-opt: + wasm32-wasi-cabal build -O2 + configure: cabal configure +wasm32-configure: + wasm32-wasi-cabal configure + check: test -test: - cabal test +wasm32-check: wasm32-test + +wasm32-test: + tasty-discover test/Test.hs _ test/TestsWasm32.hs --tree-display + wasm32-wasi-cabal build test:primer-api-test + wasmtime --dir test::test "$(wasm32-primer-api-test)" + +wasm32-test-opt: wasm32-build-opt + tasty-discover test/Test.hs _ test/TestsWasm32.hs --tree-display + wasm32-wasi-cabal build -O2 test:primer-api-test + wasmtime --dir test::test "$(wasm32-primer-api-test-opt)" docs: cabal haddock @@ -20,6 +41,9 @@ docs: clean: cabal clean +wasm32-clean: + wasm32-wasi-cabal clean + bench: realclean: diff --git a/primer-api/primer-api.cabal b/primer-api/primer-api.cabal index b6b06895e..a82770245 100644 --- a/primer-api/primer-api.cabal +++ b/primer-api/primer-api.cabal @@ -101,8 +101,6 @@ library primer-api-testlib , stm-containers test-suite primer-api-test - type: exitcode-stdio-1.0 - main-is: Test.hs hs-source-dirs: test other-modules: Tests.API @@ -119,38 +117,45 @@ test-suite primer-api-test OverloadedLists OverloadedStrings - ghc-options: - -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates - -Wcompat -Widentities -Wredundant-constraints - -Wmissing-deriving-strategies -fhide-source-paths -threaded - -rtsopts -with-rtsopts=-N - - if impl(ghcjs) - buildable: False + if arch(wasm32) + type: exitcode-stdio-1.0 + main-is: TestsWasm32.hs + ghc-options: + -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wcompat -Widentities -Wredundant-constraints + -Wmissing-deriving-strategies -fhide-source-paths else - build-depends: - , base - , bytestring - , containers - , hedgehog - , logging-effect - , mtl - , optics - , pretty-simple ^>=4.1 - , primer-api - , primer-api-testlib - , primer:{primer, primer-hedgehog, primer-testlib} - , protolude - , stm - , stm-containers - , tasty ^>=1.4.2.1 - , tasty-discover - , tasty-golden ^>=2.3.5 - , tasty-hunit - , text - , transformers - , uuid-types ^>=1.0.5.1 + type: exitcode-stdio-1.0 + main-is: Test.hs + ghc-options: + -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wcompat -Widentities -Wredundant-constraints + -Wmissing-deriving-strategies -fhide-source-paths -threaded + -rtsopts -with-rtsopts=-N + + build-depends: + , base + , bytestring + , containers + , hedgehog + , logging-effect + , mtl + , optics + , pretty-simple ^>=4.1 + , primer-api + , primer-api-testlib + , primer:{primer, primer-hedgehog, primer-testlib} + , protolude + , stm + , stm-containers + , tasty ^>=1.4.2.1 + , tasty-discover + , tasty-golden ^>=2.3.5 + , tasty-hunit + , text + , transformers + , uuid-types ^>=1.0.5.1 --TODO This currently breaks with haskell.nix, so we manually add it to `flake.nix` instead. -- See: https://github.com/input-output-hk/haskell.nix/issues/839 diff --git a/primer-selda/primer-selda.cabal b/primer-selda/primer-selda.cabal index 4d9cb54d8..d6b5391e7 100644 --- a/primer-selda/primer-selda.cabal +++ b/primer-selda/primer-selda.cabal @@ -105,27 +105,23 @@ test-suite primer-selda-test -Wmissing-deriving-strategies -fhide-source-paths -threaded -rtsopts -with-rtsopts=-N - if impl(ghcjs) - buildable: False - - else - build-depends: - , aeson - , base - , containers - , exceptions - , filepath - , logging-effect - , primer-selda:{primer-selda, primer-selda-testlib} - , primer:{primer, primer-testlib} - , selda - , selda-sqlite - , tasty ^>=1.4.2.1 - , tasty-discover ^>=5.0 - , tasty-hunit ^>=0.10.0 - , text - , time - , uuid-types + build-depends: + , aeson + , base + , containers + , exceptions + , filepath + , logging-effect + , primer-selda:{primer-selda, primer-selda-testlib} + , primer:{primer, primer-testlib} + , selda + , selda-sqlite + , tasty ^>=1.4.2.1 + , tasty-discover ^>=5.0 + , tasty-hunit ^>=0.10.0 + , text + , time + , uuid-types --TODO This currently breaks with haskell.nix, so we manually add it to `flake.nix` instead. -- See: https://github.com/input-output-hk/haskell.nix/issues/839 diff --git a/primer-service/src/Primer/OpenAPI.hs b/primer-service/src/Primer/OpenAPI.hs index 7a6cddff7..b640dd2f2 100644 --- a/primer-service/src/Primer/OpenAPI.hs +++ b/primer-service/src/Primer/OpenAPI.hs @@ -96,9 +96,10 @@ deriving via PrimerJSON a instance (Generic a, GToJSON Zero (Rep a), GToEncoding -- $orphanInstances -- -- We define some OpenApi orphan instances in primer-service, to avoid --- pulling in the openapi3 dependency into primer core. This is necessary to --- build primer with ghcjs, because openapi3 transitively depends on network, --- which ghcjs currently cannot build. +-- pulling in the openapi3 dependency into primer core. This is +-- necessary to build primer with ghcjs and/or the wasm32-wasi target, +-- because openapi3 transitively depends on network, which these +-- targets currently cannot build. -- Suitable for deriving via, when the ToJSON instance is via PrimerJSON instance diff --git a/primer/.gitignore b/primer/.gitignore new file mode 100644 index 000000000..3e255afe5 --- /dev/null +++ b/primer/.gitignore @@ -0,0 +1 @@ +test/TestsWasm32.hs diff --git a/primer/Makefile b/primer/Makefile index fae76a93e..3596213ea 100644 --- a/primer/Makefile +++ b/primer/Makefile @@ -3,16 +3,37 @@ # Most commands assume you're running this from the top-level `nix # develop` shell. +wasm32-primer-test := $(shell wasm32-wasi-cabal list-bin -v0 test:primer-test) +wasm32-primer-test-opt := $(shell wasm32-wasi-cabal list-bin -O2 -v0 test:primer-test) + build: cabal build +wasm32-build: + wasm32-wasi-cabal build + +wasm32-build-opt: + wasm32-wasi-cabal build -O2 + configure: cabal configure +wasm32-configure: + wasm32-wasi-cabal configure + check: test -test: - cabal test +wasm32-check: wasm32-test + +wasm32-test: + tasty-discover test/Test.hs _ test/TestsWasm32.hs --tree-display + wasm32-wasi-cabal build test:primer-test + wasmtime --dir test::test "$(wasm32-primer-test)" + +wasm32-test-opt: wasm32-build-opt + tasty-discover test/Test.hs _ test/TestsWasm32.hs --tree-display + wasm32-wasi-cabal build -O2 test:primer-test + wasmtime --dir test::test "$(wasm32-primer-test-opt)" # Update any test files which differ from the expected result. serialization-outputs: @@ -26,6 +47,9 @@ docs: clean: cabal clean +wasm32-clean: + wasm32-wasi-cabal clean + bench: realclean: diff --git a/primer/primer.cabal b/primer/primer.cabal index 9d9d5164d..dfb2a17c5 100644 --- a/primer/primer.cabal +++ b/primer/primer.cabal @@ -116,7 +116,6 @@ library , exceptions >=0.10.4 && <0.11.0 , extra >=1.7.10 && <1.8.0 , generic-optics >=2.0 && <2.3.0 - , JuicyPixels ^>=3.3.8 , list-t >=1.0 && <1.1.0 , logging-effect ^>=1.4 , mmorph ^>=1.2.0 @@ -206,8 +205,6 @@ library primer-testlib , tasty-hunit ^>=0.10.0 test-suite primer-test - type: exitcode-stdio-1.0 - main-is: Test.hs hs-source-dirs: test other-modules: Tests.Action @@ -257,45 +254,52 @@ test-suite primer-test OverloadedLists OverloadedStrings - ghc-options: - -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates - -Wcompat -Widentities -Wredundant-constraints - -Wmissing-deriving-strategies -fhide-source-paths -threaded - -rtsopts -with-rtsopts=-N - - if impl(ghcjs) - buildable: False + if arch(wasm32) + type: exitcode-stdio-1.0 + main-is: TestsWasm32.hs + ghc-options: + -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wcompat -Widentities -Wredundant-constraints + -Wmissing-deriving-strategies -fhide-source-paths else - build-depends: - , aeson - , aeson-pretty ^>=0.8.9 - , base - , bytestring - , containers - , extra - , filepath - , hedgehog - , hedgehog-classes ^>=0.2.5.4 - , logging-effect - , mtl - , optics - , pretty-simple ^>=4.1 - , prettyprinter - , prettyprinter-ansi-terminal - , primer - , primer-hedgehog - , primer-testlib - , protolude - , tasty ^>=1.4.2.1 - , tasty-discover - , tasty-golden ^>=2.3.5 - , tasty-hedgehog - , tasty-hunit - , text - , transformers - , uniplate - , uuid-types + type: exitcode-stdio-1.0 + main-is: Test.hs + ghc-options: + -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wcompat -Widentities -Wredundant-constraints + -Wmissing-deriving-strategies -fhide-source-paths -threaded + -rtsopts -with-rtsopts=-N + + build-depends: + , aeson + , aeson-pretty ^>=0.8.9 + , base + , bytestring + , containers + , extra + , filepath + , hedgehog + , hedgehog-classes ^>=0.2.5.4 + , logging-effect + , mtl + , optics + , pretty-simple ^>=4.1 + , prettyprinter + , prettyprinter-ansi-terminal + , primer + , primer-hedgehog + , primer-testlib + , protolude + , tasty ^>=1.4.2.1 + , tasty-discover + , tasty-golden ^>=2.3.5 + , tasty-hedgehog + , tasty-hunit + , text + , transformers + , uniplate + , uuid-types --TODO This currently breaks with haskell.nix, so we manually add it to `flake.nix` instead. -- See: https://github.com/input-output-hk/haskell.nix/issues/839