Job queue, matrix builder, concurrency control

.env.example

@@ -1,38 +1,44 @@
-# =====
-# Dev Configuration
-# The devShell reads this file to set defaults, so changing values here
-# affects local development.
-# =====
+# -----------------------------------------------------------------------------
+# Server Configuration (dev defaults, required in all environments)
+# -----------------------------------------------------------------------------
 
-# Server port - used by both the server and E2E tests
+# Port the registry server listens on
+# - Dev/Test: 9000 (from this file)
+# - Prod: Set in deployment config
 SERVER_PORT=9000
 
 # SQLite database path (relative to working directory)
+# - Dev: Uses local ./db directory
+# - Test: Overridden to use temp state directory
+# - Prod: Set to production database path
 DATABASE_URL="sqlite:db/registry.sqlite3"
 
-# =====
-# Dev Secrets
-# these must be set in .env when running scripts like legacy-importer
-# =====
+# -----------------------------------------------------------------------------
+# Secrets (required for production, use dummy values for local dev)
+# -----------------------------------------------------------------------------
+# IMPORTANT: Never commit real secrets. The values below are dummies for testing.
 
-# GitHub personal access token for API requests when running scripts
-GITHUB_TOKEN="ghp_your_personal_access_token"
-
-# =====
-# Prod Secrets
-# these must be set in .env to run the production server and some scripts
-# =====
-
-# DigitalOcean Spaces credentials for S3-compatible storage
-SPACES_KEY="digitalocean_spaces_key"
-SPACES_SECRET="digitalocean_spaces_secret"
-
-# Pacchettibotti bot account credentials
-# Used for automated registry operations (commits, releases, etc.)
+# GitHub personal access token for pacchettibotti bot
+# Used for: commits to registry repos, issue management
 PACCHETTIBOTTI_TOKEN="ghp_pacchettibotti_token"
 
 # Pacchettibotti SSH keys (base64-encoded)
+# Used for: signing authenticated operations (unpublish, transfer)
 # Generate with: ssh-keygen -t ed25519 -C "[email protected]"
 # Encode with: cat key | base64 | tr -d '\n'
 PACCHETTIBOTTI_ED25519_PUB="c3NoLWVkMjU1MTkgYWJjeHl6IHBhY2NoZXR0aWJvdHRpQHB1cmVzY3JpcHQub3Jn"
 PACCHETTIBOTTI_ED25519="YWJjeHl6"
+
+# DigitalOcean Spaces credentials for S3-compatible storage
+# Used for: uploading/downloading package tarballs
+SPACES_KEY="digitalocean_spaces_key"
+SPACES_SECRET="digitalocean_spaces_secret"
+
+
+# -----------------------------------------------------------------------------
+# Script-only Secrets (not used by server, used by scripts like legacy-importer)
+# -----------------------------------------------------------------------------
+
+# Personal GitHub token for API requests when running scripts
+# This is YOUR token, not pacchettibotti's
+GITHUB_TOKEN="ghp_your_personal_access_token"

AGENTS.md

@@ -10,26 +10,63 @@ This project uses Nix with direnv. You should already be in the Nix shell automa
 nix develop
 ```
 
-### Build and Test
+Watch out for these Nix quirks:
+- If Nix tries to fetch from git during a build, it is likely that spago.yaml files were changed but the lock file was not updated; if so, update the lockfile with `spago build`
+- If a Nix build appears to be stale, then it is likely files were modified but are untracked by Git; if so, add modified files with `git add` and retry.
 
-The registry is implemented in PureScript. Use spago to build it and run PureScript tests. These are cheap and fast and should be used when working on the registry packages.
+### Build
+
+The registry is implemented in PureScript. Use spago to build it.
 
 ```sh
 spago build  # Build all PureScript code
-spago test   # Run unit tests
 ```
 
-Integration tests require two terminals (or the use of test-env in detached mode). The integration tests are only necessary to run if working on the server (app).
+The registry infrastructure is defined in Nix. Build it with Nix:
+
+```sh
+nix build .#server
+```
+
+### Test
+
+The registry contains a mixture of unit tests, e2e tests, and nix flake checks. When you complete a change you should generally run the unit tests. When working on the server, you should generally also run the e2e tests. If you are on a Linux system, you can run `nix flake check -L` to run the flake checks prior to committing code to ensure it works.
+
+#### Unit Tests
+
+Unit tests can be run with `spago`. They are fast and cheap.
+
+```sh
+spago test   # Run all unit tests
+spago test -p <package-name>  # Run tests for a specific package
+```
+
+#### End-to-End Tests
+
+The end-to-end (integration) tests are in `app-e2e`. They can be run via Nix on Linux:
+
+```sh
+nix build .#checks.x86_64-linux.integration
+```
+
+Alternately, they can be run on macOS or for more iterative development of tests using two terminals: one to start the test env, and one to execute the tests.
 
 ```sh
 # Terminal 1: Start test environment (wiremock mocks + registry server on port 9000)
 nix run .#test-env
 
 # Terminal 2: Run E2E tests once server is ready
-spago run -p registry-app-e2e
+spago-test-e2e
 ```
 
-Options: `nix run .#test-env -- --tui` for interactive TUI, `-- --detached` for background mode.
+Options: `nix run .#test-env -- --tui` for interactive TUI, `-- --detached` for background mode to use a single terminal.
+
+State is stored in `/tmp/registry-test-env` and cleaned up on each `nix run .#test-env`. To examine state after a test run (for debugging), stop the test-env but don't restart it. This is useful, for example, to read the logs of the most recent run. For example:
+
+```sh
+# after a test run, see the logs (log name is today's date)
+cat /tmp/registry-test-env/scratch/logs/*.log
+```
 
 #### Smoke Test (Linux only)
 

CONTRIBUTING.md

@@ -72,20 +72,29 @@ nix build .#checks.x86_64-linux.smoke -L
 
 ### Integration Test
 
+You can run the integration tests with the following on Linux:
+
+```sh
+nix build .#checks.x86_64-linux.integration -L
+```
+
+On macOS or for iterative development, you can instead start the test environment and run the tests separately.
+
 ```sh
 # Terminal 1: Start the test environment (wiremock mocks + registry server)
 nix run .#test-env
 
-# Terminal 2: Once the server is ready, run the E2E tests
-spago run -p registry-app-e2e
+# Terminal 2: Run E2E tests once server is ready
+spago-test-e2e
 ```
 
 The test environment:
 - Starts wiremock services mocking GitHub, S3, Pursuit, etc.
-- Starts the registry server on port 9000 with a temporary SQLite database
+- Starts the registry server with a temporary SQLite database
 - Uses fixture data from `app/fixtures/`
+- State is stored in `/tmp/registry-test-env` and cleaned up on each `nix run .#test-env`
 
-Press `Ctrl+C` in Terminal 1 to stop all services. State is cleaned up automatically.
+Press `Ctrl+C` in Terminal 1 to stop all services.
 
 All arguments after `--` are passed directly to process-compose:
 
@@ -101,7 +110,11 @@ process-compose attach           # Attach TUI
 process-compose down             # Stop all services
 ```
 
-You can also set `STATE_DIR` to use a persistent state directory instead of a temp dir.
+To examine state after a test run (e.g., for debugging), stop the test-env but don't restart it. The state remains in `/tmp/registry-test-env`:
+- `db/registry.sqlite3` — SQLite database
+- `scratch/registry/` — Local registry clone with metadata
+- `scratch/registry-index/` — Local manifest index clone
+- `repo-fixtures/` — Git fixture repositories
 
 ## Available Nix Commands
 

SPEC.md

@@ -197,6 +197,7 @@ All packages in the registry contain a `purs.json` manifest file in their root d
 - `version`: a valid [`Version`](#version)
 - `license`: a valid [`License`](#license)
 - `location`: a valid [`Location`](#location)
+- `ref`: a `string` representing the reference (e.g., a Git commit or Git tag) at the `location` that was used to fetch this version's source code
 - `owners` (optional): a non-empty array of [`Owner`](#owner)
 - `description` (optional): a description of your library as a plain text string, not markdown, up to 300 characters
 - `includeFiles` (optional): a non-empty array of globs, where globs are used to match file paths (in addition to the `src` directory and other [always-included files](#always-included-files)) that you want included in your package tarball
@@ -221,6 +222,7 @@ For example:
     "githubOwner": "purescript",
     "githubRepo": "purescript-control"
   },
+  "ref": "v4.2.0",
   "include": ["test/**/*.purs"],
   "exclude": ["test/graphs"],
   "dependencies": { "newtype": ">=3.0.0 <4.0.0", "prelude": ">=4.0.0 <5.0.0" }

app-e2e/spago.yaml

@@ -5,16 +5,27 @@ package:
   dependencies:
     - aff
     - arrays
+    - codec-json
     - console
     - datetime
-    - effect
-    - either
-    - maybe
-    - prelude
+    - exceptions
+    - fetch
+    - integers
+    - json
+    - node-child-process
+    - node-execa
+    - node-fs
+    - node-path
+    - node-process
+    - ordered-collections
+    - registry-app
+    - registry-foreign
     - registry-lib
     - registry-test-utils
+    - routing-duplex
     - spec
     - spec-node
     - strings
+    - transformers
   run:
     main: Test.E2E.Main

app-e2e/src/Test/E2E/Endpoint/Jobs.purs

@@ -0,0 +1,63 @@
+module Test.E2E.Endpoint.Jobs (spec) where
+
+import Registry.App.Prelude
+
+import Data.Array as Array
+import Registry.API.V1 (JobId(..))
+import Registry.API.V1 as V1
+import Registry.Test.Assert as Assert
+import Test.E2E.Support.Client as Client
+import Test.E2E.Support.Env (E2ESpec)
+import Test.E2E.Support.Env as Env
+import Test.E2E.Support.Fixtures as Fixtures
+import Test.Spec as Spec
+
+spec :: E2ESpec
+spec = do
+  Spec.describe "Status endpoint" do
+    Spec.it "can reach the status endpoint" do
+      Client.getStatus
+
+  Spec.describe "Jobs API" do
+    Spec.it "query parameters and filtering work correctly" do
+      -- Publish once and test all Jobs API features
+      { jobId } <- Client.publish Fixtures.effectPublishData
+      job <- Env.pollJobOrFail jobId
+      let info = V1.jobInfo job
+
+      -- Test: include_completed filtering
+      recentJobs <- Client.getJobsWith Client.ActiveOnly
+      allJobs <- Client.getJobsWith Client.IncludeCompleted
+      let allCount = Array.length allJobs
+      Assert.shouldSatisfy allCount (_ > 0)
+      let recentCount = Array.length recentJobs
+      Assert.shouldSatisfy recentCount (_ <= allCount)
+      let completedJob = Array.find (\j -> isJust (V1.jobInfo j).finishedAt) allJobs
+      case completedJob of
+        Just completed -> do
+          let
+            completedId = (V1.jobInfo completed).jobId
+            inRecent = Array.any (\j -> (V1.jobInfo j).jobId == completedId) recentJobs
+          when inRecent do
+            Assert.fail $ "Completed job " <> unwrap completedId <> " should be excluded from include_completed=false results"
+        Nothing -> pure unit
+
+      -- Test: query parameters (level and since)
+      baseJob <- Client.getJob jobId Nothing Nothing
+      Assert.shouldEqual (V1.jobInfo baseJob).jobId info.jobId
+      debugJob <- Client.getJob jobId (Just V1.Debug) Nothing
+      Assert.shouldEqual (V1.jobInfo debugJob).jobId info.jobId
+      let sinceTime = fromMaybe info.createdAt info.finishedAt
+      sinceJob <- Client.getJob jobId Nothing (Just sinceTime)
+      Assert.shouldEqual (V1.jobInfo sinceJob).jobId info.jobId
+
+    Spec.it "returns HTTP 404 for non-existent job ID" do
+      let fakeJobId = JobId "nonexistent-job-id-12345"
+      result <- Client.tryGetJob fakeJobId Nothing Nothing
+      case result of
+        Right _ ->
+          Assert.fail "Expected HTTP 404 for non-existent job"
+        Left err ->
+          case Client.clientErrorStatus err of
+            Just 404 -> pure unit
+            _ -> Assert.fail $ "Expected HTTP 404, got: " <> Client.printClientError err

app-e2e/src/Test/E2E/Endpoint/PackageSets.purs

@@ -0,0 +1,52 @@
+module Test.E2E.Endpoint.PackageSets (spec) where
+
+import Registry.App.Prelude
+
+import Control.Monad.Reader (ask)
+import Effect.Aff as Aff
+import Registry.API.V1 as V1
+import Registry.Test.Assert as Assert
+import Test.E2E.Support.Client as Client
+import Test.E2E.Support.Env (E2ESpec)
+import Test.E2E.Support.Env as Env
+import Test.E2E.Support.Fixtures as Fixtures
+import Test.Spec as Spec
+
+spec :: E2ESpec
+spec = do
+  Spec.describe "Package Sets endpoint" do
+    Spec.it "accepts unauthenticated add/upgrade requests" do
+      { jobId } <- Client.packageSets Fixtures.packageSetAddRequest
+      job <- Env.pollJobOrFail jobId
+      Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
+
+    Spec.it "rejects unauthenticated compiler change requests" do
+      result <- Client.tryPackageSets Fixtures.packageSetCompilerChangeRequest
+      case result of
+        Left err -> do
+          Assert.shouldSatisfy (Client.clientErrorStatus err) (_ == Just 400)
+        Right _ ->
+          Assert.fail "Expected 400 error for unauthenticated compiler change"
+
+    Spec.it "rejects unauthenticated package removal requests" do
+      result <- Client.tryPackageSets Fixtures.packageSetRemoveRequest
+      case result of
+        Left err -> do
+          Assert.shouldSatisfy (Client.clientErrorStatus err) (_ == Just 400)
+        Right _ ->
+          Assert.fail "Expected 400 error for unauthenticated package removal"
+
+    Spec.it "accepts authenticated compiler change requests" do
+      { privateKey } <- ask
+      case Fixtures.signPackageSet privateKey Fixtures.packageSetCompilerChangeRequest of
+        Left err ->
+          liftAff $ Aff.throwError $ Aff.error $ "Failed to sign request: " <> err
+        Right signedRequest -> do
+          { jobId } <- Client.packageSets signedRequest
+          job <- Env.pollJobOrFail jobId
+          Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
+
+    Spec.it "returns existing job for duplicate requests" do
+      { jobId: firstJobId } <- Client.packageSets Fixtures.packageSetAddRequest
+      { jobId: secondJobId } <- Client.packageSets Fixtures.packageSetAddRequest
+      Assert.shouldEqual firstJobId secondJobId